chore: import upstream snapshot with attribution
Docker Image CI / build-ubuntu2004 (push) Waiting to run
Docker Image CI / build-ubuntu2004 (push) Waiting to run
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,276 @@
|
||||
---
|
||||
name: trt-cpp-runtime-quickstart
|
||||
description: >-
|
||||
Load and run a TensorRT engine (.plan / .engine) from C++ using the
|
||||
TensorRT 11 / 10.x **modern Runtime API**, avoiding the deprecated TRT
|
||||
8.x binding-index APIs that older guidance still promotes. Use whenever
|
||||
the user asks about loading
|
||||
or running a TensorRT .plan/.engine from C++, even on "minimal example"
|
||||
requests — without this skill the default reply uses deprecated
|
||||
enqueueV2-style code. Also use when the user hits "Engine plan file
|
||||
is generated on an incompatible device", deserializeCudaEngine returns
|
||||
nullptr, gets an enqueueV2 / IStreamReader deprecation warning, or
|
||||
wants to stream a .plan via IStreamReaderV2. Triggers: TensorRT C++
|
||||
inference, load TensorRT plan C++, run .plan from C++, IRuntime
|
||||
example, deserializeCudaEngine, enqueueV3, enqueueV2 deprecated,
|
||||
setTensorAddress, getBindingIndex, IStreamReaderV2, libnvinfer C++.
|
||||
NOT for building engines (`trt-onnx-quickstart`), Python deploy,
|
||||
plugins, multi-GPU.
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: NVIDIA Corporation
|
||||
version: "1.0"
|
||||
tags:
|
||||
- tensorrt
|
||||
- cpp
|
||||
- inference
|
||||
- deployment
|
||||
- runtime
|
||||
---
|
||||
|
||||
# TensorRT C++ Runtime Deploy
|
||||
|
||||
Load a serialized TensorRT engine from disk and run inference from C++ using only the modern Runtime API. Produces a minimal, copy-pasteable deploy harness that drops next to any `.plan` / `.engine` file and extends to production.
|
||||
|
||||
Reference samples to open before writing new code:
|
||||
|
||||
- `quickstart/SemanticSegmentation/tutorial-runtime.cpp` — cleanest minimal load-and-run example. Mirrors Steps 1–7 below.
|
||||
- `samples/sampleOnnxMNIST/sampleOnnxMNIST.cpp` — end-to-end sample that also builds the engine; the runtime portion shows realistic I/O wiring.
|
||||
- Public headers: `include/NvInferRuntime.h` — read `IRuntime`, `ICudaEngine`, `IExecutionContext`, `IStreamReaderV2`.
|
||||
|
||||
## When to Use
|
||||
|
||||
| Situation | Use this skill? |
|
||||
|-------------------------------------------------------------------------------------------|-----------------|
|
||||
| You have a `.plan`/`.engine` and need to run it from a C++ binary | Yes |
|
||||
| You need a minimal harness that uses `enqueueV3` + `setTensorAddress` | Yes |
|
||||
| You want to load an engine from a `std::istream` or large file via `IStreamReaderV2` | Yes |
|
||||
| You need to wire dynamic shapes (`setInputShape`) before inference | Yes |
|
||||
| You are *building* / optimizing the engine (calibration, INT8, sparsity, builder configs) | No - use trtexec or `IBuilder` directly |
|
||||
| You are deploying in Python | No - use `tensorrt` Python bindings |
|
||||
| You are writing a plugin (`IPluginV3`) or custom layer | No - separate plugin skill |
|
||||
| You need multi-GPU, MPS, MIG, or process-level orchestration | No - out of scope |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **TensorRT installed.** Verify `NvInferRuntime.h` is on the include path
|
||||
and `libnvinfer.so` is on the link path. On a TRT dev container these are
|
||||
in `/usr/include/x86_64-linux-gnu/` and `/usr/lib/x86_64-linux-gnu/` (or
|
||||
`/opt/tensorrt/...` for tarball installs).
|
||||
2. **CUDA toolkit available.** `cuda_runtime_api.h` and `libcudart.so` must
|
||||
be reachable; `nvcc --version` should match the CUDA version the engine
|
||||
was built against.
|
||||
3. **A serialized engine.** A `.plan`/`.engine` file built **on the same
|
||||
major TRT version and the same GPU architecture (compute capability) you
|
||||
will deploy on**. Engines are not portable across major TRT versions or
|
||||
across SMs unless the builder was given `--hardwareCompatibilityLevel`.
|
||||
4. **The engine's I/O tensor names.** Inspect with:
|
||||
```bash
|
||||
trtexec --loadEngine=model.plan --verbose 2>&1 | grep -E 'Input|Output'
|
||||
```
|
||||
5. A C++17 compiler (`g++ >= 9` or `clang++ >= 10`).
|
||||
|
||||
## Step 1: Create the IRuntime
|
||||
|
||||
The runtime owns engine deserialization and must outlive every
|
||||
`ICudaEngine` it creates. Construct one per process for typical deployments.
|
||||
|
||||
```cpp
|
||||
class Logger : public nvinfer1::ILogger {
|
||||
public:
|
||||
void log(Severity severity, char const* msg) noexcept override {
|
||||
if (severity <= Severity::kWARNING) {
|
||||
std::cerr << msg << std::endl;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Logger gLogger;
|
||||
std::unique_ptr<nvinfer1::IRuntime> runtime{
|
||||
nvinfer1::createInferRuntime(gLogger)};
|
||||
if (!runtime) throw std::runtime_error("createInferRuntime failed");
|
||||
```
|
||||
|
||||
A custom logger is mandatory - TensorRT does not log internally. Keep it
|
||||
process-global so deserialization warnings (version skew, calibrator
|
||||
mismatch) are not lost.
|
||||
|
||||
## Step 2: Read the Plan into Memory
|
||||
|
||||
For small/medium engines (< ~1 GiB) read the whole file into a
|
||||
`std::vector<char>` and hand the pointer to
|
||||
`IRuntime::deserializeCudaEngine(blob, size)`. This is what the
|
||||
`SemanticSegmentation` tutorial does and the simplest correct path:
|
||||
|
||||
```cpp
|
||||
std::ifstream f(planPath, std::ios::binary);
|
||||
if (!f) throw std::runtime_error("cannot open " + planPath);
|
||||
f.seekg(0, std::ios::end);
|
||||
auto size = static_cast<size_t>(f.tellg());
|
||||
f.seekg(0, std::ios::beg);
|
||||
std::vector<char> blob(size);
|
||||
if (!f.read(blob.data(), size))
|
||||
throw std::runtime_error("short read on " + planPath);
|
||||
```
|
||||
|
||||
For very large engines, or when the bytes live behind a stream (HTTP,
|
||||
mmap'd archive, encrypted store), implement an `IStreamReaderV2` - see
|
||||
Step 3.
|
||||
|
||||
## Step 3 (optional): Use IStreamReaderV2 for Streaming Loads
|
||||
|
||||
`IStreamReader` (v1) is **deprecated in TensorRT 11.0**. Always use
|
||||
`IStreamReaderV2`: it reads into both host and device memory and is the
|
||||
only stream-reader form guaranteed for new code. Subclass and implement
|
||||
`read(...)` and `seek(...)`:
|
||||
|
||||
```cpp
|
||||
class FileStreamReader : public nvinfer1::IStreamReaderV2 {
|
||||
public:
|
||||
explicit FileStreamReader(std::string const& path)
|
||||
: mFile(path, std::ios::binary) {
|
||||
if (!mFile) throw std::runtime_error("open failed: " + path);
|
||||
}
|
||||
int64_t read(void* dst, int64_t n,
|
||||
cudaStream_t /*stream*/) noexcept override {
|
||||
mFile.read(static_cast<char*>(dst), n);
|
||||
return mFile.gcount();
|
||||
}
|
||||
bool seek(int64_t off, nvinfer1::SeekPosition where) noexcept override {
|
||||
auto dir = (where == nvinfer1::SeekPosition::kSET) ? std::ios::beg
|
||||
: (where == nvinfer1::SeekPosition::kCUR) ? std::ios::cur
|
||||
: std::ios::end;
|
||||
mFile.clear();
|
||||
mFile.seekg(off, dir);
|
||||
return static_cast<bool>(mFile);
|
||||
}
|
||||
private:
|
||||
std::ifstream mFile;
|
||||
};
|
||||
|
||||
FileStreamReader rd{planPath};
|
||||
std::unique_ptr<nvinfer1::ICudaEngine> engine{
|
||||
runtime->deserializeCudaEngine(rd)};
|
||||
```
|
||||
|
||||
## Step 4: Deserialize and Create an Execution Context
|
||||
|
||||
`ICudaEngine` is thread-safe for read-only queries; `IExecutionContext`
|
||||
is **not** - allocate one per inference thread.
|
||||
|
||||
```cpp
|
||||
std::unique_ptr<nvinfer1::ICudaEngine> engine{
|
||||
runtime->deserializeCudaEngine(blob.data(), blob.size())};
|
||||
if (!engine) throw std::runtime_error("deserializeCudaEngine failed");
|
||||
|
||||
std::unique_ptr<nvinfer1::IExecutionContext> ctx{
|
||||
engine->createExecutionContext()};
|
||||
if (!ctx) throw std::runtime_error("createExecutionContext failed");
|
||||
```
|
||||
|
||||
## Step 5: Wire Tensors with setTensorAddress
|
||||
|
||||
Enumerate I/O tensors via `getNbIOTensors()` + `getIOTensorName(i)`. Use
|
||||
`getTensorIOMode`, `getTensorDataType`, and `getTensorShape` to size and
|
||||
allocate buffers. **Set every tensor address before `enqueueV3`** - the
|
||||
modern API has no implicit binding-index map.
|
||||
|
||||
```cpp
|
||||
for (int i = 0; i < engine->getNbIOTensors(); ++i) {
|
||||
char const* name = engine->getIOTensorName(i);
|
||||
auto mode = engine->getTensorIOMode(name);
|
||||
auto shape = engine->getTensorShape(name); // -1 = dynamic dim
|
||||
if (mode == nvinfer1::TensorIOMode::kINPUT && hasDynamic(shape)) {
|
||||
// Fill in concrete shape, e.g. batch=1
|
||||
shape.d[0] = 1;
|
||||
ctx->setInputShape(name, shape);
|
||||
}
|
||||
}
|
||||
// After setInputShape on all dynamic inputs, query output shapes.
|
||||
for (int i = 0; i < engine->getNbIOTensors(); ++i) {
|
||||
char const* name = engine->getIOTensorName(i);
|
||||
auto bytes = elementCount(ctx->getTensorShape(name))
|
||||
* dtypeSize(engine->getTensorDataType(name));
|
||||
void* dev = nullptr;
|
||||
cudaMalloc(&dev, bytes);
|
||||
ctx->setTensorAddress(name, dev);
|
||||
}
|
||||
```
|
||||
|
||||
Always call `setInputShape` for dynamic inputs **before** querying output
|
||||
shapes - the latter depends on the former.
|
||||
|
||||
## Step 6: Run enqueueV3
|
||||
|
||||
`enqueueV3(stream)` is the only non-deprecated enqueue API;
|
||||
`enqueueV2`/`execute*` are gone in modern flows.
|
||||
|
||||
```cpp
|
||||
cudaStream_t stream{};
|
||||
cudaStreamCreate(&stream);
|
||||
|
||||
cudaMemcpyAsync(devInput, hostInput, inBytes,
|
||||
cudaMemcpyHostToDevice, stream);
|
||||
if (!ctx->enqueueV3(stream))
|
||||
throw std::runtime_error("enqueueV3 failed");
|
||||
cudaMemcpyAsync(hostOutput, devOutput, outBytes,
|
||||
cudaMemcpyDeviceToHost, stream);
|
||||
cudaStreamSynchronize(stream);
|
||||
```
|
||||
|
||||
If you reuse buffers across iterations, skip the per-call
|
||||
`setTensorAddress` - addresses persist on the context until overwritten.
|
||||
|
||||
## Step 7: Shutdown Order
|
||||
|
||||
Destroy in reverse construction order: contexts -> engines -> runtime,
|
||||
then free CUDA memory and destroy the stream. With `std::unique_ptr` this
|
||||
is automatic as long as the context is declared *after* the engine, and
|
||||
the engine *after* the runtime. Free `cudaMalloc` allocations explicitly
|
||||
(RAII wrapper recommended).
|
||||
|
||||
## Build
|
||||
|
||||
Wire the steps above into your application's build system. For a standalone smoke test, a minimal build is:
|
||||
|
||||
```bash
|
||||
g++ -std=c++17 runtime.cpp -o run -lnvinfer -lcudart # adjust CUDA/TRT include + lib paths
|
||||
./run model.plan
|
||||
```
|
||||
|
||||
## Common Errors
|
||||
|
||||
| Symptom | Likely cause |
|
||||
|----------------------------------------------------------------------|------------------------------------------------------------------------------|
|
||||
| `deserializeCudaEngine` returns `nullptr`, log says "version tag" | Engine built on a different TRT major version. Rebuild on the deploy version |
|
||||
| `nullptr` with "engine plan file is generated on an incompatible device" | SM mismatch. Rebuild on the target SM or use `--hardwareCompatibilityLevel` |
|
||||
| `enqueueV3` returns false, log mentions "Tensor X has no address" | Forgot `setTensorAddress` for one of the I/O tensors |
|
||||
| `enqueueV3` false, "shape" in message | Forgot `setInputShape` for a dynamic input, or supplied an out-of-profile shape |
|
||||
| `cudaErrorIllegalAddress` on H->D / D->H copy | Mismatched element count / dtype between host buffer and engine tensor |
|
||||
| Process crashes inside TRT during destruction | Wrong destruction order - context outlived engine, or engine outlived runtime |
|
||||
| `cudaErrorMemoryAllocation` during context creation | Workspace too big for the device; rebuild with smaller workspace |
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **Do not use `IStreamReader` v1.** Deprecated in TRT 11.0. Use
|
||||
`IStreamReaderV2` (note `cudaStream_t` parameter on `read`).
|
||||
- **Do not use `enqueueV2` / `execute` / binding indices.** These are
|
||||
legacy paths; the only stable modern path is name-based
|
||||
`setTensorAddress` + `enqueueV3`.
|
||||
- **One `IExecutionContext` per thread.** Sharing contexts across threads
|
||||
is undefined behavior; sharing the engine is fine.
|
||||
- **Stream lifetime.** The CUDA stream passed to `enqueueV3` must outlive
|
||||
the inference. Destroying it while work is in flight crashes or corrupts
|
||||
output.
|
||||
- **Async vs sync copies.** Mixing synchronous `cudaMemcpy` with
|
||||
`enqueueV3` on a stream serializes the GPU; always pair `enqueueV3`
|
||||
with `cudaMemcpyAsync` on the same stream.
|
||||
- **Engine portability.** A `.plan` is tied to (TRT major version, GPU SM,
|
||||
CUDA major version). Never check engines into a repo without recording
|
||||
these three facts.
|
||||
- **Logger lifetime.** The logger passed to `createInferRuntime` must
|
||||
outlive the runtime; a stack-local logger in `main` is fine, a function-
|
||||
scope local is a use-after-free.
|
||||
- **Refit / weight streaming.** Engines built with refit or weight
|
||||
streaming enabled need extra setup calls (`setWeightStreamingBudgetV2`,
|
||||
`IRefitter`); out of scope here.
|
||||
@@ -0,0 +1,343 @@
|
||||
---
|
||||
name: trt-onnx-quickstart
|
||||
description: >
|
||||
Build and verify a TensorRT engine from a Hugging Face model ID or ONNX
|
||||
file, with numerical parity checked against ONNX Runtime. Use when the
|
||||
user imports a non-LLM model to TensorRT,
|
||||
needs a verified engine from ONNX, hits trtexec "unsupported operator",
|
||||
must verify the engine matches ONNX numerically, debugs a polygraphy
|
||||
parity failure (large max abs diff at FP16), or configures multi-input
|
||||
dynamic shapes. Triggers: convert ONNX to TensorRT, Hugging Face to
|
||||
TensorRT, trtexec onnx, trtexec unsupported operator, optimum-cli
|
||||
export, polygraphy parity check, polygraphy run --trt --onnxrt, parity
|
||||
check failed, max abs diff, verify engine matches ONNX, --minShapes,
|
||||
dynamic shapes trtexec, multi-input shape profile, FP16 engine, INT64
|
||||
warning. Adjacent skills: `trt-torch-quickstart` (PyTorch frontend),
|
||||
`trt-cpp-runtime-quickstart` (C++ engine load). LLM token generation
|
||||
belongs in TensorRT-LLM, not here.
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: NVIDIA Corporation
|
||||
version: "1.0"
|
||||
tags:
|
||||
- onnx
|
||||
- import
|
||||
- huggingface
|
||||
- quickstart
|
||||
- fp16
|
||||
---
|
||||
|
||||
# TensorRT ONNX Quickstart
|
||||
|
||||
Take a developer from "I have a Hugging Face model ID" or "an ONNX file" to "a TensorRT engine whose outputs match the source model within tolerance." Follows Path 1 (ONNX → TensorRT) of the [Import Workflows Guide](https://github.com/NVIDIA/TensorRT/blob/main/documents/import_workflows.md), specialized for the most common starting point: a Hugging Face Hub model.
|
||||
|
||||
## When to Use
|
||||
|
||||
| Scenario | Use this skill? |
|
||||
|----------|-----------------|
|
||||
| Has a Hugging Face model ID (`google-bert/bert-base-uncased`) and wants TRT-accelerated inference | Yes |
|
||||
| Has an `.onnx` file and wants a `.plan` engine with verified parity | Yes |
|
||||
| Ran `trtexec --onnx=...` and hit a warning/error they don't recognize | Yes |
|
||||
| Wants LLM token generation (Llama, Mistral, Qwen text generation) | **No** — route to [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) |
|
||||
| Has a PyTorch model and wants to stay in PyTorch | No — use `trt-torch-quickstart` |
|
||||
| Already has a `.plan` engine and wants to run inference from C++ | No — use `trt-cpp-runtime-quickstart` |
|
||||
| Is migrating a weakly-typed network to strongly-typed | No — use `trt-strong-typing-migration` |
|
||||
| Has a full diffusion pipeline (SD, FLUX) | Partial — must be component-split first; this skill imports one component at a time |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Verify each before continuing. Failures here surface as confusing errors later, so fix them before running the import.
|
||||
|
||||
1. **NVIDIA GPU + driver matching TensorRT 11.x.** TRT 11 requires CUDA 13.x. Confirm:
|
||||
```bash
|
||||
nvidia-smi # driver + CUDA runtime version
|
||||
```
|
||||
2. **Python ≥ 3.10.** TRT 11 dropped 3.9 and earlier.
|
||||
3. **TensorRT 11 installed — including the `trtexec` CLI**, this skill's primary build tool. **The pip wheel does not ship `trtexec`.** Pick one:
|
||||
|
||||
- **NGC container (recommended for first-time users):** `nvcr.io/nvidia/tensorrt:<tag>` — bundles `trtexec`, Python bindings, libraries, and most common dependencies. Run with `--gpus all`.
|
||||
- **`.tar.gz` or `.deb` from [TensorRT downloads](https://developer.nvidia.com/tensorrt-download):** installs `trtexec` to a system path.
|
||||
|
||||
The pip path (`pip install --extra-index-url https://pypi.nvidia.com tensorrt-cu13`) provides Python bindings but not `trtexec`, so it is **insufficient on its own**. For pip-only, route the developer to the alternative Python-builder flow in `import_workflows.md` Path 1 Option B; this skill assumes `trtexec` is available.
|
||||
|
||||
Verification:
|
||||
```bash
|
||||
python3 -c "import tensorrt; print(tensorrt.__version__)" # expect 11.x.y.z
|
||||
trtexec --help | head -5 # MUST work; if not, install is incomplete
|
||||
```
|
||||
4. **Polygraphy and `optimum-onnx` installed.**
|
||||
```bash
|
||||
pip install polygraphy onnx onnxruntime onnx_graphsurgeon onnxsim 'optimum-onnx'
|
||||
# If the model ID starts with `sentence-transformers/`, also:
|
||||
pip install sentence-transformers
|
||||
```
|
||||
`onnx_graphsurgeon` is required by `polygraphy surgeon sanitize` (Step 3) — Polygraphy does not bundle it. `onnxruntime` is required by `optimum-cli export` (Step 2, to fix dynamic axes) and Polygraphy's `--onnxrt` parity check (Step 5); stock NGC containers do not always ship it.
|
||||
Note: `optimum-onnx` is the active maintained package — the older `optimum` no longer includes ONNX integration. Do **not** use `optimum-nvidia` (last release 2025-01, unmaintained).
|
||||
|
||||
## Step 1 — Choose a model and confirm it's supported
|
||||
|
||||
Consult the [Supported Models](https://github.com/NVIDIA/TensorRT/blob/main/documents/supported_models.md) matrix. If listed, the dtype column gives a known-good starting precision. If not, it is still expected to work — file an issue if it doesn't.
|
||||
|
||||
If the model is a diffusion pipeline, **stop here** — TRT cannot ingest the whole pipeline object. Split it into components (text encoder, UNet/DiT, VAE) and import each separately. See `supported_models.md` for validated splits, and consider whether `trt-export-rewrite` is a better fit.
|
||||
|
||||
This skill's worked example uses `google-bert/bert-base-uncased` from the encoder-NLP section.
|
||||
|
||||
## Step 2 — Export to ONNX
|
||||
|
||||
The `optimum-cli export onnx` command's `--task` flag selects which head of the model to export. The wrong task yields a valid graph but the wrong output — and Step 5 parity validation won't catch it, because reference and engine are both wrong.
|
||||
|
||||
Pick the task from this table based on what the developer wants to do with the model:
|
||||
|
||||
| Developer's intent | `--task` | Typical model families |
|
||||
|--------------------|----------|------------------------|
|
||||
| Get hidden-state embeddings or pooled output | `feature-extraction` | BERT, RoBERTa, sentence-transformers, CLIP |
|
||||
| Classify images into N classes | `image-classification` | ResNet, ViT, MobileNet, EfficientNet |
|
||||
| Classify text into N labels | `text-classification` | BERT for sentiment, DistilBERT, etc. |
|
||||
| Detect / segment objects | `object-detection`, `image-segmentation` | DETR, Mask R-CNN |
|
||||
| Transcribe speech | `automatic-speech-recognition` | Whisper, Wav2Vec2 |
|
||||
| Generate captions / VQA | `image-to-text`, `visual-question-answering` | BLIP, LLaVA |
|
||||
| Translation / summarization | `text2text-generation` | T5, BART |
|
||||
|
||||
For an unknown model, run `optimum-cli export onnx --help` or check the [Optimum supported tasks table](https://huggingface.co/docs/optimum/main/en/exporters/onnx/overview). If unsure, surface the choice to the developer rather than guessing.
|
||||
|
||||
### Sentence-transformers models need an extra install
|
||||
|
||||
If the model ID starts with `sentence-transformers/` (e.g. `all-MiniLM-L6-v2`, `bge-base-en-v1.5`), install `sentence-transformers` *before* exporting:
|
||||
|
||||
```bash
|
||||
pip install sentence-transformers
|
||||
```
|
||||
|
||||
Without it, `optimum-cli` falls back to plain `transformers` and emits raw token-level `last_hidden_state` instead of the pooled sentence embedding the model is designed to produce. The exporter prints `library name was inferred as sentence_transformers, which is not installed. Falling back to transformers.` — treat that warning as a hard error and install before retrying.
|
||||
|
||||
```bash
|
||||
# Worked example: BERT for embedding extraction
|
||||
optimum-cli export onnx \
|
||||
--model google-bert/bert-base-uncased \
|
||||
--task feature-extraction \
|
||||
bert_onnx/
|
||||
```
|
||||
|
||||
### Expected warnings (not errors)
|
||||
|
||||
The exporter often prints a "max diff between reference and ONNX exported model is not within the set tolerance 1e-05" line (with `max diff` around `1e-5` to `1e-4`). This is its own self-check at a strict tolerance — **the export still succeeded.** Step 5's Polygraphy parity check uses `--atol 1e-2 --rtol 1e-2 --check-error-stat mean`, the correct tolerance for FP16 inference. Don't surface this warning as a failure.
|
||||
|
||||
### Authentication for gated models
|
||||
|
||||
LLaMA, FLUX, and some Mistral and Qwen variants are gated behind HF agreements and require `huggingface-cli login` before download. If `optimum-cli` exits with a 401, run `huggingface-cli login` with a token from `huggingface.co/settings/tokens` and retry.
|
||||
|
||||
### If the export fails
|
||||
|
||||
The most common cause is a HF model using patterns `torch.export` / `torch.onnx.export` cannot trace — complex-number arithmetic, data-dependent control flow, non-tensor forward arguments, output dataclasses. **Do not patch the upstream library on disk.** The correct workflow is agentic monkey-patching at runtime: route the developer to the `trt-export-rewrite` skill, which drives it with five concrete patterns from the Qwen-Image case study.
|
||||
|
||||
## Step 3 — Sanitize the ONNX
|
||||
|
||||
ONNX exporters often produce graphs with constant-folding opportunities, spurious dynamic axes, or shape-inference gaps. Sanitizing cleans these up, but it is a **quality pass, not a correctness gate** — TRT parses many un-sanitized exports fine. Both stages below are therefore optional and **allowed to fail**: each falls through to its input, and Step 3 always leaves a `model.clean.onnx` for the rest of the workflow (worst case, a copy of the exported model). Run:
|
||||
|
||||
```bash
|
||||
# Seed model.clean.onnx with the raw export, then update it in-place
|
||||
cp bert_onnx/model.onnx bert_onnx/model.clean.onnx
|
||||
|
||||
# (1) onnxsim — graph simplification / constant folding.
|
||||
python3 -m onnxsim bert_onnx/model.clean.onnx bert_onnx/model.simplified.onnx \
|
||||
&& cp bert_onnx/model.simplified.onnx bert_onnx/model.clean.onnx
|
||||
|
||||
# (2) polygraphy surgeon — fold constants, prune dangling nodes.
|
||||
polygraphy surgeon sanitize bert_onnx/model.clean.onnx \
|
||||
-o bert_onnx/model.surgeon.onnx --fold-constants \
|
||||
&& cp bert_onnx/model.surgeon.onnx bert_onnx/model.clean.onnx
|
||||
```
|
||||
|
||||
## Step 3.5 — Inspect the ONNX inputs (parameterize the rest of the workflow)
|
||||
|
||||
**Every step from here uses input names and ranks that depend on the model.** Read them out of the sanitized ONNX before building. Do not assume `input_ids` / `attention_mask` — those are BERT-family-specific.
|
||||
|
||||
```bash
|
||||
polygraphy inspect model bert_onnx/model.clean.onnx
|
||||
```
|
||||
|
||||
Example output for `bert-base-uncased` (verified against `optimum-onnx` 0.1.0, opset 18):
|
||||
|
||||
```
|
||||
---- 3 Graph Input(s) ----
|
||||
{input_ids [dtype=int64, shape=('batch_size', 'sequence_length')],
|
||||
attention_mask [dtype=int64, shape=('batch_size', 'sequence_length')],
|
||||
token_type_ids [dtype=int64, shape=('batch_size', 'sequence_length')]}
|
||||
|
||||
---- 1 Graph Output(s) ----
|
||||
{last_hidden_state [dtype=float32, shape=('batch_size', 'sequence_length', 768)]}
|
||||
```
|
||||
|
||||
Inputs and outputs vary per model and per `--task`. Inspect, don't assume.
|
||||
|
||||
**Build the shape strings from this output.** Every input with a dynamic (named) axis needs a `--minShapes / --optShapes / --maxShapes` entry; fixed axes are reproduced literally. Use representative values — `--opt` is what the engine optimizes for; `--min` and `--max` define the supported range.
|
||||
|
||||
If any input reports an unknown-rank axis (printed as `?`), shape inference failed; reroute through `trt-export-rewrite` to fix the export.
|
||||
|
||||
## Step 4 — Bake FP16 into the model (TRT 11) or set the build flag (TRT 10)
|
||||
|
||||
Start with FP16 — the safe default for encoder and vision models on modern GPUs. (Use BF16 on Blackwell/Hopper if the developer reports persistent FP16 accuracy issues. Sentence-embedding models sometimes prefer FP32 over lossy FP16 — surface the option.)
|
||||
|
||||
**TRT 11 removed the weak-typing builder flags** (`--fp16`, `--int8`, `--bf16`, `BuilderFlag::kFP16`, etc. — see [the migration guide](https://docs.nvidia.com/deeplearning/tensorrt/latest/api/migration/tensorrt-10x-to-11x.html) and the sibling `trt-strong-typing-migration` SKILL). In TRT 11 you bake precision **into the model** before building, and `trtexec` honors the dtypes it finds via **ModelOpt AutoCast**.
|
||||
|
||||
Detect the version first:
|
||||
|
||||
```bash
|
||||
TRT_MAJOR=$(python3 -c "import tensorrt as t; print(t.__version__.split('.')[0])")
|
||||
echo "TensorRT major version: $TRT_MAJOR"
|
||||
```
|
||||
|
||||
### TRT 11: AutoCast the model, then build
|
||||
|
||||
```bash
|
||||
pip install nvidia-modelopt # one-time
|
||||
|
||||
# Convert FP32 ONNX to mixed FP16 (ModelOpt picks accuracy-sensitive ops to keep FP32):
|
||||
python3 -m modelopt.onnx.autocast \
|
||||
--onnx_path=<MODEL>.clean.onnx \
|
||||
--output_path=<MODEL>.fp16.onnx
|
||||
|
||||
# Build the engine. NO --fp16 flag — strong typing is the default and the model already carries FP16 dtypes.
|
||||
trtexec \
|
||||
--onnx=<MODEL>.fp16.onnx \
|
||||
--saveEngine=<MODEL>.fp16.plan \
|
||||
--memPoolSize=workspace:4096 \
|
||||
--minShapes=<INPUT1>:<MINSHAPE>,... \
|
||||
--optShapes=<INPUT1>:<OPTSHAPE>,... \
|
||||
--maxShapes=<INPUT1>:<MAXSHAPE>,...
|
||||
```
|
||||
|
||||
### TRT 10: pass the legacy --fp16 flag
|
||||
|
||||
```bash
|
||||
trtexec \
|
||||
--onnx=<MODEL>.clean.onnx \
|
||||
--saveEngine=<MODEL>.fp16.plan \
|
||||
--memPoolSize=workspace:4096 \
|
||||
--fp16 \
|
||||
--minShapes=<INPUT1>:<MINSHAPE>,... \
|
||||
--optShapes=<INPUT1>:<OPTSHAPE>,... \
|
||||
--maxShapes=<INPUT1>:<MAXSHAPE>,...
|
||||
```
|
||||
|
||||
Worked example for BERT (three inputs, all dynamic on batch and sequence axes), TRT 11 path:
|
||||
|
||||
```bash
|
||||
python3 -m modelopt.onnx.autocast \
|
||||
--onnx_path=bert_onnx/model.clean.onnx \
|
||||
--output_path=bert_onnx/model.fp16.onnx
|
||||
trtexec \
|
||||
--onnx=bert_onnx/model.fp16.onnx \
|
||||
--saveEngine=bert.fp16.plan \
|
||||
--memPoolSize=workspace:4096 \
|
||||
--minShapes=input_ids:1x32,attention_mask:1x32,token_type_ids:1x32 \
|
||||
--optShapes=input_ids:8x128,attention_mask:8x128,token_type_ids:8x128 \
|
||||
--maxShapes=input_ids:16x512,attention_mask:16x512,token_type_ids:16x512
|
||||
```
|
||||
|
||||
Worked example for ResNet-50 (one input, dynamic on batch axis only), TRT 11 path:
|
||||
|
||||
```bash
|
||||
python3 -m modelopt.onnx.autocast \
|
||||
--onnx_path=resnet50_onnx/model.clean.onnx \
|
||||
--output_path=resnet50_onnx/model.fp16.onnx
|
||||
trtexec \
|
||||
--onnx=resnet50_onnx/model.fp16.onnx \
|
||||
--saveEngine=resnet50.fp16.plan \
|
||||
--memPoolSize=workspace:4096 \
|
||||
--minShapes=pixel_values:1x3x224x224 \
|
||||
--optShapes=pixel_values:16x3x224x224 \
|
||||
--maxShapes=pixel_values:32x3x224x224
|
||||
```
|
||||
|
||||
For purely static-shape models, omit all three shape flags.
|
||||
|
||||
### Common errors at this step
|
||||
|
||||
- **"Engine plan file is generated on an incompatible device"** — plans are not portable across compute capabilities. Rebuild on the deployment GPU.
|
||||
- **OOM during build** — lower `--memPoolSize=workspace:N`. If still failing, try `--tacticSources=-CUBLAS_LT` to disable expensive tactic sources.
|
||||
- **Unsupported operator** — `trtexec` names the op. Route to `trt-unsupported-op` for triage (decompose / plugin / switch frontend / file bug).
|
||||
- **INT64 warnings** — TRT casts to INT32. Usually safe; if values exceed INT32 range, rerun Step 3's `polygraphy surgeon sanitize --fold-constants`.
|
||||
- **"Input <name> has missing shape information"** — you missed an input in the `--minShapes / --optShapes / --maxShapes` flags. Compare against Step 3.5's output and add the missing entry.
|
||||
|
||||
## Step 5 — Verify numerical parity
|
||||
|
||||
This is the step most developers skip — and the one that most often surfaces a silent correctness bug.
|
||||
|
||||
Construct the parity command from the same input names used in Step 4. Beyond input shapes, three knobs matter; get them right or the result is misleading:
|
||||
|
||||
1. **`--val-range`** for any input with constrained semantics. Polygraphy fills with `[0, 1)` floats by default. For BERT-family `input_ids` you must constrain to vocabulary range, set `attention_mask` to all-ones, and `token_type_ids` to all-zeros — otherwise the model sees garbage and FP16 noise amplifies through the attention softmax.
|
||||
2. **`--check-error-stat mean`** for transformer outputs. FP16 commonly has a few outlier positions with large abs diff; the mean across all positions is the meaningful metric for downstream use (embedding similarity, classification). Default `max` is too pessimistic for FP16 transformers and produces false failures.
|
||||
3. **`--seed`** for repeatability across runs and CI environments.
|
||||
|
||||
Worked example for BERT — run polygraphy against the **same ONNX file you fed to `trtexec`** (the AutoCast-converted `.fp16.onnx` on TRT 11; the `.clean.onnx` on TRT 10):
|
||||
|
||||
```bash
|
||||
# TRT 11: polygraphy reads the model's dtypes (already FP16 from AutoCast)
|
||||
polygraphy run bert_onnx/model.fp16.onnx \
|
||||
--trt --onnxrt \
|
||||
--atol 1e-2 --rtol 1e-2 \
|
||||
--check-error-stat mean \
|
||||
--input-shapes input_ids:[8,128] attention_mask:[8,128] token_type_ids:[8,128] \
|
||||
--val-range input_ids:[0,30000] attention_mask:[1,1] token_type_ids:[0,0] \
|
||||
--seed 42
|
||||
|
||||
# TRT 10: pass --fp16 explicitly (the polygraphy flag maps to BuilderFlag::kFP16)
|
||||
polygraphy run bert_onnx/model.clean.onnx \
|
||||
--trt --onnxrt \
|
||||
--atol 1e-2 --rtol 1e-2 \
|
||||
--check-error-stat mean \
|
||||
--input-shapes input_ids:[8,128] attention_mask:[8,128] token_type_ids:[8,128] \
|
||||
--val-range input_ids:[0,30000] attention_mask:[1,1] token_type_ids:[0,0] \
|
||||
--seed 42 \
|
||||
--fp16
|
||||
```
|
||||
|
||||
Expected output: `Pass Rate: 100.0%` with `mean_absdiff ≤ 1e-3` per output. `Pass Rate: 0.0%` with `max_absdiff` near 1e-2 but `mean_absdiff` well below tolerance is the outlier pattern above — use `--check-error-stat mean`.
|
||||
|
||||
Worked example for ResNet-50 (same FP16 recipe; TRT-11 path shown — for TRT 10 add `--fp16` and read from `model.clean.onnx`):
|
||||
|
||||
```bash
|
||||
polygraphy run resnet50_onnx/model.fp16.onnx \
|
||||
--trt --onnxrt \
|
||||
--atol 1e-2 --rtol 1e-2 \
|
||||
--check-error-stat mean \
|
||||
--input-shapes pixel_values:[16,3,224,224] \
|
||||
--val-range pixel_values:[0,1] \
|
||||
--seed 42
|
||||
```
|
||||
|
||||
Polygraphy runs the model through ONNX Runtime (reference) and TensorRT (engine) on the seeded inputs and compares outputs per the chosen error stat.
|
||||
|
||||
### If parity fails
|
||||
|
||||
- **Diverges only at FP16, not FP32**: a few layers are losing precision. Try `--strongly-typed` with an explicit FP32 cast on the offending subgraph (Polygraphy names the layer). For LLM-style models, prefer BF16 over FP16 on Hopper/Blackwell.
|
||||
- **Diverges at all precisions**: the export is wrong. Re-run Steps 2–3 fresh and check `polygraphy inspect model` for shape-inference issues. If the model uses patterns covered by `trt-export-rewrite`, reroute.
|
||||
- **Diverges only for specific input shapes**: dynamic-shape profile is too narrow. Widen `--minShapes` / `--maxShapes` and rebuild.
|
||||
|
||||
## Step 6 — Run the engine
|
||||
|
||||
```bash
|
||||
trtexec --loadEngine=bert.fp16.plan \
|
||||
--shapes=input_ids:8x128,attention_mask:8x128,token_type_ids:8x128 \
|
||||
--verbose
|
||||
```
|
||||
|
||||
This confirms the saved `.plan` deserializes and runs (Step 5's parity used its own engine, not this file) and reports throughput and per-iteration latency. For production, see the developer guide on `IExecutionContext` and stream-aware execution; the `.plan` is portable to any compatible GPU + TRT runtime.
|
||||
|
||||
## Success criteria
|
||||
|
||||
The skill is complete when all of the following are true:
|
||||
|
||||
- `bert.fp16.plan` exists and `trtexec --loadEngine=...` runs without error.
|
||||
- `polygraphy run --trt --onnxrt --atol 1e-2 --rtol 1e-2 --check-error-stat mean` reports `PASSED` across all configured input shapes.
|
||||
- The developer can articulate which precision was used and why.
|
||||
|
||||
If any are not true, hand control back with the specific failing diagnostic — do not declare success.
|
||||
|
||||
## References
|
||||
|
||||
- [Import Workflows Guide (Path 1: ONNX → TensorRT)](https://github.com/NVIDIA/TensorRT/blob/main/documents/import_workflows.md#path-1-onnx--tensorrt)
|
||||
- [Supported Models matrix](https://github.com/NVIDIA/TensorRT/blob/main/documents/supported_models.md)
|
||||
- [Polygraphy documentation](https://docs.nvidia.com/deeplearning/tensorrt/polygraphy/index.html)
|
||||
- [`samples/sampleOnnxMNIST/`](https://github.com/NVIDIA/TensorRT/tree/main/samples/sampleOnnxMNIST) for the C++ equivalent of the build flow
|
||||
@@ -0,0 +1,103 @@
|
||||
---
|
||||
name: trt-perf-analysis
|
||||
description: Validate and analyze TensorRT performance data from paired layer-info JSON and profile/latency JSON files. Use when asked to inspect TensorRT, TRT, torch-tensorrt, or ONNX-TensorRT perf reports, verify that layer/profile JSON files are valid and from the same model, infer basic model information, find likely fusion or latency optimization opportunities, and produce a concise Markdown performance report or structured JSON data.
|
||||
---
|
||||
|
||||
# TRT Perf Analysis
|
||||
|
||||
## Workflow
|
||||
|
||||
Use `scripts/run.sh` on Unix-like systems or `scripts\run.cmd` on Windows for Python scripts. These wrappers do best-effort Python 3.8+ discovery; set `SKILL_PYTHON` to a Python executable to override discovery. Replace placeholders with platform-native paths.
|
||||
|
||||
1. Choose the input scope.
|
||||
Use one folder that directly contains `layers_*.json` and/or `profile_*.json`. Do not call the packager on a parent folder that only contains component subfolders. For model suites with separate encoder, transformer, decoder, VAE, or similar components, handle each component folder separately.
|
||||
|
||||
2. Infer model identity and components.
|
||||
Prefer an explicit model name from the user prompt and pass it with `--model-name`. Otherwise rely on the analyzer/packager to inspect likely config files, the input directory name, and layer/profile filenames. Keep the name empty when confidence is low. Serialized JSON records the inferred model identity at the top level and in each successful backend's `model` object.
|
||||
|
||||
3. Run deterministic analysis.
|
||||
Run `scripts/analyze_trt_perf.py` first when you need an integrity gate before interpretation. The script uses only Python built-in modules, extracts structured analysis data, and serializes it as JSON. This JSON is the authoritative analyzer output for validation, packaging, and AI diagnosis.
|
||||
|
||||
4. Check validation before interpreting.
|
||||
The analyzer exits `0` when it can emit structured validation data, even if one or more backends fail validation. Read the generated JSON `validation` object to decide which backend reports are usable. If the analyzer exits nonzero, stop because it could not emit structured data. If the schema validator exits nonzero, stop because the generated JSON contract is invalid. Do not continue into performance interpretation for a backend unless its validation status is `passed`, its analysis mode is `layer_profile`, the layer graph is a DAG, and the layer/profile names match. For `layer_only` backends, graph and layer inspection are available but latency/performance interpretation is not.
|
||||
|
||||
5. Understand the model, then write AI diagnosis.
|
||||
Use the generated JSON as grounding material. Build a clear picture of the model or component scope, backend differences, graph structure, timing distribution, hot layers, fusion clues, and caveats before writing conclusions. The browser `Summary` panel is generated from `analyze-data.json`; do not copy that deterministic summary into `analyze.md`.
|
||||
|
||||
Keep the final AI analysis in two sections:
|
||||
|
||||
1. `## Summary` - short, high-confidence facts and the most important optimization leads.
|
||||
2. `## Details` - interpretation, cross-backend comparisons, hypotheses, ranked experiments, and human judgment that is not a mechanical restatement of the analyzer output.
|
||||
|
||||
Before sending the final response, write the final AI analysis into `analyze.md` for each report folder you created or updated. Match each file to that report folder's scope. For a single model with multiple backend JSON pairs, one `analyze.md` may compare those backends. For multiple model components packaged as multiple report folders, write distinct component-focused `analyze.md` files; do not copy a suite-level analysis into every component report.
|
||||
|
||||
`analyze.md` should not include validation checklists, complete timing tables, top-layer tables, path dumps, or other content already rendered by the report app from `analyze-data.json`. Briefly cite specific metrics or layer names only when they support a new conclusion or next experiment.
|
||||
|
||||
6. Package the report.
|
||||
Run `scripts/package_report.py` through the shared runner. Each invocation creates exactly one final report folder from one input folder that directly contains `layers_*.json` or `profile_*.json` files. The packager writes `analyze-data.json`, validates it against the schema, and copies the browser report template while preserving the generated JSON. Determine the output parent from the user's active workspace, not from a subdirectory of this skill. If the only known directory is the skill directory, omit `--output-parent`; the packaging script will fall back to the system temp directory.
|
||||
|
||||
When producing multiple reports for one user request, prefer explicit `--report-dir` values so report folder names include the model/component identity instead of relying on auto-numbered collision suffixes. If a suite-level conclusion is useful, write it as a separate overview Markdown file in the output parent, not as every component report's `analyze.md`.
|
||||
|
||||
7. Preview only the final report.
|
||||
Serve the completed report folder, not the skill directory, when the user wants a browser preview. The Python `visualize_layer_info.py` script remains available for standalone HTML layer-info debugging, but it is not part of the report packaging flow or report app dependency chain.
|
||||
|
||||
### Common Commands
|
||||
|
||||
Use `<runner>` as `<skill-dir>/scripts/run.sh` on Unix-like systems or `<skill-dir>\scripts\run.cmd` on Windows.
|
||||
|
||||
```bash
|
||||
# Validate or inspect inputs. Folder mode discovers matching backends; repeat --data for explicit backends.
|
||||
<runner> <skill-dir>/scripts/analyze_trt_perf.py <model-folder> --output <analysis.json>
|
||||
<runner> <skill-dir>/scripts/analyze_trt_perf.py --data <layers-a.json> <profile-a.json> --data <layers-b.json> --output <analysis.json>
|
||||
|
||||
# Create a report. Use --report-dir for component-specific reports and --analyze-md after writing final AI analysis.
|
||||
<runner> <skill-dir>/scripts/package_report.py <model-folder> --output-parent <workspace-folder> [--model-name <model>] [--analyze-md <final-analysis.md>]
|
||||
<runner> <skill-dir>/scripts/package_report.py --analyze-data <report-folder>/analyze-data.json [--analyze-md <final-analysis.md>]
|
||||
|
||||
# Optional preview.
|
||||
<runner> -m http.server 8765 --bind 127.0.0.1 --directory <report-folder>
|
||||
```
|
||||
|
||||
Then open `http://127.0.0.1:8765/`.
|
||||
|
||||
## Inputs
|
||||
|
||||
Expect one layer-info JSON and one profile/latency JSON per engine/report. A folder may contain multiple pairs such as:
|
||||
|
||||
- `layers_torch-trt-aot.json` with `profile_torch-trt-aot.json`
|
||||
- `layers_onnx-tensorrt.json` with `profile_onnx-tensorrt.json`
|
||||
|
||||
Layer records are expected to include `Name`, `LayerType`, `Inputs`, and `Outputs`. Profile records are expected to include `name`, `timeMs`, `averageMs`, `medianMs`, and `percentage`; a first record like `{ "count": 20 }` is allowed.
|
||||
|
||||
Infer a per-layer `Type` for report output. Default to the raw `LayerType`, then apply narrow special-case overrides when the layer name, tactic, or metadata makes the specialization clear. Known special cases:
|
||||
|
||||
- `kgen_mha`: a raw `LayerType` of `kgen` that is actually an MHA/FMHA-style attention layer.
|
||||
- `misc`: raw `LayerType` values `reshape`, `shape_call`, `signal`, and `wait` are grouped into this category.
|
||||
|
||||
## Model Categories
|
||||
|
||||
Use this starter category list and revise as evidence warrants:
|
||||
|
||||
- Transformer encoder / sentence embedding
|
||||
- Transformer decoder / LLM
|
||||
- Vision transformer
|
||||
- CNN / convolutional vision model
|
||||
- Diffusion / U-Net style model
|
||||
- Recommender / ranking model
|
||||
- Classical ML or feature pipeline
|
||||
- Unknown DL model
|
||||
|
||||
Report category only when the evidence is strong. Good evidence includes Hugging Face config fields, names like `encoder.layer`, attention layers, input names such as `input_ids`, or repeated convolution/GEMM patterns. Say "unknown" instead of over-claiming.
|
||||
|
||||
## Analysis Heuristics
|
||||
|
||||
Prioritize issues by likely runtime impact and confidence:
|
||||
|
||||
- Hot layers: individual layers with high `percentage` or `averageMs`.
|
||||
- Layer-type concentration: high aggregate time in `kgen`, shape, cast, gather, reshape, or pointwise/reduction layers may indicate fusion or dynamic-shape overhead.
|
||||
- Fusion clues: for transformer encoders, check whether Q/K/V matmuls are fused per block, whether attention is an obvious fused SDPA/FMHA layer, and whether layer-norm/residual/GELU patterns are fused into larger kernels.
|
||||
- Engine partitioning: one engine is usually ideal. Use adjacent `.trt`/`.engine` files, `_subgraph`, and `StreamId` as hints; state when this is only inferred.
|
||||
- GEMM tactics: GEMM layers without Tensor Core or xMMA-like tactic names may deserve inspection.
|
||||
- Many tiny kernels: if no single layer dominates but many small kernels add up, rank launch/fusion overhead above isolated micro-hotspots.
|
||||
|
||||
Keep facts and hypotheses separate. For each optimization opportunity, include evidence from the JSON and a next check or experiment.
|
||||
@@ -0,0 +1,86 @@
|
||||
Third Party Notices
|
||||
===================
|
||||
|
||||
This report includes or loads third-party open source software.
|
||||
These notices are provided for attribution and license-tracking purposes only
|
||||
and do not modify the license terms of the third-party projects.
|
||||
|
||||
|
||||
Netron
|
||||
------
|
||||
|
||||
Project: Netron
|
||||
Homepage: https://github.com/lutzroeder/netron
|
||||
Version included in this report: 9.1.1
|
||||
License: MIT
|
||||
License URL: https://github.com/lutzroeder/netron/blob/main/LICENSE
|
||||
Copyright: Copyright (c) Lutz Roeder
|
||||
|
||||
Notes:
|
||||
- The Netron files are embedded as a trimmed report-specific viewer.
|
||||
- The embedded files have been modified for use in this TensorRT performance
|
||||
report, including report-specific loading, iframe integration, and layer
|
||||
selection behavior.
|
||||
|
||||
|
||||
Dagre
|
||||
-----
|
||||
|
||||
Project: Dagre
|
||||
Homepage: https://github.com/dagrejs/dagre
|
||||
License: MIT
|
||||
License URL: https://github.com/dagrejs/dagre/blob/master/LICENSE
|
||||
Copyright: Copyright (c) 2012-2014 Chris Pettitt
|
||||
|
||||
Notes:
|
||||
- The bundled graph layout code is included through the report-specific Netron
|
||||
viewer bundle.
|
||||
|
||||
|
||||
Graphlib
|
||||
--------
|
||||
|
||||
Project: Graphlib
|
||||
Homepage: https://github.com/dagrejs/graphlib
|
||||
License: MIT
|
||||
Package metadata URL: https://github.com/dagrejs/graphlib/blob/master/package.json
|
||||
Author: Chris Pettitt
|
||||
|
||||
Notes:
|
||||
- Dagre depends on Graphlib. The bundled layout code in dev/report/netron/dagre.js may
|
||||
include or derive from Graphlib functionality.
|
||||
|
||||
|
||||
DOMPurify
|
||||
---------
|
||||
|
||||
Project: DOMPurify
|
||||
Homepage: https://github.com/cure53/DOMPurify
|
||||
Version included in this report: 3.4.9
|
||||
License: MPL-2.0 OR Apache-2.0
|
||||
License option used for this notice: Apache-2.0
|
||||
License URL: https://github.com/cure53/DOMPurify/blob/3.4.9/LICENSE
|
||||
Package metadata URL: https://github.com/cure53/DOMPurify/blob/3.4.9/package.json
|
||||
Author: Dr.-Ing. Mario Heiderich, Cure53
|
||||
|
||||
Notes:
|
||||
- DOMPurify is used at runtime to sanitize rendered Markdown content.
|
||||
|
||||
|
||||
Marked
|
||||
------
|
||||
|
||||
Project: marked
|
||||
Homepage: https://github.com/markedjs/marked
|
||||
Version included in this report: 18.0.5
|
||||
License: MIT
|
||||
License URL: https://github.com/markedjs/marked/blob/v18.0.5/LICENSE.md
|
||||
Package metadata URL: https://github.com/markedjs/marked/blob/v18.0.5/package.json
|
||||
Copyright:
|
||||
- Copyright (c) 2018-2026, MarkedJS
|
||||
- Copyright (c) 2011-2018, Christopher Jeffrey
|
||||
|
||||
Notes:
|
||||
- marked is used at runtime to render Markdown content.
|
||||
- The marked license file also includes attribution and license terms for
|
||||
Markdown by John Gruber. See the upstream license URL above.
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,11 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32">
|
||||
<title>TRT donut T v3</title>
|
||||
<rect width="32" height="32" rx="6" fill="#f5f7fb"/>
|
||||
<g transform="rotate(-90 16 16)" fill="none" stroke-width="6" stroke-linecap="butt">
|
||||
<circle cx="16" cy="16" r="11" pathLength="100" stroke="#2563eb" stroke-dasharray="50 50" stroke-dashoffset="0"/>
|
||||
<circle cx="16" cy="16" r="11" pathLength="100" stroke="#16a34a" stroke-dasharray="22 78" stroke-dashoffset="-50"/>
|
||||
<circle cx="16" cy="16" r="11" pathLength="100" stroke="#d97706" stroke-dasharray="28 72" stroke-dashoffset="-72"/>
|
||||
</g>
|
||||
<circle cx="16" cy="16" r="6.5" fill="#f5f7fb"/>
|
||||
<path d="M11 11h10v3h-3.5v8h-3v-8H11Z" fill="#1d2433"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 716 B |
@@ -0,0 +1,183 @@
|
||||
<!doctype html>
|
||||
<!-- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
|
||||
<!-- SPDX-License-Identifier: Apache-2.0 -->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>TRT Performance Report</title>
|
||||
<link rel="icon" type="image/svg+xml" href="favicon-donut-t-v3.svg">
|
||||
<link rel="stylesheet" href="styles.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="page">
|
||||
<header class="topbar" id="topbar">
|
||||
<div class="title-block">
|
||||
<p class="eyebrow">TensorRT Analyze Report</p>
|
||||
<div class="title-line">
|
||||
<h1 id="report-title">TRT Performance Report</h1>
|
||||
<span class="backend-chip" id="backend-chip">Backend</span>
|
||||
</div>
|
||||
<p class="subtitle" id="report-subtitle"></p>
|
||||
<div class="pinned-summary" id="pinned-summary" aria-label="Pinned basic report information"></div>
|
||||
</div>
|
||||
<div class="status-stack">
|
||||
<div class="status-pill" id="status-pill">
|
||||
<span class="status-dot"></span>
|
||||
<span id="status-text">Loading data</span>
|
||||
</div>
|
||||
<div class="graph-status" id="graph-status">Loading graph</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<nav class="result-tabs" id="result-tabs" aria-label="Analysis results"></nav>
|
||||
|
||||
<section class="metrics" id="metrics" aria-label="Performance metrics"></section>
|
||||
|
||||
<details class="panel summary-panel analyze-panel" id="analyze-panel" aria-labelledby="analyze-title" open hidden>
|
||||
<summary class="summary-panel-header" aria-label="Toggle AI-generated analysis">
|
||||
<span class="details-toggle" aria-hidden="true"></span>
|
||||
<span class="summary-panel-title" id="analyze-title">Analysis (generated by AI)</span>
|
||||
<button class="panel-help-button" type="button" aria-label="Show help for Analysis generated by AI" aria-expanded="false" aria-controls="panel-help-popover" data-help-title="Analysis (generated by AI)" data-help="Generated by AI from the validated report data.">?</button>
|
||||
</summary>
|
||||
<div class="markdown-content" id="analyze-content"></div>
|
||||
</details>
|
||||
|
||||
<details class="panel summary-panel" aria-labelledby="summary-title">
|
||||
<summary class="summary-panel-header" aria-label="Toggle script-generated analysis">
|
||||
<span class="details-toggle" aria-hidden="true"></span>
|
||||
<span class="summary-panel-title" id="summary-title">Analysis (generated by script)</span>
|
||||
<button class="panel-help-button" type="button" aria-label="Show help for Analysis generated by script" aria-expanded="false" aria-controls="panel-help-popover" data-help-title="Analysis (generated by script)" data-help="Generated by script from the validated report data.">?</button>
|
||||
</summary>
|
||||
<div class="markdown-content" id="summary-content"></div>
|
||||
</details>
|
||||
|
||||
<details class="panel details-panel" aria-labelledby="findings-title">
|
||||
<summary class="details-summary" aria-label="Toggle optimization leads and positive signals">
|
||||
<span class="details-toggle" aria-hidden="true"></span>
|
||||
<span class="details-panel-title" id="findings-title">Optimization Leads & Positive Signals</span>
|
||||
<button class="panel-help-button" type="button" aria-label="Show help for Optimization Leads and Positive Signals" aria-expanded="false" aria-controls="panel-help-popover" data-help-title="Optimization Leads & Positive Signals" data-help="Generated by script from the validated report data.">?</button>
|
||||
</summary>
|
||||
<div class="details-grid">
|
||||
<article class="list-card">
|
||||
<h2>Optimization Leads</h2>
|
||||
<ul class="finding-list" id="issues"></ul>
|
||||
</article>
|
||||
<article class="list-card">
|
||||
<h2>Positive Signals</h2>
|
||||
<ul class="finding-list" id="positives"></ul>
|
||||
<p class="path-line" id="paths"></p>
|
||||
</article>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<main class="analysis-shell">
|
||||
<div class="report-column">
|
||||
<section class="panel workspace" aria-label="Layer performance report">
|
||||
<section class="chart-panel" aria-labelledby="chart-title">
|
||||
<div class="panel-header">
|
||||
<div>
|
||||
<div class="panel-title-line">
|
||||
<h2 class="panel-title" id="chart-title">Layers</h2>
|
||||
<button class="panel-help-button" type="button" aria-label="Show help for Layers" aria-expanded="false" aria-controls="panel-help-popover" data-help-title="Layers" data-help="Layer latency/type breakdown and searchable layer table.">?</button>
|
||||
</div>
|
||||
<p class="panel-kicker" id="chart-kicker"></p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="type-summary" id="type-summary"></div>
|
||||
<div class="chart-body">
|
||||
<div class="chart-wrap">
|
||||
<svg class="pie" id="type-pie" viewBox="0 0 320 320" role="img" aria-labelledby="chart-title"></svg>
|
||||
</div>
|
||||
<div class="legend" id="legend"></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="layers-panel" aria-labelledby="layers-title">
|
||||
<div class="table-toolbar">
|
||||
<div class="table-title">
|
||||
<strong id="layers-title">Layers</strong>
|
||||
<span id="layers-count"></span>
|
||||
</div>
|
||||
<label class="sr-only" for="layer-filter">Filter layers</label>
|
||||
<input class="filter-input" id="layer-filter" type="search" placeholder="Filter layers" autocomplete="off">
|
||||
</div>
|
||||
<div class="table-scroll">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col">
|
||||
<button class="table-sort-button" type="button" data-layer-sort-key="topo">
|
||||
<span>Topo</span>
|
||||
<span class="sort-indicator" aria-hidden="true"></span>
|
||||
</button>
|
||||
</th>
|
||||
<th scope="col">Layer</th>
|
||||
<th scope="col">
|
||||
<button class="table-sort-button" type="button" data-layer-sort-key="avg">
|
||||
<span>Avg ms</span>
|
||||
<span class="sort-indicator" aria-hidden="true"></span>
|
||||
</button>
|
||||
</th>
|
||||
<th scope="col">
|
||||
<button class="table-sort-button" type="button" data-layer-sort-key="total">
|
||||
<span>Total ms</span>
|
||||
<span class="sort-indicator" aria-hidden="true"></span>
|
||||
</button>
|
||||
</th>
|
||||
<th scope="col">
|
||||
<button class="table-sort-button" type="button" data-layer-sort-key="share">
|
||||
<span>Share</span>
|
||||
<span class="sort-indicator" aria-hidden="true"></span>
|
||||
</button>
|
||||
</th>
|
||||
<th scope="col">Tags</th>
|
||||
<th scope="col">
|
||||
<button class="table-sort-button" type="button" data-layer-sort-key="stream">
|
||||
<span>Stream</span>
|
||||
<span class="sort-indicator" aria-hidden="true"></span>
|
||||
</button>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="layer-rows"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="empty" id="empty-state" hidden>No matching layers</div>
|
||||
</section>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<aside class="panel graph-panel" aria-labelledby="graph-title">
|
||||
<div class="panel-header graph-header">
|
||||
<div>
|
||||
<div class="panel-title-line">
|
||||
<h2 class="panel-title" id="graph-title">Model Graph</h2>
|
||||
<button class="panel-help-button" type="button" aria-label="Show help for Model Graph" aria-expanded="false" aria-controls="panel-help-popover" data-help-title="Model Graph" data-help="Visual model graph linked to selected layers.">?</button>
|
||||
</div>
|
||||
<p class="panel-kicker" id="graph-kicker">Rows with matching graph nodes can be selected.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="netron-views" id="netron-views"></div>
|
||||
</aside>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<div class="panel-help-popover" id="panel-help-popover" role="tooltip" hidden>
|
||||
<strong id="panel-help-popover-title"></strong>
|
||||
<p id="panel-help-popover-text"></p>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
window.trtReportOptions = {
|
||||
dataName: 'analyze-data.json',
|
||||
analyzeName: 'analyze.md',
|
||||
graphMode: 'onnx-json',
|
||||
modelName: 'model.trt_inspect.onnx.json',
|
||||
netronBase: 'netron/',
|
||||
netronFrameMode: 'srcdoc'
|
||||
};
|
||||
</script>
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,596 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="description" content="Visualizer for neural network, deep learning and machine learning models." />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, viewport-fit=cover">
|
||||
<meta http-equiv="Content-Security-Policy" content="script-src 'self'">
|
||||
<meta name="version" content="9.1.1">
|
||||
<meta name="date" content="2026-06-05 14:43:12">
|
||||
<title>Netron</title>
|
||||
<link rel="stylesheet" type="text/css" href="grapher.css">
|
||||
<link rel="icon" href="data:,">
|
||||
<script type="text/javascript" src="netron.js"></script>
|
||||
<style>
|
||||
html { touch-action: none; overflow: hidden; width: 100%; height: 100%; -ms-text-size-adjust: 100%; -webkit-text-size-adjust: 100%; text-rendering: optimizeLegibility; -webkit-text-rendering: optimizeLegibility; -moz-text-rendering: optimizeLegibility; -ms-text-rendering: optimizeLegibility; -o-text-rendering: optimizeLegibility; -webkit-font-smoothing: antialiased; -moz-font-smoothing: antialiased; -ms-font-smoothing: antialiased; -o-font-smoothing: antialiased; }
|
||||
body { touch-action: none; overflow: hidden; width: 100%; height: 100%; margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe WPC", "Segoe UI", "Ubuntu", "Droid Sans", sans-serif, "PingFang SC"; font-size: 12px; text-rendering: geometricPrecision; }
|
||||
button { font-family: -apple-system, BlinkMacSystemFont, "Segoe WPC", "Segoe UI", "Ubuntu", "Droid Sans", sans-serif, "PingFang SC"; }
|
||||
.center { position: absolute; margin: auto; top: 0; right: 0; bottom: 0; left: 0; user-select: none; -webkit-user-select: none; -moz-user-select: none; }
|
||||
.select { user-select: text; -webkit-user-select: text; -moz-user-select: text; }
|
||||
.target { display: flex; height: 100%; width: 100%; overflow: auto; outline: none; touch-action: pan-x pan-y; }
|
||||
.canvas { margin: auto; flex-shrink: 0; text-rendering: geometricPrecision; user-select: none; -webkit-user-select: none; -moz-user-select: none; }
|
||||
.open-file-dialog { display: none; }
|
||||
.default { background-color: #ffffff; }
|
||||
.default .logo { display: none; }
|
||||
.default .target { display: flex; opacity: 1; }
|
||||
.default .toolbar { display: flex; }
|
||||
.toolbar { display: flex; align-items: center; position: absolute; bottom: 10px; left: 10px; padding: 0; margin: 0; user-select: none; -webkit-user-select: none; -moz-user-select: none; }
|
||||
.toolbar button:focus { outline: 0; }
|
||||
.toolbar-button { background: None; border-radius: 6px; border: 0; margin: 0; margin-right: 1px; padding: 0; fill: None; stroke: #777; cursor: pointer; width: 32px; height: 32px; user-select: none; }
|
||||
.toolbar-path { display: flex; align-items: center; }
|
||||
.toolbar-select { background: transparent; position: relative; width: 170px; margin: 4px 4px 4px 4px; }
|
||||
.toolbar-select select { width: 100%; appearance: none; -webkit-appearance: none; -moz-appearance: none; font-family: inherit; font-size: 12px; line-height: 16px; border: 1px solid; padding: 3px 18px 3px 10px; border-radius: 6px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; box-sizing: border-box; }
|
||||
.toolbar-select select { background: #777; border-color: #777; color: #fff; }
|
||||
.toolbar-select-arrow { position: absolute; top: 50%; right: 8px; transform: translateY(-50%); pointer-events: none; }
|
||||
.toolbar-select-arrow { fill: #ffffff; }
|
||||
.toolbar-select select:hover { background: #000000; border-color: #000000; }
|
||||
.toolbar-select select:focus { outline: 0; }
|
||||
.toolbar-path-back-button { background: #777; border-top-left-radius: 6px; border-bottom-left-radius: 6px; border: 1px solid; border-color: #777; margin: 4px 0px 4px 4px; padding: 5px 8px 5px 8px; cursor: pointer; color: #ffffff; font-size: 12px; line-height: 12px; transition: 0.1s; }
|
||||
.toolbar-path-back-button:hover { background: #000000; border-color: #000000; }
|
||||
.toolbar-path-name-button { background: #777; border: 0px; border-color: #777; color: #ffffff; border-left: 1px; border-left-color: #ffffff; margin: 4px 0 4px 1px; padding: 6px 8px 6px 8px; cursor: pointer; width: auto; font-size: 12px; line-height: 12px; transition: 0.1s; }
|
||||
.toolbar-path-name-button:hover { background: #000000; border-color: #000000; }
|
||||
.toolbar-path-name-button:last-child { border-top-right-radius: 6px; border-bottom-right-radius: 6px; }
|
||||
.toolbar-icon .border { stroke: #fff; }
|
||||
.toolbar-icon .stroke { stroke: #808080; }
|
||||
.toolbar-icon .fill { fill: #808080; }
|
||||
.toolbar-icon:hover .stroke { stroke: #000000; }
|
||||
.toolbar-icon:hover .fill { fill: #000000; }
|
||||
.message { display: none; opacity: 0; position: absolute; top: 0; left: 0; width: 100%; height: 100%; flex-direction: column; justify-content: flex-start; }
|
||||
.message-text { display: inline; text-align: center; width: 562px; font-size: 13px; line-height: 20px; margin-top: 50vh; padding-top: 56px; padding-bottom: 20px; margin-left: auto; margin-right: auto; user-select: text; -webkit-user-select: text; -moz-user-select: text; }
|
||||
.welcome .message-text a { text-decoration: none; color: #666666; }
|
||||
.welcome .message-text a:visited { color: inherit; }
|
||||
.welcome .message-text a:hover { color: #242424; text-decoration: underline; }
|
||||
.message-button { display: inline; text-align: center; width: 125px; margin-left: auto; margin-right: auto; }
|
||||
.logo-text { top: -57px; width: 582px; transition: 0.1s; }
|
||||
.logo-name { top: -170px; width: 582px; transition: 0.1s; }
|
||||
.logo-icon { left: 248px; top: -18px; width: 106px; height: 106px; transition: 0.1s; }
|
||||
.logo-spinner { left: 248px; top: -18px; width: 106px; height: 106px; display: none; }
|
||||
.logo-stroke { stroke: #444444; }
|
||||
.logo-fill { fill: #444444; }
|
||||
.logo-border { stroke: #555555; }
|
||||
.logo-glyph { fill: #444444; }
|
||||
.logo-button { font-size: 12px; font-weight: bold; line-height: 1.25; text-align: center; vertical-align: middle; min-width: 5em; height: 2.7em; border-radius: 1.3em; transition: 0.1s; user-select: none; -webkit-user-select: none; -moz-user-select: none; color: #444444; background-color: #ececec; border: 1px solid #444444; }
|
||||
.logo-button:hover { color: #ececec; background-color: #444444; cursor: pointer; transition: 0.2s; }
|
||||
.logo-button:focus { outline: 0; }
|
||||
.logo-message { display: none; height: 0px; }
|
||||
.logo-github { display: none; }
|
||||
.open-file-button { top: 170px; left: 0px; width: 10.5em; }
|
||||
.progress { top: 120px; height: 2px; width: 400px; }
|
||||
.progress-bar { height: 100%; width: 0%; background-color: #444444; }
|
||||
.notification .logo-name { display: none; }
|
||||
.notification .open-file-button { display: none; }
|
||||
.notification .progress { display: none; }
|
||||
.welcome body { background-color: #ececec; }
|
||||
.welcome { background-color: #ececec; color: #242424; }
|
||||
.welcome .message-text { display: none; opacity: 0; }
|
||||
.welcome .message-button { display: none; opacity: 0; }
|
||||
.welcome .target { display: none; opacity: 0; }
|
||||
.welcome .menu { background-color: #ffffff; }
|
||||
.welcome.spinner .logo-spinner { display: block; -webkit-animation: orbit 0.5s infinite linear; animation: orbit 0.5s infinite linear; cursor: wait; }
|
||||
.welcome.spinner .menu-button { display: none; }
|
||||
.welcome.notification .menu-button { display: none; }
|
||||
.notification body { background-color: #ececec; }
|
||||
.notification .message { display: flex; opacity: 1; }
|
||||
.notification .message-text { display: inline; opacity: 1; }
|
||||
.notification .message-button { display: inline; opacity: 1; }
|
||||
.alert { background-color: #ececec; color: #242424; }
|
||||
.alert .target { display: none; opacity: 0; }
|
||||
.alert .toolbar { display: none; opacity: 0; }
|
||||
.alert .menu { display: none; opacity: 0; }
|
||||
.alert .logo { display: none; opacity: 0; }
|
||||
.alert .message { display: flex; opacity: 1; }
|
||||
.alert .message-text { display: inline; opacity: 1; width: 50%; padding-top: 0px; }
|
||||
.alert .message-button { display: inline; opacity: 1; }
|
||||
.about { overflow: hidden; }
|
||||
.about .toolbar { display: none; }
|
||||
.about .logo { display: block; background-color: #ececec; color: #666666; }
|
||||
.about .logo-message { display: block; top: 132px; font-size: 14px; }
|
||||
.about .logo-github { display: block; top: 340px; width: 48px; height: 48px; }
|
||||
.about a { text-decoration: none; color: #666666; }
|
||||
.about a:visited { color: inherit; }
|
||||
.about a:hover { color: #242424; }
|
||||
.about .open-file-button { display: none; }
|
||||
.about .logo-name { display: none; }
|
||||
.about .notification { display: none; }
|
||||
.about .progress { display: none; }
|
||||
.about .menu-button { display: none; }
|
||||
.titlebar { color: #aaaaaa; display: none; height: 32px; position: fixed; top: 0; left: 0; right: 0; bottom: 0; z-index: 2; -webkit-app-region: drag; }
|
||||
.titlebar-visible { display: block; }
|
||||
.titlebar-content { display: block; padding: 0 142px; height: 100%; text-align: center; font-size: 14px; line-height: 32px; transition: all .1s ease-in-out; user-select: none; }
|
||||
.titlebar-content-text { display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.spinner .titlebar-content { opacity: 0; }
|
||||
.active .titlebar { color: #464646; transition: all 0.05s ease-in-out; }
|
||||
.titlebar-control-box { display: none; align-items: center; flex-direction: row-reverse; height: 100%; position: absolute; top: 0; right: 0; width: 138px; }
|
||||
.titlebar-control-box-visible { display: flex; }
|
||||
.titlebar-icon { width: 1em; height: 1em; vertical-align: -0.15em; fill: currentColor; overflow: hidden; }
|
||||
.titlebar-button { display: flex; justify-content: center; align-items: center; width: 46px; height: 32px; user-select: none; -webkit-app-region: no-drag; }
|
||||
.titlebar-button:hover { color: #000000; background-color: rgba(0, 0, 0, 0.15); }
|
||||
.titlebar-button-close:hover { color: #ffffff; background-color: #b43029; }
|
||||
.menu-button { display: flex; justify-content: center; align-items: center; color: #aaaaaa; font-size: 20px; height: 32px; width: 32px; position: fixed; top: 0; left: 0; right: 0; bottom: 0; z-index: 2; -webkit-app-region: no-drag; -webkit-app-region: no-drag; user-select: none; }
|
||||
.menu-button:hover { color: #000000; }
|
||||
.menu { display: block; position: absolute; left: -17em; width: 17em; top: 0; height: 100%; z-index: 2; background-color: #ececec; border-right: 1px solid rgba(255, 255, 255, 0.5); padding-top: 40px; padding-bottom: 2px; margin-left: 0; margin-top: 0; overflow: hidden; transition: 0.1s; }
|
||||
.menu .menu-group { margin-bottom: 12px; }
|
||||
.menu .menu-group .menu-group-header { display: block; border: none; border-radius: 0; color: black; width: 100%; text-align: left; margin: 4px 12px 5px 12px; white-space: no-wrap; font-size: 11px; font-weight: bold; color: #bbbbbb; white-space: nowrap; }
|
||||
.menu .menu-group .menu-command { display: block; border: none; border-radius: 0; background-color: transparent; color: black; width: 100%; text-align: left; padding: 4px 12px 5px 12px; font-size: 12px; }
|
||||
.menu .menu-group .menu-command:focus { color: #ffffff; background-color: #2e6bd2; outline: none; }
|
||||
.menu .menu-group .menu-command:disabled { color: #888888; }
|
||||
.menu .menu-group .menu-command .menu-label { display: block; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; }
|
||||
.menu .menu-group .menu-command .menu-shortcut { display: block; float: right; margin-left: 25px; color: #888888; }
|
||||
.menu .menu-group .menu-separator { border-top: 1px; border-bottom: 0; border-style: solid; border-color: #e5e5e5; margin-left: 12px; margin-right: 12px; }
|
||||
.about .titlebar-visible { opacity: 0; }
|
||||
@-webkit-keyframes orbit { 0% { -webkit-transform: rotate(0deg); transform: rotate(0deg); } 100% { -webkit-transform: rotate(360deg); transform: rotate(360deg); } }
|
||||
@keyframes orbit { 0% { -webkit-transform: rotate(0deg); transform: rotate(0deg); } 100% { -webkit-transform: rotate(360deg); transform: rotate(360deg); } }
|
||||
.welcome.spinner .logo-spinner-stroke { stroke: #ececec; }
|
||||
.welcome.spinner .logo-name { display: none; }
|
||||
.welcome.spinner .open-file-button { display: none; }
|
||||
.welcome.spinner .target { display: flex; opacity: 0; }
|
||||
.welcome .notification .logo-name { display: none; }
|
||||
.welcome .toolbar { display: none; }
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root { color-scheme: dark; }
|
||||
.default { background-color: #404040; }
|
||||
.target { background-color: #404040; }
|
||||
.welcome { background-color: #1e1e1e; color: #888888; }
|
||||
.alert { background-color: #1e1e1e; color: #888888; }
|
||||
.logo-stroke { stroke: #888888; }
|
||||
.logo-fill { fill: #888888; }
|
||||
.logo-border { stroke: #000000; }
|
||||
.logo-glyph { fill: #888888; }
|
||||
.logo-spinner-stroke { stroke: #ffffff; }
|
||||
.logo-button { color: #888888; background-color: #1e1e1e; border-color: #888888; }
|
||||
.logo-button:hover { color: #1e1e1e; background-color: #888888; }
|
||||
.welcome .progress-bar { background-color: #888888; }
|
||||
.welcome .menu { background-color: #2d2d2d }
|
||||
.about .logo { background-color: #1e1e1e; color: #888888; }
|
||||
.about a { color: #c6c6c6; }
|
||||
.about a:hover { color: #565656; }
|
||||
.welcome .message-text a { color: #c6c6c6; }
|
||||
.welcome .message-text a:visited { color: #c6c6c6; }
|
||||
.welcome .message-text a:hover { color: #565656; }
|
||||
.toolbar-icon .border { stroke: #333333; }
|
||||
.toolbar-icon .stroke { stroke: #aaaaaa; }
|
||||
.toolbar-icon .fill { fill: #aaaaaa; }
|
||||
.toolbar-icon:hover .stroke { stroke: #dfdfdf; }
|
||||
.toolbar-icon:hover .fill { fill: #dfdfdf; }
|
||||
.toolbar-path-back-button { background: #aaaaaa; border-color: #aaaaaa; color: #333333; }
|
||||
.toolbar-path-back-button:hover { background: #dfdfdf; border-color: #dfdfdf; }
|
||||
.toolbar-path-name-button { background: #aaaaaa ; border-color: #aaaaaa; color: #404040; }
|
||||
.toolbar-path-name-button:hover { background: #dfdfdf; border-color: #dfdfdf; }
|
||||
.toolbar-select select { background: #aaaaaa; border-color: #aaaaaa; color: #404040; }
|
||||
.toolbar-select select:hover { background: #dfdfdf; border-color: #dfdfdf; color: #404040; }
|
||||
.toolbar-select-arrow { fill: #404040; }
|
||||
.titlebar { color: #949494; }
|
||||
.welcome body { background-color: #1e1e1e; }
|
||||
.default body { background-color: #404040; }
|
||||
.active .titlebar { color: #c4c4c4; }
|
||||
.titlebar-button:hover { color: #ffffff; background-color: rgba(0, 0, 0, 0.15); }
|
||||
.titlebar-button-close:hover { color: #ffffff; background-color: #b43029; }
|
||||
.menu-button { color: #aaaaaa; }
|
||||
.menu-button:hover { color: #ffffff; }
|
||||
.menu { background-color: #2d2d2d; border-color: rgba(0, 0, 0, 0); }
|
||||
.menu .menu-group .menu-group-header { color: #666666; }
|
||||
.menu .menu-group .menu-command { color: #ffffff; }
|
||||
.menu .menu-group .menu-command:focus { color: #ffffff; background-color: #2e6bd2; }
|
||||
.menu .menu-group .menu-command:disabled { color: #888888; }
|
||||
.menu .menu-group .menu-command .shortcut { color: #888888; }
|
||||
.menu .menu-group .menu-separator { border-color: #363636; }
|
||||
}
|
||||
@media all and (max-width: 640px) {
|
||||
.logo { width: 240px; }
|
||||
.logo-text { opacity: 0; }
|
||||
.logo-name { opacity: 0; }
|
||||
.logo-icon { left: 0; width: 128px; height: 128px; }
|
||||
.logo-spinner { left: 0; width: 128px; height: 128px; }
|
||||
.logo .open-file-button { top: 204px; left: 0; }
|
||||
.message-text { padding-top: 68px; width: 320px; }
|
||||
.progress { top: 160px; height: 2px; width: 100px; }
|
||||
.about .logo { width: 100%; padding-left: 0; padding-right: 0; }
|
||||
.about .logo-message { top: 175px; font-size: 12px; }
|
||||
.about .logo-github { top: 370px; }
|
||||
}
|
||||
.sidebar { display: flex; flex-direction: column; font-family: -apple-system, BlinkMacSystemFont, "Segoe WPC", "Segoe UI", "Ubuntu", "Droid Sans", sans-serif; font-size: 12px; height: 100%; right: -100%; position: fixed; transition: 0.1s; top: 0; background-color: #ececec; color: #242424; overflow: hidden; border-left: 1px solid rgba(255, 255, 255, 0.5); opacity: 0; }
|
||||
.sidebar-title { font-weight: bold; font-size: 12px; letter-spacing: 0.5px; text-transform: uppercase; height: 20px; margin: 0; padding: 20px; user-select: none; -webkit-user-select: none; -moz-user-select: none; }
|
||||
.sidebar-closebutton { padding: 8px 8px 8px 32px; text-decoration: none; font-size: 25px; color: #777777; opacity: 1.0; display: block; transition: 0.2s; position: absolute; top: 0; right: 15px; margin-left: 50px; user-select: none; -webkit-user-select: none; -moz-user-select: none; }
|
||||
.sidebar-closebutton:hover { color: #242424; }
|
||||
.sidebar-content { display: flex; flex-direction: column; flex-grow: 1; height: 0; }
|
||||
.sidebar-header { font-weight: bold; font-size: 12px; letter-spacing: 0.5px; text-transform: uppercase; height: 20px; margin: 0; margin-top: 30px; padding-top: 10px; padding-bottom: 10px; user-select: none; -webkit-user-select: none; -moz-user-select: none; }
|
||||
.sidebar-section { font-weight: bold; font-size: 11px; text-transform: uppercase; line-height: 1.25; margin-top: 16px; margin-bottom: 16px; display: block; user-select: none; -webkit-user-select: none; -moz-user-select: none; cursor: default; }
|
||||
.sidebar-object { flex-grow: 1; padding: 0px 20px 20px 20px; overflow-y: auto; }
|
||||
.sidebar-item { margin-bottom: 0px; display: block; }
|
||||
.sidebar-item-name { float: left; font-size: 11px; min-width: 95px; max-width: 95px; padding-right: 5px; padding-top: 7px; display: block; }
|
||||
.sidebar-item-name input { color: #777; font-family: inherit; font-size: inherit; color: inherit; background-color: inherit; width: 100%; text-align: right; margin: 0; padding: 0; border: 0; outline: none; text-overflow: ellipsis; }
|
||||
.sidebar-item-value-list { margin: 0; margin-left: 105px; overflow: hidden; display: block; padding: 0; }
|
||||
.sidebar-item-value { font-size: 11px; background-color: #fcfcfc; border-radius: 2px; border: 1px solid #fcfcfc; margin-top: 3px; margin-bottom: 3px; overflow: auto; }
|
||||
.sidebar-item-value-content { background-color: #f8f8f8; border: 1px solid #f8f8f8; }
|
||||
.sidebar-item-value b { font-weight: bold; }
|
||||
.sidebar-item-value code { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace; overflow: auto; white-space: pre-wrap; word-wrap: break-word; }
|
||||
.sidebar-item-value pre { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace; margin: 0; overflow: auto; white-space: pre; word-wrap: normal; display: block; }
|
||||
.sidebar-item-value-line { padding: 4px 6px 4px 6px; overflow-x: auto; white-space: pre; }
|
||||
.sidebar-item-value-line-wrap { white-space: pre-wrap; overflow-x: hidden; overflow-wrap: break-word; }
|
||||
.sidebar-item-value-line-break { padding: 4px 6px 4px 6px; overflow-x: auto; white-space: pre; }
|
||||
.sidebar-item-value-line-link { padding: 4px 6px 4px 6px; cursor: default; overflow-x: auto; white-space: nowrap; }
|
||||
.sidebar-item-value-line-link:hover { text-decoration: underline; }
|
||||
.sidebar-item-value-line-border { padding: 4px 6px 4px 6px; border-top: 1px solid rgba(27, 31, 35, 0.05); }
|
||||
.sidebar-item-value-line-content { white-space: pre; word-wrap: normal; overflow: auto; display: block; }
|
||||
.sidebar-item-value-expander { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace; float: right; color: #aaa; cursor: pointer; user-select: none; -webkit-user-select: none; -moz-user-select: none; padding: 4px 6px 4px 4px; }
|
||||
.sidebar-item-value-expander:hover { color: #333; }
|
||||
.sidebar-item-value-button { display: flex; justify-content: center; align-items: center; font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace; width: 22px; height: 22px; cursor: pointer; user-select: none; -webkit-user-select: none; -moz-user-select: none; color: #aaa; }
|
||||
.sidebar-item-value-button svg use { fill: #aaa; stroke: #aaa; }
|
||||
.sidebar-item-value-button:hover svg use { fill: #333; stroke: #333; }
|
||||
.sidebar-item-value-button-tool { float: right; padding-left: 3px; }
|
||||
.sidebar-item-value-button-context { float: right; }
|
||||
.sidebar-item-selector {
|
||||
font-family: inherit; font-size: 12px;
|
||||
background-color: #fcfcfc; border: #fcfcfc; color: #333;
|
||||
border-radius: 2px; width: 100%; height: 23px; padding: 3px 12px 3px 7px;
|
||||
margin-top: 3px; margin-bottom: 3px; outline: none;
|
||||
box-sizing: border-box; -moz-box-sizing: border-box;
|
||||
appearance: none; -webkit-appearance: none; -moz-appearance: none;
|
||||
background-image: linear-gradient(45deg, transparent 50%, #333 50%), linear-gradient(135deg, #333 50%, transparent 50%);
|
||||
background-position: calc(100% - 12px) calc(10px), calc(100% - 7px) calc(10px);
|
||||
background-size: 5px 5px, 5px 5px;
|
||||
background-repeat: no-repeat;
|
||||
}
|
||||
.sidebar-separator { margin-bottom: 20px; }
|
||||
.sidebar-find-search { display: flex; align-items: center; background: #fff; border-radius: 16px; margin: 0px 20px 8px 20px; padding: 0px 8px 0px 8px; }
|
||||
.sidebar-find-query { width: 100vw; background: none; font-family: inherit; font-size: 13px; padding: 8px 8px 8px 8px; border: 0; outline: 0; }
|
||||
.sidebar-find-toggle { margin-left: auto; margin-right: 2px; width: 16px; height: 16px; cursor: pointer; }
|
||||
.sidebar-find-toggle input[type="checkbox"] { display: none; }
|
||||
.sidebar-find-toggle input[type="checkbox"]:not(:checked) + svg { fill: #ccc; stroke: #ccc; }
|
||||
.sidebar-find-toggle input[type="checkbox"]:checked + svg { fill: #555; stroke: #555; }
|
||||
.sidebar-find-toggle-icon { stroke: #555; fill: #555; width: 16px; height: 16px; }
|
||||
.sidebar-find-content { flex-grow: 1; padding: 0px 20px 20px 20px; overflow-y: auto; list-style-type: none; margin: 0; outline: 0; }
|
||||
.sidebar-find-content li *:first-child { margin-right: 2px; }
|
||||
.sidebar-find-content li { color: #666; font-size: 13px; height: 22px; line-height: 22px; padding: 0 12px 0 12px; outline: 0; white-space: nowrap; border-radius: 3px; user-select: none; -webkit-user-select: none; -moz-user-select: none; }
|
||||
.sidebar-find-content li.focus { background: #e5e5e5; color: #000; }
|
||||
.sidebar-find-content-icon { stroke: #555; fill: #555; float: left; width: 16px; height: 16px; padding: 3px; pointer-events: none; }
|
||||
.sidebar-documentation { flex-grow: 1; padding: 0px 20px 20px 20px; overflow-y: auto; font-size: 13px; line-height: 1.5; margin: 0; }
|
||||
.sidebar-documentation h1 { font-weight: bold; font-size: 13px; line-height: 1.25; border-bottom: 1px solid #e8e8e8; padding-bottom: 0.3em; margin-top: 0; margin-bottom: 16px; }
|
||||
.sidebar-documentation h2 { font-weight: bold; font-size: 13px; line-height: 1.25; margin-top: 20px; margin-bottom: 16px; text-transform: uppercase; border: 0; }
|
||||
.sidebar-documentation h3 { font-weight: bold; font-size: 11px; line-height: 1.25; }
|
||||
.sidebar-documentation p { margin-top: 4px; margin-bottom: 4px; margin-left: 0px; }
|
||||
.sidebar-documentation a { color: #237; }
|
||||
.sidebar-documentation code { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace; font-size: 12px; background-color: rgba(27, 31, 35, 0.05); padding: 0.2em 0.4em; margin: 0; border-radius: 3px; }
|
||||
.sidebar-documentation pre { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace; font-size: 12px; padding: 16px; overflow: auto; line-height: 1.45; background-color: rgba(27, 31, 35, 0.05); border-radius: 3px; }
|
||||
.sidebar-documentation pre code { font-size: 13px; padding: 16px; line-height: 1.45; background-color: transparent; padding: 0; border-radius: 0; }
|
||||
.sidebar-documentation tt { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace; font-weight: bold; font-size: 90%; background-color: rgba(27, 31, 35, 0.05); border-radius: 3px; padding: 0.2em 0.4em; margin: 0; }
|
||||
.sidebar-documentation dl dt { font-size: 13px; font-weight: bold; padding: 0; margin-top: 16px; margin-left: 0px; }
|
||||
.sidebar-documentation dd { padding: 0 16px; margin-left: 0; margin-bottom: 16px; }
|
||||
.sidebar-documentation ul { margin-top: 6px; margin-bottom: 6px; padding-left: 20px; }
|
||||
.sidebar-documentation blockquote { margin-left: 15px; margin-right: 15px; }
|
||||
@media (prefers-color-scheme: dark) {
|
||||
.sidebar html { color: #dfdfdf; }
|
||||
.sidebar { background-color: #2d2d2d; color: #dfdfdf; border-left: 1px solid rgba(0, 0, 0, 0); }
|
||||
.sidebar-closebutton { padding: 8px 8px 8px 32px; text-decoration: none; font-size: 25px; color: #777777; opacity: 1.0; display: block; transition: 0.2s; position: absolute; top: 0; right: 15px; margin-left: 50px; user-select: none; -webkit-user-select: none; -moz-user-select: none; }
|
||||
.sidebar-closebutton:hover { color: #ffffff; }
|
||||
.sidebar-item-value { background-color: #383838; border-color: #383838; }
|
||||
.sidebar-item-value-content { background-color: #3e3e3e; border-color: #3e3e3e; }
|
||||
.sidebar-item-value-line-border { border-color: rgba(0, 0, 0, 0.09); }
|
||||
.sidebar-item-selector { background-color: #383838; border: #383838; color: #dfdfdf; background-image: linear-gradient(45deg, transparent 50%, #aaa 50%), linear-gradient(135deg, #aaa 50%, transparent 50%); }
|
||||
.sidebar-item-disable-select { user-select: none; -webkit-user-select: none; -moz-user-select: none; }
|
||||
.sidebar-header { border-bottom-color: #2d2d2d; color: #dfdfdf; }
|
||||
.sidebar-documentation h1 { border-bottom: 1px solid #424242; color: #dfdfdf; }
|
||||
.sidebar-documentation h2 { color: #dfdfdf; }
|
||||
.sidebar-documentation p { color: #aaaaaa; }
|
||||
.sidebar-documentation a { color: #6688aa; }
|
||||
.sidebar-documentation tt { background-color:#1e1e1e; }
|
||||
.sidebar-documentation code { background-color: #1e1e1e; }
|
||||
.sidebar-documentation pre { background-color: #1e1e1e; }
|
||||
.sidebar-find-search { background: #383838; color: #dfdfdf; border-color: #424242; }
|
||||
.sidebar-find-toggle input[type="checkbox"]:not(:checked) + svg { fill: #555; stroke: #555; }
|
||||
.sidebar-find-toggle input[type="checkbox"]:checked + svg { fill: #aaa; stroke: #aaa; }
|
||||
.sidebar-find-content li { color: #aaaaaa; }
|
||||
.sidebar-find-content li.focus { background: #383838; color: #dfdfdf; }
|
||||
.sidebar-find-content-icon { stroke: #888888; fill: #888888; }
|
||||
.sidebar-item-value-expander { color: #888; }
|
||||
.sidebar-item-value-expander:hover { color: #e5e5e5; }
|
||||
.sidebar-item-value-button { color: #888; }
|
||||
.sidebar-item-value-button svg use { fill: #888; stroke: #888; }
|
||||
.sidebar-item-value-button:hover svg use { fill: #e5e5e5; stroke: #e5e5e5; }
|
||||
}
|
||||
@media screen and (prefers-reduced-motion: reduce) {
|
||||
.menu { transition: none; }
|
||||
.sidebar { transition: none; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body class="welcome spinner">
|
||||
<div id="target" class="target" tabindex="0">
|
||||
</div>
|
||||
<div id="sidebar" class="sidebar">
|
||||
<h1 id="sidebar-title" class="sidebar-title"></h1>
|
||||
<a id="sidebar-closebutton" class="sidebar-closebutton" href="javascript:void(0)" draggable="false">×</a>
|
||||
<div id="sidebar-content" class="sidebar-content"></div>
|
||||
<svg width="0" height="0" display="none">
|
||||
<defs>
|
||||
<symbol id="sidebar-icon-node" viewBox="0 0 20 20">
|
||||
<circle cx="10" cy="10" r="6" stroke-width="2" fill="none"/>
|
||||
</symbol>
|
||||
<symbol id="sidebar-icon-connection" viewBox="0 0 20 20">
|
||||
<line x1="4" y1="10" x2="15" y2="10" stroke-width="2" />
|
||||
<polyline points="11,6 15,10 11,14" stroke-width="2" />
|
||||
</symbol>
|
||||
<symbol id="sidebar-icon-weight" viewBox="0 0 20 20">
|
||||
<circle cx="5" cy="5" r="1" />
|
||||
<circle cx="10" cy="5" r="1" />
|
||||
<circle cx="15" cy="5" r="1" />
|
||||
<circle cx="5" cy="10" r="1" />
|
||||
<circle cx="10" cy="10" r="1" />
|
||||
<circle cx="15" cy="10" r="1" />
|
||||
<circle cx="5" cy="15" r="1" />
|
||||
<circle cx="10" cy="15" r="1" />
|
||||
<circle cx="15" cy="15" r="1" />
|
||||
</symbol>
|
||||
</defs>
|
||||
</svg>
|
||||
</div>
|
||||
<div id="toolbar" class="toolbar">
|
||||
<button id="zoom-in-button" class="toolbar-button" title="Zoom In">
|
||||
<svg class="toolbar-icon" viewbox="0 0 100 100">
|
||||
<circle class="border" cx="50" cy="50" r="35" stroke-width="8" stroke="#fff"></circle>
|
||||
<line class="border" x1="50" y1="38" x2="50" y2="62" stroke-width="8" stroke-linecap="round" stroke="#fff"></line>
|
||||
<line class="border" x1="38" y1="50" x2="62" y2="50" stroke-width="8" stroke-linecap="round" stroke="#fff"></line>
|
||||
<line class="border" x1="78" y1="78" x2="82" y2="82" stroke-width="12" stroke-linecap="square" stroke="#fff"></line>
|
||||
<circle class="stroke" cx="50" cy="50" r="35" stroke-width="4"></circle>
|
||||
<line class="stroke" x1="50" y1="38" x2="50" y2="62" stroke-width="4" stroke-linecap="round"></line>
|
||||
<line class="stroke" x1="38" y1="50" x2="62" y2="50" stroke-width="4" stroke-linecap="round"></line>
|
||||
<line class="stroke" x1="78" y1="78" x2="82" y2="82" stroke-width="8" stroke-linecap="square"></line>
|
||||
</svg>
|
||||
</button>
|
||||
<button id="zoom-out-button" class="toolbar-button" title="Zoom Out">
|
||||
<svg class="toolbar-icon" viewbox="0 0 100 100">
|
||||
<circle class="border" cx="50" cy="50" r="35" stroke-width="8" stroke="#fff"></circle>
|
||||
<line class="border" x1="38" y1="50" x2="62" y2="50" stroke-width="8" stroke-linecap="round" stroke="#fff"></line>
|
||||
<line class="border" x1="78" y1="78" x2="82" y2="82" stroke-width="12" stroke-linecap="square" stroke="#fff"></line>
|
||||
<circle class="stroke" cx="50" cy="50" r="35" stroke-width="4"></circle>
|
||||
<line class="stroke" x1="38" y1="50" x2="62" y2="50" stroke-width="4" stroke-linecap="round"></line>
|
||||
<line class="stroke" x1="78" y1="78" x2="82" y2="82" stroke-width="8" stroke-linecap="square"></line>
|
||||
</svg>
|
||||
</button>
|
||||
<button id="sidebar-model-button" class="toolbar-button" title="Model Properties">
|
||||
<svg class="toolbar-icon" viewbox="0 0 100 100">
|
||||
<rect class="border" x="12" y="12" width="76" height="76" rx="16" ry="16" stroke-width="8"></rect>
|
||||
<line class="border" x1="28" y1="37" x2="32" y2="37" stroke-width="8" stroke-linecap="round" stroke="#fff"></line>
|
||||
<line class="border" x1="28" y1="50" x2="32" y2="50" stroke-width="8" stroke-linecap="round" stroke="#fff"></line>
|
||||
<line class="border" x1="28" y1="63" x2="32" y2="63" stroke-width="8" stroke-linecap="round" stroke="#fff"></line>
|
||||
<line class="border" x1="40" y1="37" x2="70" y2="37" stroke-width="8" stroke-linecap="round" stroke="#fff"></line>
|
||||
<line class="border" x1="40" y1="50" x2="70" y2="50" stroke-width="8" stroke-linecap="round" stroke="#fff"></line>
|
||||
<line class="border" x1="40" y1="63" x2="70" y2="63" stroke-width="8" stroke-linecap="round" stroke="#fff"></line>
|
||||
<rect class="stroke" x="12" y="12" width="76" height="76" rx="16" ry="16" stroke-width="4"></rect>
|
||||
<line class="stroke" x1="28" y1="37" x2="32" y2="37" stroke-width="4" stroke-linecap="round"></line>
|
||||
<line class="stroke" x1="28" y1="50" x2="32" y2="50" stroke-width="4" stroke-linecap="round"></line>
|
||||
<line class="stroke" x1="28" y1="63" x2="32" y2="63" stroke-width="4" stroke-linecap="round"></line>
|
||||
<line class="stroke" x1="40" y1="37" x2="70" y2="37" stroke-width="4" stroke-linecap="round"></line>
|
||||
<line class="stroke" x1="40" y1="50" x2="70" y2="50" stroke-width="4" stroke-linecap="round"></line>
|
||||
<line class="stroke" x1="40" y1="63" x2="70" y2="63" stroke-width="4" stroke-linecap="round"></line>
|
||||
</svg>
|
||||
</button>
|
||||
<button id="sidebar-target-button" class="toolbar-button" title="Target Properties">
|
||||
<svg class="toolbar-icon" viewBox="0 0 100 100">
|
||||
<rect class="border" x="12" y="12" width="76" height="76" rx="16" ry="16" stroke-width="8"></rect>
|
||||
<circle class="border" cx="50" cy="50" r="18" stroke-width="8"></circle>
|
||||
<circle class="border" cx="50" cy="50" r="3" stroke-width="8"></circle>
|
||||
<rect class="stroke" x="12" y="12" width="76" height="76" rx="16" ry="16" stroke-width="4"></rect>
|
||||
<circle class="stroke" cx="50" cy="50" r="18" stroke-width="4"></circle>
|
||||
<circle class="stroke fill" cx="50" cy="50" r="3" stroke-width="4"></circle>
|
||||
</svg>
|
||||
</button>
|
||||
<div id="toolbar-navigator" class="toolbar-select">
|
||||
<select id="toolbar-target-selector">
|
||||
<option value="Graphs" disabled="true">— Graphs ——</option>
|
||||
<option value="add">add</option>
|
||||
<option value="subtract">subtract</option>
|
||||
<option value="multiply">multiply</option>
|
||||
<option value="Functions" disabled="true">— Functions ——</option>
|
||||
<option value="line">foo_bar.foo</option>
|
||||
</select>
|
||||
<svg class="toolbar-select-arrow" viewBox="0 0 10 6" width="10" height="6">
|
||||
<path d="M0 0L5 6L10 0Z" />
|
||||
</svg>
|
||||
</div>
|
||||
<div id="toolbar-path" class="toolbar-path">
|
||||
<button id="toolbar-path-back-button" class="toolbar-path-back-button" title="Back">
|
||||
❮
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="logo" class="center logo">
|
||||
<a href="https://github.com/lutzroeder/netron" target="blank_">
|
||||
<svg class="center logo-text" viewbox="0 0 5120 1024">
|
||||
<g transform="scale(9) translate(-44,-15)">
|
||||
<g transform="matrix(100,0,0,100,60.9965,126)">
|
||||
<path class="logo-glyph" d="M0.089,0L0.089,-0.745L0.595,-0.147L0.595,-0.715L0.656,-0.715L0.656,0.021L0.15,-0.578L0.15,0L0.089,0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(100,0,0,100,164.341,126)">
|
||||
<path class="logo-glyph" d="M0.089,0L0.089,-0.715L0.443,-0.715L0.443,-0.654L0.154,-0.654L0.154,-0.43L0.443,-0.43L0.443,-0.369L0.154,-0.369L0.154,-0.061L0.443,-0.061L0.443,0L0.089,0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(100,0,0,100,244.491,126)">
|
||||
<path class="logo-glyph" d="M0.216,0L0.216,-0.654L0.019,-0.654L0.019,-0.715L0.478,-0.715L0.478,-0.654L0.281,-0.654L0.281,0L0.216,0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(100,0,0,100,323.031,126)">
|
||||
<path class="logo-glyph" d="M0.154,-0.658L0.154,-0.394L0.219,-0.394C0.28,-0.394 0.322,-0.404 0.346,-0.423C0.37,-0.442 0.382,-0.475 0.382,-0.522C0.382,-0.571 0.369,-0.606 0.345,-0.627C0.32,-0.648 0.278,-0.658 0.219,-0.658L0.154,-0.658ZM0.523,0L0.444,0L0.193,-0.341L0.154,-0.341L0.154,0L0.089,0L0.089,-0.715L0.22,-0.715C0.298,-0.715 0.356,-0.699 0.394,-0.667C0.433,-0.634 0.452,-0.585 0.452,-0.52C0.452,-0.464 0.436,-0.421 0.403,-0.389C0.37,-0.357 0.324,-0.341 0.266,-0.341L0.523,0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(100,0,0,100,520.979,126)">
|
||||
<path class="logo-glyph" d="M0.089,0L0.089,-0.745L0.595,-0.147L0.595,-0.715L0.656,-0.715L0.656,0.021L0.15,-0.578L0.15,0L0.089,0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
<svg class="center logo-icon" viewbox="0 0 1024 1024">
|
||||
<circle class="logo-stroke" cx="512" cy="512" r="431" fill="none" stroke-width="32"></circle>
|
||||
<circle class="logo-border" cx="512" cy="512" r="450" fill="none" stroke-width="6"></circle>
|
||||
<circle class="logo-border" cx="512" cy="512" r="412" fill="none" stroke-width="6"></circle>
|
||||
<line class="logo-stroke" x1="296" y1="392" x2="540" y2="280" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="296" y1="632" x2="540" y2="280" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="296" y1="392" x2="540" y2="435" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="296" y1="632" x2="540" y2="435" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="296" y1="392" x2="540" y2="590" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="296" y1="632" x2="540" y2="590" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="296" y1="392" x2="540" y2="744" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="296" y1="632" x2="540" y2="744" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="540" y1="280" x2="785" y2="512" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="540" y1="590" x2="785" y2="512" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="540" y1="435" x2="785" y2="512" stroke-width="12"></line>
|
||||
<line class="logo-stroke" x1="540" y1="744" x2="785" y2="512" stroke-width="12"></line>
|
||||
<g transform="translate(296, 392)">
|
||||
<circle class="logo-fill" cx="0" cy="0" r="51"></circle>
|
||||
<circle class="logo-border" cx="0" cy="0" r="51" fill="none" stroke-width="6"></circle>
|
||||
</g>
|
||||
<g transform="translate(296, 632)">
|
||||
<circle class="logo-fill" cx="0" cy="0" r="51"></circle>
|
||||
<circle class="logo-border" cx="0" cy="0" r="51" fill="none" stroke-width="6"></circle>
|
||||
</g>
|
||||
<g transform="translate(540, 280)">
|
||||
<circle class="logo-fill" cx="0" cy="0" r="51"></circle>
|
||||
<circle class="logo-border" cx="0" cy="0" r="51" fill="none" stroke-width="6"></circle>
|
||||
</g>
|
||||
<g transform="translate(540, 435)">
|
||||
<circle class="logo-fill" cx="0" cy="0" r="51"></circle>
|
||||
<circle class="logo-border" cx="0" cy="0" r="51" fill="none" stroke-width="6"></circle>
|
||||
</g>
|
||||
<g transform="translate(540, 590)">
|
||||
<circle class="logo-fill" cx="0" cy="0" r="51"></circle>
|
||||
<circle class="logo-border" cx="0" cy="0" r="51" fill="none" stroke-width="6"></circle>
|
||||
</g>
|
||||
<g transform="translate(540, 744)">
|
||||
<circle class="logo-fill" cx="0" cy="0" r="51"></circle>
|
||||
<circle class="logo-border" cx="0" cy="0" r="51" fill="none" stroke-width="6"></circle>
|
||||
</g>
|
||||
<g transform="translate(785, 512)">
|
||||
<circle class="logo-fill" cx="0" cy="0" r="51"></circle>
|
||||
<circle class="logo-border" cx="0" cy="0" r="51" fill="none" stroke-width="6"></circle>
|
||||
</g>
|
||||
</svg>
|
||||
<svg id="logo-spinner" class="center logo-spinner" viewbox="0 0 1024 1024">
|
||||
<g transform="translate(512, 512)" style="opacity: 1">
|
||||
<path class="logo-spinner-stroke" d="M-431,0 A-431,-431 0 0,1 0,-431" stroke-width="24" fill="None"></path>
|
||||
</g>
|
||||
</svg>
|
||||
</a>
|
||||
<a href="https://www.lutzroeder.com" target="blank_">
|
||||
<svg class="center logo-name" viewbox="0 0 5120 300">
|
||||
<g transform="scale(5.8) translate(20, 0)">
|
||||
<g transform="matrix(30,0,0,30,18.9123,38)">
|
||||
<path class="logo-glyph" d="M0.089,-0L0.089,-0.715L0.154,-0.715L0.154,-0.061L0.399,-0.061L0.399,-0L0.089,-0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,46.7613,38)">
|
||||
<path class="logo-glyph" d="M0.086,-0.715L0.15,-0.715L0.15,-0.248C0.15,-0.177 0.166,-0.125 0.198,-0.091C0.23,-0.056 0.28,-0.039 0.346,-0.039C0.412,-0.039 0.46,-0.056 0.493,-0.091C0.525,-0.125 0.541,-0.177 0.541,-0.248L0.541,-0.715L0.606,-0.715L0.606,-0.269C0.606,-0.172 0.584,-0.1 0.542,-0.052C0.499,-0.005 0.433,0.019 0.346,0.019C0.259,0.019 0.193,-0.005 0.15,-0.052C0.107,-0.1 0.086,-0.172 0.086,-0.269L0.086,-0.715Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,83.5133,38)">
|
||||
<path class="logo-glyph" d="M0.216,-0L0.216,-0.654L0.019,-0.654L0.019,-0.715L0.478,-0.715L0.478,-0.654L0.281,-0.654L0.281,-0L0.216,-0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,114.421,38)">
|
||||
<path class="logo-glyph" d="M0.012,-0L0.437,-0.656L0.074,-0.656L0.074,-0.715L0.548,-0.715L0.125,-0.06L0.505,-0.06L0.505,-0L0.012,-0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,171.777,38)">
|
||||
<path class="logo-glyph" d="M0.154,-0.658L0.154,-0.394L0.219,-0.394C0.28,-0.394 0.322,-0.404 0.346,-0.423C0.37,-0.442 0.382,-0.475 0.382,-0.522C0.382,-0.571 0.369,-0.606 0.345,-0.627C0.32,-0.648 0.278,-0.658 0.219,-0.658L0.154,-0.658ZM0.523,-0L0.444,-0L0.193,-0.341L0.154,-0.341L0.154,-0L0.089,-0L0.089,-0.715L0.22,-0.715C0.298,-0.715 0.356,-0.699 0.394,-0.667C0.433,-0.634 0.452,-0.585 0.452,-0.52C0.452,-0.464 0.436,-0.421 0.403,-0.389C0.37,-0.357 0.324,-0.341 0.266,-0.341L0.523,-0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,203.607,38)">
|
||||
<path class="logo-glyph" d="M0.437,-0.039C0.479,-0.039 0.519,-0.047 0.557,-0.063C0.595,-0.078 0.629,-0.101 0.659,-0.131C0.689,-0.161 0.712,-0.196 0.727,-0.234C0.743,-0.273 0.751,-0.313 0.751,-0.356C0.751,-0.399 0.743,-0.44 0.728,-0.478C0.712,-0.516 0.689,-0.55 0.659,-0.581C0.63,-0.611 0.596,-0.634 0.558,-0.649C0.52,-0.665 0.48,-0.673 0.437,-0.673C0.395,-0.673 0.355,-0.665 0.317,-0.649C0.28,-0.634 0.246,-0.611 0.216,-0.581C0.186,-0.55 0.163,-0.516 0.147,-0.478C0.132,-0.44 0.124,-0.399 0.124,-0.356C0.124,-0.313 0.132,-0.272 0.147,-0.234C0.163,-0.196 0.186,-0.161 0.216,-0.131C0.246,-0.101 0.279,-0.078 0.316,-0.062C0.354,-0.047 0.394,-0.039 0.437,-0.039ZM0.82,-0.356C0.82,-0.306 0.81,-0.258 0.791,-0.212C0.772,-0.167 0.744,-0.126 0.708,-0.091C0.671,-0.055 0.63,-0.028 0.583,-0.009C0.537,0.01 0.488,0.019 0.437,0.019C0.386,0.019 0.337,0.01 0.291,-0.009C0.245,-0.028 0.203,-0.055 0.167,-0.091C0.131,-0.127 0.103,-0.168 0.084,-0.213C0.065,-0.258 0.055,-0.306 0.055,-0.356C0.055,-0.407 0.065,-0.455 0.084,-0.501C0.103,-0.546 0.131,-0.587 0.167,-0.623C0.203,-0.659 0.244,-0.685 0.29,-0.704C0.335,-0.722 0.385,-0.731 0.437,-0.731C0.49,-0.731 0.539,-0.722 0.585,-0.703C0.631,-0.685 0.672,-0.658 0.708,-0.623C0.744,-0.587 0.772,-0.546 0.791,-0.501C0.81,-0.455 0.82,-0.407 0.82,-0.356Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,245.853,38)">
|
||||
<path class="logo-glyph" d="M0.089,-0L0.089,-0.715L0.443,-0.715L0.443,-0.654L0.154,-0.654L0.154,-0.43L0.443,-0.43L0.443,-0.369L0.154,-0.369L0.154,-0.061L0.443,-0.061L0.443,-0L0.089,-0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,277.243,38)">
|
||||
<path class="logo-glyph" d="M0.154,-0.056L0.245,-0.056C0.319,-0.056 0.371,-0.06 0.402,-0.069C0.433,-0.077 0.459,-0.091 0.481,-0.111C0.511,-0.139 0.534,-0.174 0.549,-0.215C0.564,-0.257 0.572,-0.305 0.572,-0.358C0.572,-0.413 0.564,-0.461 0.549,-0.502C0.533,-0.544 0.51,-0.578 0.48,-0.605C0.457,-0.625 0.429,-0.64 0.396,-0.648C0.364,-0.657 0.306,-0.661 0.224,-0.661L0.154,-0.661L0.154,-0.056ZM0.089,-0L0.089,-0.715L0.2,-0.715C0.299,-0.715 0.37,-0.71 0.412,-0.7C0.453,-0.69 0.489,-0.674 0.519,-0.65C0.559,-0.618 0.589,-0.578 0.61,-0.528C0.631,-0.478 0.641,-0.421 0.641,-0.357C0.641,-0.293 0.631,-0.236 0.61,-0.186C0.589,-0.136 0.559,-0.096 0.52,-0.066C0.489,-0.042 0.454,-0.025 0.414,-0.015C0.374,-0.005 0.31,-0 0.222,-0L0.089,-0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,314.142,38)">
|
||||
<path class="logo-glyph" d="M0.089,-0L0.089,-0.715L0.443,-0.715L0.443,-0.654L0.154,-0.654L0.154,-0.43L0.443,-0.43L0.443,-0.369L0.154,-0.369L0.154,-0.061L0.443,-0.061L0.443,-0L0.089,-0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,345.532,38)">
|
||||
<path class="logo-glyph" d="M0.154,-0.658L0.154,-0.394L0.219,-0.394C0.28,-0.394 0.322,-0.404 0.346,-0.423C0.37,-0.442 0.382,-0.475 0.382,-0.522C0.382,-0.571 0.369,-0.606 0.345,-0.627C0.32,-0.648 0.278,-0.658 0.219,-0.658L0.154,-0.658ZM0.523,-0L0.444,-0L0.193,-0.341L0.154,-0.341L0.154,-0L0.089,-0L0.089,-0.715L0.22,-0.715C0.298,-0.715 0.356,-0.699 0.394,-0.667C0.433,-0.634 0.452,-0.585 0.452,-0.52C0.452,-0.464 0.436,-0.421 0.403,-0.389C0.37,-0.357 0.324,-0.341 0.266,-0.341L0.523,-0Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,376.911,38)">
|
||||
<path class="logo-glyph" d="M0.06,-0.468L0.155,-0.731L0.228,-0.699L0.1,-0.452L0.06,-0.468Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
<g transform="matrix(30,0,0,30,401.549,38)">
|
||||
<path class="logo-glyph" d="M0.034,-0.12L0.09,-0.15C0.1,-0.115 0.118,-0.087 0.144,-0.068C0.169,-0.049 0.2,-0.039 0.236,-0.039C0.281,-0.039 0.316,-0.052 0.342,-0.079C0.367,-0.106 0.38,-0.143 0.38,-0.19C0.38,-0.224 0.371,-0.253 0.354,-0.276C0.337,-0.299 0.3,-0.325 0.244,-0.355C0.172,-0.393 0.124,-0.427 0.101,-0.456C0.077,-0.485 0.065,-0.519 0.065,-0.56C0.065,-0.611 0.082,-0.652 0.116,-0.684C0.151,-0.716 0.195,-0.732 0.25,-0.732C0.286,-0.732 0.317,-0.724 0.344,-0.709C0.37,-0.694 0.392,-0.671 0.408,-0.641L0.358,-0.611C0.347,-0.631 0.333,-0.647 0.314,-0.658C0.295,-0.668 0.272,-0.674 0.246,-0.674C0.211,-0.674 0.183,-0.663 0.162,-0.643C0.141,-0.622 0.131,-0.594 0.131,-0.559C0.131,-0.509 0.172,-0.462 0.255,-0.419C0.27,-0.411 0.281,-0.405 0.289,-0.401C0.35,-0.367 0.391,-0.336 0.411,-0.306C0.432,-0.276 0.442,-0.237 0.442,-0.19C0.442,-0.126 0.423,-0.075 0.386,-0.037C0.348,0 0.297,0.019 0.233,0.019C0.186,0.019 0.146,0.007 0.113,-0.016C0.079,-0.039 0.053,-0.074 0.034,-0.12Z" style="fill-rule:nonzero;"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
</a>
|
||||
<div class="center logo-message">
|
||||
<div style="height: 30px; text-align: center;">Version <span id="version" class="select">{version}</span></div>
|
||||
<div style="height: 30px; text-align: center;">Copyright © <a href="https://www.lutzroeder.com" target="blank_">Lutz Roeder</a></div>
|
||||
</div>
|
||||
<a id="logo-github" class="center logo-github" href="https://github.com/lutzroeder/netron" target="blank_">
|
||||
<svg viewbox="0 0 438.549 438.549">
|
||||
<path class="logo-fill" d="M409.132,114.573c-19.608-33.596-46.205-60.194-79.798-79.8C295.736,15.166,259.057,5.365,219.271,5.365
|
||||
c-39.781,0-76.472,9.804-110.063,29.408c-33.596,19.605-60.192,46.204-79.8,79.8C9.803,148.168,0,184.854,0,224.63
|
||||
c0,47.78,13.94,90.745,41.827,128.906c27.884,38.164,63.906,64.572,108.063,79.227c5.14,0.954,8.945,0.283,11.419-1.996
|
||||
c2.475-2.282,3.711-5.14,3.711-8.562c0-0.571-0.049-5.708-0.144-15.417c-0.098-9.709-0.144-18.179-0.144-25.406l-6.567,1.136
|
||||
c-4.187,0.767-9.469,1.092-15.846,1c-6.374-0.089-12.991-0.757-19.842-1.999c-6.854-1.231-13.229-4.086-19.13-8.559
|
||||
c-5.898-4.473-10.085-10.328-12.56-17.556l-2.855-6.57c-1.903-4.374-4.899-9.233-8.992-14.559
|
||||
c-4.093-5.331-8.232-8.945-12.419-10.848l-1.999-1.431c-1.332-0.951-2.568-2.098-3.711-3.429c-1.142-1.331-1.997-2.663-2.568-3.997
|
||||
c-0.572-1.335-0.098-2.43,1.427-3.289c1.525-0.859,4.281-1.276,8.28-1.276l5.708,0.853c3.807,0.763,8.516,3.042,14.133,6.851
|
||||
c5.614,3.806,10.229,8.754,13.846,14.842c4.38,7.806,9.657,13.754,15.846,17.847c6.184,4.093,12.419,6.136,18.699,6.136
|
||||
c6.28,0,11.704-0.476,16.274-1.423c4.565-0.952,8.848-2.383,12.847-4.285c1.713-12.758,6.377-22.559,13.988-29.41
|
||||
c-10.848-1.14-20.601-2.857-29.264-5.14c-8.658-2.286-17.605-5.996-26.835-11.14c-9.235-5.137-16.896-11.516-22.985-19.126
|
||||
c-6.09-7.614-11.088-17.61-14.987-29.979c-3.901-12.374-5.852-26.648-5.852-42.826c0-23.035,7.52-42.637,22.557-58.817
|
||||
c-7.044-17.318-6.379-36.732,1.997-58.24c5.52-1.715,13.706-0.428,24.554,3.853c10.85,4.283,18.794,7.952,23.84,10.994
|
||||
c5.046,3.041,9.089,5.618,12.135,7.708c17.705-4.947,35.976-7.421,54.818-7.421s37.117,2.474,54.823,7.421l10.849-6.849
|
||||
c7.419-4.57,16.18-8.758,26.262-12.565c10.088-3.805,17.802-4.853,23.134-3.138c8.562,21.509,9.325,40.922,2.279,58.24
|
||||
c15.036,16.18,22.559,35.787,22.559,58.817c0,16.178-1.958,30.497-5.853,42.966c-3.9,12.471-8.941,22.457-15.125,29.979
|
||||
c-6.191,7.521-13.901,13.85-23.131,18.986c-9.232,5.14-18.182,8.85-26.84,11.136c-8.662,2.286-18.415,4.004-29.263,5.146
|
||||
c9.894,8.562,14.842,22.077,14.842,40.539v60.237c0,3.422,1.19,6.279,3.572,8.562c2.379,2.279,6.136,2.95,11.276,1.995
|
||||
c44.163-14.653,80.185-41.062,108.068-79.226c27.88-38.161,41.825-81.126,41.825-128.906
|
||||
C438.536,184.851,428.728,148.168,409.132,114.573z"/>
|
||||
</svg>
|
||||
</a>
|
||||
<button id="open-file-button" class="center logo-button open-file-button" tabindex="0">Open Model…</button>
|
||||
<div class="center progress">
|
||||
<div id="progress-bar" class="progress-bar"></div>
|
||||
</div>
|
||||
<input type="file" id="open-file-dialog" class="open-file-dialog" multiple="false" accept="">
|
||||
<!-- Preload fonts to workaround Chrome SVG layout issue -->
|
||||
<div style="font-weight: normal; color: rgba(0, 0, 0, 0.01); user-select: none;">.</div>
|
||||
<div style="font-weight: bold; color: rgba(0, 0, 0, 0.01); user-select: none;">.</div>
|
||||
<div style="font-weight: bold; color: rgba(0, 0, 0, 0.01); user-select: none;">.</div>
|
||||
</div>
|
||||
<div id="message" class="message">
|
||||
<div id="message-text" class="message-text"></div>
|
||||
<button id="message-button" class="logo-button message-button" tabindex="0">OK</button>
|
||||
</div>
|
||||
<div id="titlebar" class="titlebar">
|
||||
<svg style="position: absolute; width: 0px; height: 0px; overflow: hidden;" aria-hidden="true">
|
||||
<symbol id="icon-arrow-right" viewBox="0 0 1024 1024">
|
||||
<path d="M698.75712 565.02272l-191.488 225.4848a81.73568 81.73568 0 0 1-62.48448 28.89728 81.89952 81.89952 0 0 1-62.40256-134.94272l146.432-172.4416-146.432-172.4416a81.92 81.92 0 0 1 124.88704-106.06592l191.488 225.4848a81.87904 81.87904 0 0 1 0 106.02496z"></path>
|
||||
</symbol>
|
||||
</svg>
|
||||
<div id="titlebar-content" class="titlebar-content">
|
||||
<span id="titlebar-content-text" class="titlebar-content-text"></span>
|
||||
</div>
|
||||
<div id="titlebar-control-box" class="titlebar-control-box">
|
||||
<div id="titlebar-close" class="titlebar-button titlebar-button-close" title="Close">
|
||||
<svg class="titlebar-icon" aria-hidden="true">
|
||||
<path d="M 0,0 0,0.7 4.3,5 0,9.3 0,10 0.7,10 5,5.7 9.3,10 10,10 10,9.3 5.7,5 10,0.7 10,0 9.3,0 5,4.3 0.7,0 Z"></path>
|
||||
</svg>
|
||||
</div>
|
||||
<div id="titlebar-toggle" class="titlebar-button" title="Maximize">
|
||||
<svg id="titlebar-maximize" class="titlebar-icon" aria-hidden="true" style="position: absolute;">
|
||||
<path d="M 0,0 0,10 10,10 10,0 Z M 1,1 9,1 9,9 1,9 Z"></path>
|
||||
</svg>
|
||||
<svg id="titlebar-restore" class="titlebar-icon" aria-hidden="true" style="position: absolute;">
|
||||
<path d="m 2,1e-5 0,2 -2,0 0,8 8,0 0,-2 2,0 0,-8 z m 1,1 6,0 0,6 -1,0 0,-5 -5,0 z m -2,2 6,0 0,6 -6,0 z"></path>
|
||||
</svg>
|
||||
</div>
|
||||
<div id="titlebar-minimize" class="titlebar-button" title="Minimize">
|
||||
<svg class="titlebar-icon" aria-hidden="true">
|
||||
<path d="M 0,5 10,5 10,6 0,6 Z"></path>
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="menu" class="menu"></div>
|
||||
<div id="menu-button" class="menu-button">≡</div>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,832 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://codex.local/trt-perf-analysis/analyze-data.schema.json",
|
||||
"title": "TensorRT Performance Analysis Data",
|
||||
"description": "Structured JSON emitted by scripts/analyze_trt_perf.py and consumed by the report app.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"schema_version",
|
||||
"folder",
|
||||
"validation",
|
||||
"results"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"const": "1.0"
|
||||
},
|
||||
"folder": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"model_name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"model_name_confidence": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
]
|
||||
},
|
||||
"model_name_evidence": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
},
|
||||
"model_components": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
},
|
||||
"validation": {
|
||||
"$ref": "#/$defs/top_level_validation"
|
||||
},
|
||||
"results": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"$ref": "#/$defs/result"
|
||||
}
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"top_level_validation": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"status",
|
||||
"backend_count",
|
||||
"passed_count",
|
||||
"failed_count",
|
||||
"warnings"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"passed",
|
||||
"partial",
|
||||
"failed"
|
||||
]
|
||||
},
|
||||
"backend_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"passed_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"failed_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"warnings": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
}
|
||||
}
|
||||
},
|
||||
"layer_profile_validation": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"status",
|
||||
"analysis_mode",
|
||||
"messages",
|
||||
"source_paths",
|
||||
"report_available"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"status": {
|
||||
"const": "passed"
|
||||
},
|
||||
"analysis_mode": {
|
||||
"const": "layer_profile"
|
||||
},
|
||||
"messages": {
|
||||
"$ref": "#/$defs/validation_messages"
|
||||
},
|
||||
"source_paths": {
|
||||
"$ref": "#/$defs/source_paths"
|
||||
},
|
||||
"report_available": {
|
||||
"const": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"layer_only_validation": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"status",
|
||||
"analysis_mode",
|
||||
"messages",
|
||||
"source_paths",
|
||||
"report_available"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"status": {
|
||||
"const": "passed"
|
||||
},
|
||||
"analysis_mode": {
|
||||
"const": "layer_only"
|
||||
},
|
||||
"messages": {
|
||||
"$ref": "#/$defs/validation_messages"
|
||||
},
|
||||
"source_paths": {
|
||||
"$ref": "#/$defs/source_paths"
|
||||
},
|
||||
"report_available": {
|
||||
"const": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"failed_validation": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"status",
|
||||
"analysis_mode",
|
||||
"messages",
|
||||
"source_paths",
|
||||
"report_available"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"status": {
|
||||
"const": "failed"
|
||||
},
|
||||
"analysis_mode": {
|
||||
"const": "unusable"
|
||||
},
|
||||
"messages": {
|
||||
"$ref": "#/$defs/validation_messages"
|
||||
},
|
||||
"source_paths": {
|
||||
"$ref": "#/$defs/source_paths"
|
||||
},
|
||||
"report_available": {
|
||||
"const": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"source_paths": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"layer",
|
||||
"profile"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"layer": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"profile": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"validation_messages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/validation_message"
|
||||
}
|
||||
},
|
||||
"validation_message": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"level",
|
||||
"message"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"level": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"error",
|
||||
"warning",
|
||||
"info"
|
||||
]
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/$defs/layer_profile_result"
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/layer_only_result"
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/failure_result"
|
||||
}
|
||||
]
|
||||
},
|
||||
"layer_profile_result": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"label",
|
||||
"validation",
|
||||
"layers",
|
||||
"profile",
|
||||
"layer_count",
|
||||
"profile_count",
|
||||
"graph",
|
||||
"count",
|
||||
"count_source",
|
||||
"timing",
|
||||
"model",
|
||||
"engines",
|
||||
"records",
|
||||
"type_breakdown",
|
||||
"tag_breakdown",
|
||||
"issues",
|
||||
"positives"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": "string"
|
||||
},
|
||||
"validation": {
|
||||
"$ref": "#/$defs/layer_profile_validation"
|
||||
},
|
||||
"layers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/layer"
|
||||
}
|
||||
},
|
||||
"profile": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/profile_entry"
|
||||
}
|
||||
},
|
||||
"layer_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"profile_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"graph": {
|
||||
"$ref": "#/$defs/graph"
|
||||
},
|
||||
"count": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 0
|
||||
},
|
||||
"count_source": {
|
||||
"type": "string"
|
||||
},
|
||||
"timing": {
|
||||
"$ref": "#/$defs/timing"
|
||||
},
|
||||
"model": {
|
||||
"$ref": "#/$defs/model"
|
||||
},
|
||||
"engines": {
|
||||
"$ref": "#/$defs/engines"
|
||||
},
|
||||
"records": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/record"
|
||||
}
|
||||
},
|
||||
"type_breakdown": {
|
||||
"$ref": "#/$defs/breakdown"
|
||||
},
|
||||
"tag_breakdown": {
|
||||
"$ref": "#/$defs/breakdown"
|
||||
},
|
||||
"issues": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/issue"
|
||||
}
|
||||
},
|
||||
"positives": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
}
|
||||
}
|
||||
},
|
||||
"layer_only_result": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"label",
|
||||
"validation",
|
||||
"layers",
|
||||
"layer_count",
|
||||
"graph",
|
||||
"model",
|
||||
"engines"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": "string"
|
||||
},
|
||||
"validation": {
|
||||
"$ref": "#/$defs/layer_only_validation"
|
||||
},
|
||||
"layers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/layer"
|
||||
}
|
||||
},
|
||||
"layer_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"graph": {
|
||||
"$ref": "#/$defs/graph"
|
||||
},
|
||||
"model": {
|
||||
"$ref": "#/$defs/model"
|
||||
},
|
||||
"engines": {
|
||||
"$ref": "#/$defs/engines"
|
||||
}
|
||||
}
|
||||
},
|
||||
"failure_result": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"label",
|
||||
"validation"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": "string"
|
||||
},
|
||||
"validation": {
|
||||
"$ref": "#/$defs/failed_validation"
|
||||
},
|
||||
"layer_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"profile_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"layer": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"Name",
|
||||
"LayerType",
|
||||
"Inputs",
|
||||
"Outputs"
|
||||
],
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"Name": {
|
||||
"type": "string"
|
||||
},
|
||||
"LayerType": {
|
||||
"type": "string"
|
||||
},
|
||||
"Inputs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/tensor"
|
||||
}
|
||||
},
|
||||
"Outputs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/tensor"
|
||||
}
|
||||
},
|
||||
"Constants": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/tensor"
|
||||
}
|
||||
},
|
||||
"Metadata": {
|
||||
"type": [
|
||||
"string",
|
||||
"object",
|
||||
"array",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"StreamId": {
|
||||
"type": [
|
||||
"integer",
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"TacticName": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"tensor": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"Name"
|
||||
],
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"Name": {
|
||||
"type": "string"
|
||||
},
|
||||
"Datatype": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"Dimensions": {
|
||||
"$ref": "#/$defs/number_array"
|
||||
},
|
||||
"Format": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"StrideOrder": {
|
||||
"$ref": "#/$defs/number_array"
|
||||
},
|
||||
"Strides": {
|
||||
"$ref": "#/$defs/number_array"
|
||||
}
|
||||
}
|
||||
},
|
||||
"profile_entry": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"name",
|
||||
"timeMs",
|
||||
"averageMs",
|
||||
"medianMs",
|
||||
"percentage"
|
||||
],
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"timeMs": {
|
||||
"type": "number"
|
||||
},
|
||||
"averageMs": {
|
||||
"type": "number"
|
||||
},
|
||||
"medianMs": {
|
||||
"type": "number"
|
||||
},
|
||||
"percentage": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"record": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"name",
|
||||
"layer_type",
|
||||
"raw_layer_type",
|
||||
"tactic",
|
||||
"time_ms",
|
||||
"avg_ms",
|
||||
"median_ms",
|
||||
"percentage",
|
||||
"tags",
|
||||
"layer"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"layer_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"raw_layer_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"tactic": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"time_ms": {
|
||||
"type": "number"
|
||||
},
|
||||
"avg_ms": {
|
||||
"type": "number"
|
||||
},
|
||||
"median_ms": {
|
||||
"type": "number"
|
||||
},
|
||||
"percentage": {
|
||||
"type": "number"
|
||||
},
|
||||
"tags": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
},
|
||||
"layer": {
|
||||
"$ref": "#/$defs/layer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"graph": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"edge_count",
|
||||
"external_inputs",
|
||||
"graph_outputs"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"edge_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"external_inputs": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
},
|
||||
"graph_outputs": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
}
|
||||
}
|
||||
},
|
||||
"timing": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"average_inference_ms",
|
||||
"total_time_ms",
|
||||
"total_percentage"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"average_inference_ms": {
|
||||
"type": "number"
|
||||
},
|
||||
"total_time_ms": {
|
||||
"type": "number"
|
||||
},
|
||||
"total_percentage": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"model": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"category",
|
||||
"confidence",
|
||||
"evidence",
|
||||
"config_path",
|
||||
"hidden_size",
|
||||
"intermediate_size",
|
||||
"num_layers",
|
||||
"num_heads",
|
||||
"max_position_embeddings",
|
||||
"sequence_length",
|
||||
"dynamic_dims",
|
||||
"input_shapes"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_confidence": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
]
|
||||
},
|
||||
"name_evidence": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
},
|
||||
"components": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
},
|
||||
"category": {
|
||||
"type": "string"
|
||||
},
|
||||
"confidence": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
]
|
||||
},
|
||||
"evidence": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
},
|
||||
"config_path": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"hidden_size": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"intermediate_size": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"num_layers": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"num_heads": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"max_position_embeddings": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequence_length": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"dynamic_dims": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"input_shapes": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"$ref": "#/$defs/number_array"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"engines": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"engine_files",
|
||||
"subgraphs",
|
||||
"streams",
|
||||
"inferred_count",
|
||||
"source"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"engine_files": {
|
||||
"$ref": "#/$defs/string_array"
|
||||
},
|
||||
"subgraphs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
]
|
||||
}
|
||||
},
|
||||
"streams": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
]
|
||||
}
|
||||
},
|
||||
"inferred_count": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"minimum": 0
|
||||
},
|
||||
"source": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"breakdown": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/breakdown_item"
|
||||
}
|
||||
},
|
||||
"breakdown_item": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"name",
|
||||
"count",
|
||||
"time_ms",
|
||||
"percentage"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"time_ms": {
|
||||
"type": "number"
|
||||
},
|
||||
"percentage": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
},
|
||||
"issue": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"score",
|
||||
"title",
|
||||
"evidence",
|
||||
"next_check",
|
||||
"confidence"
|
||||
],
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"score": {
|
||||
"type": "number"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"evidence": {
|
||||
"type": "string"
|
||||
},
|
||||
"next_check": {
|
||||
"type": "string"
|
||||
},
|
||||
"confidence": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"string_array": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"number_array": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Validate and extract structured TensorRT layer/profile analysis data.
|
||||
|
||||
The script intentionally uses only Python built-in modules so it can run in
|
||||
minimal benchmark environments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from typing import Optional, Sequence
|
||||
|
||||
from trt_perf.data import DataError, extract_perf_data
|
||||
from trt_perf.render_json import render_results_json
|
||||
|
||||
|
||||
def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate and analyze TensorRT layer/profile JSON performance data."
|
||||
)
|
||||
parser.add_argument(
|
||||
"path",
|
||||
nargs="?",
|
||||
help="Folder directly containing one or more layers_*.json/profile_*.json backend pairs. Omit when using --data.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--data",
|
||||
dest="data_specs",
|
||||
action="append",
|
||||
nargs="+",
|
||||
metavar="PATH",
|
||||
help="Explicit backend input. Use --data layers.json for layer-only analysis or --data layers.json profile.json for a layer/profile pair. Repeat for multiple backends.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-name",
|
||||
help="Optional model name from the user prompt or caller context. It is cleaned before serialization.",
|
||||
)
|
||||
parser.add_argument("--output", help="Optional path to write the generated analysis JSON.")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
args = parse_args(argv)
|
||||
|
||||
try:
|
||||
data = extract_perf_data(args.path, args.data_specs, args.model_name)
|
||||
except DataError as exc:
|
||||
sys.stderr.write(f"error: {exc}\n")
|
||||
return 2
|
||||
|
||||
report = render_results_json(data)
|
||||
|
||||
if args.output:
|
||||
try:
|
||||
with open(args.output, "w", encoding="utf-8") as handle:
|
||||
handle.write(report)
|
||||
except OSError as exc:
|
||||
sys.stderr.write(f"error: unable to write {args.output}: {exc}\n")
|
||||
return 2
|
||||
else:
|
||||
sys.stdout.write(report)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Create a standalone TensorRT performance report folder.
|
||||
|
||||
This script keeps report packaging deterministic and cross-platform: it runs
|
||||
the analyzer, validates the generated JSON, then copies the browser report
|
||||
template without relying on shell tools such as rsync.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Sequence
|
||||
|
||||
from trt_perf.data import infer_model_name_metadata
|
||||
|
||||
|
||||
def script_dir() -> Path:
|
||||
return Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def skill_root() -> Path:
|
||||
return script_dir().parent
|
||||
|
||||
|
||||
def default_template_dir() -> Path:
|
||||
return skill_root() / "report_template"
|
||||
|
||||
|
||||
def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate and package a standalone TensorRT performance report."
|
||||
)
|
||||
parser.add_argument(
|
||||
"path",
|
||||
nargs="?",
|
||||
help="Folder directly containing layers_*.json and/or profile_*.json inputs. Pass one component/report folder per invocation. Omit when using --analyze-data.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--analyze-data",
|
||||
help="Existing analyze-data.json to validate and package in place. The file must be named analyze-data.json.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-parent",
|
||||
help="Directory where an auto-named report folder should be created. Defaults to the current working directory unless that is inside the skill.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--report-dir",
|
||||
help="Exact report directory to create. For new analysis this directory must be empty or absent; useful for component-specific reports.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-name",
|
||||
help="Model name from the user prompt or caller context. It is cleaned and stored in analyze-data.json.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timestamp",
|
||||
help="Timestamp for auto-generated report folder names, formatted as YYYYMMDD_HHMMSS. Defaults to local time.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--template-dir",
|
||||
default=str(default_template_dir()),
|
||||
help="Report template directory. Defaults to report_template in this skill.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--analyze-md",
|
||||
help="AI-generated Markdown analysis to copy into the report folder as analyze.md after the final analysis is written.",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def input_source_paths(input_path: Path) -> list[str]:
|
||||
paths: list[Path] = []
|
||||
for pattern in ("layer*.json", "profile*.json"):
|
||||
paths.extend(input_path.glob(pattern))
|
||||
return [str(path) for path in sorted(paths)]
|
||||
|
||||
|
||||
def inferred_analyzer_model_name(input_path: Path, explicit_name: Optional[str]) -> Optional[str]:
|
||||
metadata = infer_model_name_metadata(str(input_path), explicit_name, input_source_paths(input_path))
|
||||
return metadata["name"]
|
||||
|
||||
|
||||
def infer_report_model_name(input_path: Path, explicit_name: Optional[str]) -> str:
|
||||
metadata = infer_model_name_metadata(str(input_path), explicit_name, input_source_paths(input_path))
|
||||
return metadata["name"] or "unknown_model"
|
||||
|
||||
|
||||
def is_relative_to(path: Path, parent: Path) -> bool:
|
||||
try:
|
||||
path.relative_to(parent)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def default_output_parent() -> Path:
|
||||
cwd = Path.cwd().resolve()
|
||||
root = skill_root().resolve()
|
||||
if is_relative_to(cwd, root):
|
||||
return Path(tempfile.gettempdir()).resolve()
|
||||
return cwd
|
||||
|
||||
|
||||
def parse_timestamp(value: Optional[str]) -> str:
|
||||
if value is None:
|
||||
return datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
if not re.fullmatch(r"\d{8}_\d{6}", value):
|
||||
raise ValueError("--timestamp must use YYYYMMDD_HHMMSS format")
|
||||
return value
|
||||
|
||||
|
||||
def has_files(path: Path) -> bool:
|
||||
return path.exists() and any(path.iterdir())
|
||||
|
||||
|
||||
def has_input_files(path: Path) -> bool:
|
||||
return any(path.glob("layers_*.json")) or any(path.glob("profile_*.json"))
|
||||
|
||||
|
||||
def next_available_report_dir(output_parent: Path, model_name: str, timestamp: str) -> Path:
|
||||
base_name = f"report_{model_name}_{timestamp}"
|
||||
candidate = output_parent / base_name
|
||||
if not has_files(candidate):
|
||||
return candidate
|
||||
|
||||
index = 2
|
||||
while True:
|
||||
candidate = output_parent / f"{base_name}_{index}"
|
||||
if not has_files(candidate):
|
||||
return candidate
|
||||
index += 1
|
||||
|
||||
|
||||
def resolve_new_report_dir(args: argparse.Namespace, input_path: Path) -> Path:
|
||||
if args.report_dir:
|
||||
report_dir = Path(args.report_dir).expanduser()
|
||||
if has_files(report_dir):
|
||||
raise ValueError(f"report directory is not empty: {report_dir}")
|
||||
return report_dir
|
||||
|
||||
output_parent = Path(args.output_parent).expanduser() if args.output_parent else default_output_parent()
|
||||
timestamp = parse_timestamp(args.timestamp)
|
||||
model_name = infer_report_model_name(input_path, args.model_name)
|
||||
return next_available_report_dir(output_parent, model_name, timestamp)
|
||||
|
||||
|
||||
def validate_template_dir(template_dir: Path) -> None:
|
||||
required = [
|
||||
"index.html",
|
||||
"app.js",
|
||||
"styles.css",
|
||||
"netron/grapher.css",
|
||||
"netron/index.html",
|
||||
"netron/netron.js",
|
||||
"netron/worker.js",
|
||||
]
|
||||
missing = [name for name in required if not (template_dir / name).is_file()]
|
||||
if missing:
|
||||
raise ValueError(f"report template is missing required file(s): {', '.join(missing)}")
|
||||
|
||||
|
||||
def run_command(command: Sequence[str]) -> int:
|
||||
process = subprocess.run(command, text=True, check=False)
|
||||
return process.returncode
|
||||
|
||||
|
||||
def run_analyzer(input_path: Path, output_path: Path, model_name: Optional[str]) -> int:
|
||||
analyzer = script_dir() / "analyze_trt_perf.py"
|
||||
command = [
|
||||
sys.executable,
|
||||
str(analyzer),
|
||||
str(input_path),
|
||||
"--output",
|
||||
str(output_path),
|
||||
]
|
||||
if model_name:
|
||||
command.extend(["--model-name", model_name])
|
||||
return run_command(command)
|
||||
|
||||
|
||||
def run_validator(analyze_data_path: Path) -> int:
|
||||
validator = script_dir() / "validate_analyze_data.py"
|
||||
return run_command([sys.executable, str(validator), str(analyze_data_path)])
|
||||
|
||||
|
||||
def copy_template(template_dir: Path, report_dir: Path) -> None:
|
||||
validate_template_dir(template_dir)
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for item in template_dir.iterdir():
|
||||
if item.name in {"analyze-data.json", "analyze.md"}:
|
||||
continue
|
||||
|
||||
destination = report_dir / item.name
|
||||
if item.is_dir():
|
||||
shutil.copytree(item, destination, dirs_exist_ok=True)
|
||||
else:
|
||||
shutil.copy2(item, destination)
|
||||
|
||||
|
||||
def copy_analyze_markdown(analyze_md: Optional[str], report_dir: Path) -> Optional[Path]:
|
||||
if not analyze_md:
|
||||
return None
|
||||
|
||||
source = Path(analyze_md).expanduser()
|
||||
if not source.is_file():
|
||||
raise ValueError(f"analyze markdown file not found: {source}")
|
||||
|
||||
destination = report_dir / "analyze.md"
|
||||
if source.resolve() == destination.resolve():
|
||||
return destination
|
||||
|
||||
shutil.copy2(source, destination)
|
||||
return destination
|
||||
|
||||
|
||||
def package_existing_json(args: argparse.Namespace) -> int:
|
||||
analyze_data_path = Path(args.analyze_data).expanduser()
|
||||
if not analyze_data_path.is_file():
|
||||
sys.stderr.write(f"error: analyze-data file not found: {analyze_data_path}\n")
|
||||
return 2
|
||||
if analyze_data_path.name != "analyze-data.json":
|
||||
sys.stderr.write("error: --analyze-data must point to a file named analyze-data.json\n")
|
||||
return 2
|
||||
|
||||
validate_result = run_validator(analyze_data_path)
|
||||
if validate_result != 0:
|
||||
return validate_result
|
||||
|
||||
try:
|
||||
copy_template(Path(args.template_dir).expanduser(), analyze_data_path.parent)
|
||||
analyze_markdown_path = copy_analyze_markdown(args.analyze_md, analyze_data_path.parent)
|
||||
except (OSError, ValueError) as exc:
|
||||
sys.stderr.write(f"error: unable to package report template: {exc}\n")
|
||||
return 2
|
||||
|
||||
print(f"report_dir: {analyze_data_path.parent}")
|
||||
print(f"analyze_data: {analyze_data_path}")
|
||||
if analyze_markdown_path:
|
||||
print(f"analyze_md: {analyze_markdown_path}")
|
||||
return 0
|
||||
|
||||
|
||||
def package_input_folder(args: argparse.Namespace, input_path: Path) -> int:
|
||||
try:
|
||||
report_dir = resolve_new_report_dir(args, input_path)
|
||||
except ValueError as exc:
|
||||
sys.stderr.write(f"error: {exc}\n")
|
||||
return 2
|
||||
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
analyze_data_path = report_dir / "analyze-data.json"
|
||||
|
||||
model_name = inferred_analyzer_model_name(input_path, args.model_name)
|
||||
analyze_result = run_analyzer(input_path, analyze_data_path, model_name)
|
||||
if analyze_result != 0:
|
||||
return analyze_result
|
||||
|
||||
validate_result = run_validator(analyze_data_path)
|
||||
if validate_result != 0:
|
||||
return validate_result
|
||||
|
||||
try:
|
||||
copy_template(Path(args.template_dir).expanduser(), report_dir)
|
||||
analyze_markdown_path = copy_analyze_markdown(args.analyze_md, report_dir)
|
||||
except (OSError, ValueError) as exc:
|
||||
sys.stderr.write(f"error: unable to package report template: {exc}\n")
|
||||
return 2
|
||||
|
||||
print(f"report_dir: {report_dir}")
|
||||
print(f"analyze_data: {analyze_data_path}")
|
||||
if analyze_markdown_path:
|
||||
print(f"analyze_md: {analyze_markdown_path}")
|
||||
return 0
|
||||
|
||||
|
||||
def package_new_analysis(args: argparse.Namespace) -> int:
|
||||
if not args.path:
|
||||
sys.stderr.write("error: provide an input folder, or use --analyze-data\n")
|
||||
return 2
|
||||
|
||||
input_path = Path(args.path).expanduser()
|
||||
if not input_path.is_dir():
|
||||
sys.stderr.write(f"error: input folder not found: {input_path}\n")
|
||||
return 2
|
||||
|
||||
if has_input_files(input_path):
|
||||
return package_input_folder(args, input_path)
|
||||
|
||||
sys.stderr.write(
|
||||
f"error: no layers_*.json or profile_*.json files found directly in {input_path}. "
|
||||
"package_report.py creates exactly one report per invocation; pass a single "
|
||||
"component/report folder instead of a parent folder.\n"
|
||||
)
|
||||
return 2
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
args = parse_args(argv)
|
||||
|
||||
if args.path and args.analyze_data:
|
||||
sys.stderr.write("error: use either an input folder or --analyze-data, not both\n")
|
||||
return 2
|
||||
|
||||
if args.analyze_data:
|
||||
return package_existing_json(args)
|
||||
return package_new_analysis(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,76 @@
|
||||
@echo off
|
||||
:: SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
:: SPDX-License-Identifier: Apache-2.0
|
||||
setlocal
|
||||
|
||||
if "%~1"=="" (
|
||||
echo usage: run.cmd ^<python-script^> [args...] 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
|
||||
set "PY_CMD="
|
||||
set "PY_ARG="
|
||||
|
||||
if not defined SKILL_PYTHON goto after_skill_python
|
||||
call :select_plain "%SKILL_PYTHON%"
|
||||
if defined PY_CMD goto run_python
|
||||
:after_skill_python
|
||||
|
||||
if not defined PYTHON goto after_python_env
|
||||
call :select_plain "%PYTHON%"
|
||||
if defined PY_CMD goto run_python
|
||||
:after_python_env
|
||||
|
||||
if not defined PYTHON3 goto after_python3_env
|
||||
call :select_plain "%PYTHON3%"
|
||||
if defined PY_CMD goto run_python
|
||||
:after_python3_env
|
||||
|
||||
call :select_with_arg py -3
|
||||
if defined PY_CMD goto run_python
|
||||
call :select_plain python
|
||||
if defined PY_CMD goto run_python
|
||||
call :select_plain python3
|
||||
if defined PY_CMD goto run_python
|
||||
call :select_plain python3.12
|
||||
if defined PY_CMD goto run_python
|
||||
call :select_plain python3.11
|
||||
if defined PY_CMD goto run_python
|
||||
call :select_plain python3.10
|
||||
if defined PY_CMD goto run_python
|
||||
call :select_plain python3.9
|
||||
if defined PY_CMD goto run_python
|
||||
call :select_plain python3.8
|
||||
if defined PY_CMD goto run_python
|
||||
|
||||
echo error: unable to find Python 3.8+ on PATH. Set SKILL_PYTHON to a Python executable. 1>&2
|
||||
exit /b 127
|
||||
|
||||
:run_python
|
||||
if "%PY_ARG%"=="" (
|
||||
"%PY_CMD%" %*
|
||||
) else (
|
||||
"%PY_CMD%" "%PY_ARG%" %*
|
||||
)
|
||||
exit /b %ERRORLEVEL%
|
||||
|
||||
:select_plain
|
||||
set "PY_CMD=%~1"
|
||||
set "PY_ARG="
|
||||
if "%PY_CMD%"=="" goto clear_selection
|
||||
"%PY_CMD%" -c "import sys; raise SystemExit(0 if sys.version_info >= (3, 8) else 1)" >nul 2>nul
|
||||
if errorlevel 1 goto clear_selection
|
||||
exit /b 0
|
||||
|
||||
:select_with_arg
|
||||
set "PY_CMD=%~1"
|
||||
set "PY_ARG=%~2"
|
||||
if "%PY_CMD%"=="" goto clear_selection
|
||||
"%PY_CMD%" "%PY_ARG%" -c "import sys; raise SystemExit(0 if sys.version_info >= (3, 8) else 1)" >nul 2>nul
|
||||
if errorlevel 1 goto clear_selection
|
||||
exit /b 0
|
||||
|
||||
:clear_selection
|
||||
set "PY_CMD="
|
||||
set "PY_ARG="
|
||||
exit /b 1
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env sh
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
set -u
|
||||
|
||||
if [ $# -lt 1 ]; then
|
||||
printf '%s\n' "usage: run.sh <python-script> [args...]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
SCRIPT_PATH=$1
|
||||
shift
|
||||
|
||||
try_python() {
|
||||
candidate=$1
|
||||
launcher_arg=$2
|
||||
shift 2
|
||||
|
||||
if [ -z "$candidate" ]; then
|
||||
return 1
|
||||
fi
|
||||
if ! command -v "$candidate" >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -n "$launcher_arg" ]; then
|
||||
"$candidate" "$launcher_arg" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 8) else 1)' >/dev/null 2>&1 || return 1
|
||||
exec "$candidate" "$launcher_arg" "$SCRIPT_PATH" "$@"
|
||||
fi
|
||||
|
||||
"$candidate" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 8) else 1)' >/dev/null 2>&1 || return 1
|
||||
exec "$candidate" "$SCRIPT_PATH" "$@"
|
||||
}
|
||||
|
||||
if [ -n "${SKILL_PYTHON:-}" ]; then
|
||||
try_python "$SKILL_PYTHON" "" "$@"
|
||||
fi
|
||||
if [ -n "${PYTHON:-}" ]; then
|
||||
try_python "$PYTHON" "" "$@"
|
||||
fi
|
||||
if [ -n "${PYTHON3:-}" ]; then
|
||||
try_python "$PYTHON3" "" "$@"
|
||||
fi
|
||||
|
||||
try_python python3 "" "$@"
|
||||
try_python python "" "$@"
|
||||
try_python py -3 "$@"
|
||||
try_python python3.12 "" "$@"
|
||||
try_python python3.11 "" "$@"
|
||||
try_python python3.10 "" "$@"
|
||||
try_python python3.9 "" "$@"
|
||||
try_python python3.8 "" "$@"
|
||||
|
||||
printf '%s\n' "error: unable to find Python 3.8+ on PATH. Set SKILL_PYTHON to a Python executable." >&2
|
||||
exit 127
|
||||
@@ -0,0 +1,4 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Implementation modules for the TensorRT performance analysis scripts."""
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,536 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Process TensorRT layer-info JSON and write visualization targets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
STANDARD_LAYER_KEYS = {
|
||||
"Name",
|
||||
"LayerType",
|
||||
"Inputs",
|
||||
"Constants",
|
||||
"Outputs",
|
||||
"TacticName",
|
||||
"StreamId",
|
||||
"Metadata",
|
||||
"_subgraph",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProcessedInitializer:
|
||||
name: str
|
||||
desc: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProcessedNode:
|
||||
name: str
|
||||
op_type: str
|
||||
inputs: list[str]
|
||||
outputs: list[str]
|
||||
attrs: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProcessedLayerInfo:
|
||||
source_path: Path
|
||||
layers: list[dict[str, Any]]
|
||||
nodes: list[ProcessedNode]
|
||||
graph_inputs: list[str]
|
||||
graph_outputs: list[str]
|
||||
graph_value_info: list[str]
|
||||
value_descriptors: dict[str, dict[str, Any]]
|
||||
initializers: list[ProcessedInitializer]
|
||||
layer_type_counts: Counter[str]
|
||||
node_op_type_counts: Counter[str]
|
||||
extra_layer_fields: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TargetSpec:
|
||||
name: str
|
||||
module_name: str
|
||||
default_suffix: str
|
||||
description: str
|
||||
aliases: tuple[str, ...] = ()
|
||||
|
||||
|
||||
TARGET_SPECS = (
|
||||
TargetSpec(
|
||||
name="html",
|
||||
module_name="trt_perf.layer_info_html",
|
||||
default_suffix=".html",
|
||||
description="Standalone HTML layer graph summary",
|
||||
),
|
||||
)
|
||||
|
||||
TARGETS_BY_NAME: dict[str, TargetSpec] = {}
|
||||
for target_spec in TARGET_SPECS:
|
||||
TARGETS_BY_NAME[target_spec.name] = target_spec
|
||||
for alias in target_spec.aliases:
|
||||
TARGETS_BY_NAME[alias] = target_spec
|
||||
|
||||
|
||||
class NameRegistry:
|
||||
"""Keep target value names valid under SSA while preserving TRT names."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.used: set[str] = set()
|
||||
|
||||
def reserve(self, preferred: Any, fallback: str) -> str:
|
||||
base = str(preferred).strip() if preferred is not None else ""
|
||||
if not base:
|
||||
base = fallback
|
||||
|
||||
candidate = base
|
||||
suffix = 1
|
||||
while candidate in self.used:
|
||||
candidate = f"{base}__trt_dup{suffix}"
|
||||
suffix += 1
|
||||
self.used.add(candidate)
|
||||
return candidate
|
||||
|
||||
|
||||
def compact_json(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def load_layers(path: Path) -> list[dict[str, Any]]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
|
||||
if isinstance(payload, list):
|
||||
layers = payload
|
||||
elif isinstance(payload, dict):
|
||||
for key in ("Layers", "layers", "LayerInfo", "layerInfo"):
|
||||
if isinstance(payload.get(key), list):
|
||||
layers = payload[key]
|
||||
break
|
||||
else:
|
||||
raise ValueError("JSON object does not contain a layer list")
|
||||
else:
|
||||
raise ValueError("top-level JSON value must be a list or object")
|
||||
|
||||
bad_indexes = [idx for idx, layer in enumerate(layers) if not isinstance(layer, dict)]
|
||||
if bad_indexes:
|
||||
preview = ", ".join(str(idx) for idx in bad_indexes[:5])
|
||||
raise ValueError(f"layer entries must be objects; invalid indexes: {preview}")
|
||||
|
||||
return layers
|
||||
|
||||
|
||||
def as_descriptor_list(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, dict)]
|
||||
|
||||
|
||||
def tensor_name(desc: dict[str, Any], fallback: str) -> str:
|
||||
name = desc.get("Name")
|
||||
if name is None or str(name).strip() == "":
|
||||
return fallback
|
||||
return str(name)
|
||||
|
||||
|
||||
def safe_dim_param(text: Any, fallback: str) -> str:
|
||||
value = re.sub(r"[^0-9A-Za-z_]+", "_", str(text)).strip("_")
|
||||
if not value:
|
||||
value = fallback
|
||||
if value[0].isdigit():
|
||||
value = f"d_{value}"
|
||||
return value
|
||||
|
||||
|
||||
def tensor_shape(desc: dict[str, Any], tensor: str) -> list[int | str] | None:
|
||||
dims = desc.get("Dimensions")
|
||||
if dims is None:
|
||||
return None
|
||||
if not isinstance(dims, list):
|
||||
return None
|
||||
|
||||
shape: list[int | str] = []
|
||||
prefix = safe_dim_param(tensor, "tensor")
|
||||
for idx, dim in enumerate(dims):
|
||||
if isinstance(dim, bool):
|
||||
shape.append(int(dim))
|
||||
elif isinstance(dim, int) and dim >= 0:
|
||||
shape.append(dim)
|
||||
else:
|
||||
shape.append(f"{prefix}_dim{idx}")
|
||||
return shape
|
||||
|
||||
|
||||
def sanitize_op_type(layer_type: Any) -> str:
|
||||
text = str(layer_type).strip() if layer_type is not None else ""
|
||||
op_type = re.sub(r"[^0-9A-Za-z_]+", "_", text).strip("_")
|
||||
if not op_type:
|
||||
op_type = "Layer"
|
||||
if not re.match(r"[A-Za-z_]", op_type[0]):
|
||||
op_type = f"_{op_type}"
|
||||
return op_type[:200]
|
||||
|
||||
|
||||
KGEN_TOKEN_MAP = {
|
||||
"and": "And",
|
||||
"add": "Add",
|
||||
"cast": "Cast",
|
||||
"div": "Div",
|
||||
"erf": "Erf",
|
||||
"gath": "Gather",
|
||||
"gemm": "Gemm",
|
||||
"mean": "Mean",
|
||||
"mha": "MHA",
|
||||
"move": "Move",
|
||||
"mul": "Mul",
|
||||
"resh": "Reshape",
|
||||
"sele": "Select",
|
||||
"slic": "Slice",
|
||||
"sqrt": "Sqrt",
|
||||
"sub": "Sub",
|
||||
"tanh": "Tanh",
|
||||
"tran": "Transpose",
|
||||
}
|
||||
|
||||
|
||||
def strip_kgen_signature_noise(text: str) -> str:
|
||||
text = re.sub(r"_0x[0-9A-Fa-f]+$", "", text)
|
||||
text = re.sub(r"_myl\d+(?:_\d+)?$", "", text)
|
||||
text = re.sub(r"^__?myl_", "", text)
|
||||
return text.strip("_")
|
||||
|
||||
|
||||
def split_kgen_signature(signature: str) -> list[str]:
|
||||
if not signature:
|
||||
return []
|
||||
if "_" in signature:
|
||||
return [part for part in signature.split("_") if part]
|
||||
return re.findall(r"[A-Z][a-z0-9]*|[a-z0-9]+", signature)
|
||||
|
||||
|
||||
def normalize_kgen_token(token: str) -> str:
|
||||
lowered = token.lower()
|
||||
if lowered in KGEN_TOKEN_MAP:
|
||||
return KGEN_TOKEN_MAP[lowered]
|
||||
if lowered.startswith("v") and lowered[1:].isdigit():
|
||||
return lowered.upper()
|
||||
return token[:1].upper() + token[1:]
|
||||
|
||||
|
||||
def kgen_details(layer: dict[str, Any]) -> tuple[str, list[str], str]:
|
||||
candidates = [str(layer.get("TacticName", "")), layer_name(layer, 0)]
|
||||
for candidate in candidates:
|
||||
signature = strip_kgen_signature_noise(candidate)
|
||||
tokens = [normalize_kgen_token(token) for token in split_kgen_signature(signature)]
|
||||
if tokens:
|
||||
return signature, tokens, f"KGEN_{'_'.join(tokens)}"
|
||||
return "", [], "kgen"
|
||||
|
||||
|
||||
def node_op_type(layer: dict[str, Any]) -> tuple[str, dict[str, Any]]:
|
||||
raw_layer_type = layer.get("LayerType", "Layer")
|
||||
if str(raw_layer_type).lower() != "kgen":
|
||||
return sanitize_op_type(raw_layer_type), {}
|
||||
|
||||
signature, tokens, op_type = kgen_details(layer)
|
||||
attrs: dict[str, Any] = {}
|
||||
if signature:
|
||||
attrs["trt_kgen_signature"] = signature
|
||||
if tokens:
|
||||
attrs["trt_kgen_ops_json"] = compact_json(tokens)
|
||||
attrs["trt_kgen_op_count"] = len(tokens)
|
||||
return sanitize_op_type(op_type), attrs
|
||||
|
||||
|
||||
def layer_name(layer: dict[str, Any], index: int) -> str:
|
||||
raw = layer.get("Name")
|
||||
if raw is None or str(raw).strip() == "":
|
||||
return f"trt_layer_{index}"
|
||||
return str(raw)
|
||||
|
||||
|
||||
def int_attr(value: Any, default: int = 0) -> int:
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
try:
|
||||
return int(str(value))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def process_layers(layers: list[dict[str, Any]], source_path: Path) -> ProcessedLayerInfo:
|
||||
registry = NameRegistry()
|
||||
external_by_original: dict[str, str] = {}
|
||||
latest_by_original: dict[str, str] = {}
|
||||
descriptors_by_value: dict[str, dict[str, Any]] = {}
|
||||
initializers_by_original: dict[str, str] = {}
|
||||
initializers: list[ProcessedInitializer] = []
|
||||
produced_values: list[str] = []
|
||||
consumed_values: set[str] = set()
|
||||
nodes: list[ProcessedNode] = []
|
||||
|
||||
def map_external(desc: dict[str, Any], fallback: str) -> str:
|
||||
original = tensor_name(desc, fallback)
|
||||
if original not in external_by_original:
|
||||
value_name = registry.reserve(original, fallback)
|
||||
external_by_original[original] = value_name
|
||||
descriptors_by_value[value_name] = desc
|
||||
return external_by_original[original]
|
||||
|
||||
def map_input(desc: dict[str, Any], fallback: str) -> str:
|
||||
original = tensor_name(desc, fallback)
|
||||
value_name = latest_by_original.get(original)
|
||||
if value_name is None:
|
||||
value_name = map_external(desc, fallback)
|
||||
descriptors_by_value.setdefault(value_name, desc)
|
||||
consumed_values.add(value_name)
|
||||
return value_name
|
||||
|
||||
def map_constant(desc: dict[str, Any], fallback: str) -> str:
|
||||
original = tensor_name(desc, fallback)
|
||||
if original not in initializers_by_original:
|
||||
value_name = registry.reserve(original, fallback)
|
||||
initializers_by_original[original] = value_name
|
||||
descriptors_by_value[value_name] = desc
|
||||
initializers.append(ProcessedInitializer(value_name, desc))
|
||||
value_name = initializers_by_original[original]
|
||||
descriptors_by_value.setdefault(value_name, desc)
|
||||
consumed_values.add(value_name)
|
||||
return value_name
|
||||
|
||||
def map_output(desc: dict[str, Any], fallback: str) -> str:
|
||||
original = tensor_name(desc, fallback)
|
||||
value_name = registry.reserve(original, fallback)
|
||||
latest_by_original[original] = value_name
|
||||
descriptors_by_value[value_name] = desc
|
||||
produced_values.append(value_name)
|
||||
return value_name
|
||||
|
||||
layer_type_counts: Counter[str] = Counter()
|
||||
node_op_type_counts: Counter[str] = Counter()
|
||||
unmapped_layer_fields: set[str] = set()
|
||||
|
||||
for index, layer in enumerate(layers):
|
||||
inputs = as_descriptor_list(layer.get("Inputs"))
|
||||
constants = as_descriptor_list(layer.get("Constants"))
|
||||
outputs = as_descriptor_list(layer.get("Outputs"))
|
||||
|
||||
node_inputs = [
|
||||
map_input(desc, f"trt_layer_{index}_input_{pos}")
|
||||
for pos, desc in enumerate(inputs)
|
||||
]
|
||||
node_inputs.extend(
|
||||
map_constant(desc, f"trt_layer_{index}_constant_{pos}")
|
||||
for pos, desc in enumerate(constants)
|
||||
)
|
||||
node_outputs = [
|
||||
map_output(desc, f"trt_layer_{index}_output_{pos}")
|
||||
for pos, desc in enumerate(outputs)
|
||||
]
|
||||
|
||||
raw_layer_type = layer.get("LayerType", "Layer")
|
||||
layer_type_counts[str(raw_layer_type)] += 1
|
||||
op_type, op_type_attrs = node_op_type(layer)
|
||||
node_op_type_counts[op_type] += 1
|
||||
extra = {key: value for key, value in layer.items() if key not in STANDARD_LAYER_KEYS}
|
||||
unmapped_layer_fields.update(extra)
|
||||
|
||||
attrs: dict[str, Any] = {
|
||||
"trt_layer_index": index,
|
||||
"trt_layer_type": str(raw_layer_type),
|
||||
"trt_input_count": len(inputs),
|
||||
"trt_constant_count": len(constants),
|
||||
"trt_output_count": len(outputs),
|
||||
"trt_inputs_json": compact_json(inputs),
|
||||
"trt_constants_json": compact_json(constants),
|
||||
"trt_outputs_json": compact_json(outputs),
|
||||
}
|
||||
attrs.update(op_type_attrs)
|
||||
|
||||
if "TacticName" in layer:
|
||||
attrs["trt_tactic_name"] = str(layer.get("TacticName", ""))
|
||||
if "StreamId" in layer:
|
||||
attrs["trt_stream_id"] = int_attr(layer.get("StreamId"))
|
||||
if "_subgraph" in layer:
|
||||
attrs["trt_subgraph"] = int_attr(layer.get("_subgraph"))
|
||||
if "Metadata" in layer:
|
||||
attrs["trt_metadata"] = str(layer.get("Metadata", ""))
|
||||
if extra:
|
||||
attrs["trt_extra_json"] = compact_json(extra)
|
||||
|
||||
nodes.append(
|
||||
ProcessedNode(
|
||||
name=layer_name(layer, index),
|
||||
op_type=op_type,
|
||||
inputs=node_inputs,
|
||||
outputs=node_outputs,
|
||||
attrs=attrs,
|
||||
)
|
||||
)
|
||||
|
||||
terminal_values = [value_name for value_name in produced_values if value_name not in consumed_values]
|
||||
if not terminal_values and produced_values:
|
||||
terminal_values = [produced_values[-1]]
|
||||
|
||||
graph_output_names = set(terminal_values)
|
||||
graph_input_names = set(external_by_original.values())
|
||||
graph_value_info = [
|
||||
value_name
|
||||
for value_name in descriptors_by_value
|
||||
if value_name not in graph_input_names and value_name not in graph_output_names
|
||||
]
|
||||
|
||||
return ProcessedLayerInfo(
|
||||
source_path=source_path,
|
||||
layers=layers,
|
||||
nodes=nodes,
|
||||
graph_inputs=list(external_by_original.values()),
|
||||
graph_outputs=terminal_values,
|
||||
graph_value_info=graph_value_info,
|
||||
value_descriptors=descriptors_by_value,
|
||||
initializers=initializers,
|
||||
layer_type_counts=layer_type_counts,
|
||||
node_op_type_counts=node_op_type_counts,
|
||||
extra_layer_fields=sorted(unmapped_layer_fields),
|
||||
)
|
||||
|
||||
|
||||
def load_and_process_layer_info(path: Path) -> ProcessedLayerInfo:
|
||||
return process_layers(load_layers(path), path)
|
||||
|
||||
|
||||
def resolve_target_spec(target: str) -> TargetSpec:
|
||||
try:
|
||||
return TARGETS_BY_NAME[target]
|
||||
except KeyError as exc:
|
||||
valid = ", ".join(sorted(TARGETS_BY_NAME))
|
||||
raise ValueError(f"unknown target {target!r}; valid targets: {valid}") from exc
|
||||
|
||||
|
||||
def default_output_path(input_path: Path, target: str = "html") -> Path:
|
||||
spec = resolve_target_spec(target)
|
||||
return input_path.with_name(f"{input_path.stem}{spec.default_suffix}")
|
||||
|
||||
|
||||
def target_names_help() -> str:
|
||||
return ", ".join(spec.name for spec in TARGET_SPECS)
|
||||
|
||||
|
||||
def load_target_module(spec: TargetSpec) -> Any:
|
||||
return importlib.import_module(spec.module_name)
|
||||
|
||||
|
||||
def write_target(processed: ProcessedLayerInfo, spec: TargetSpec, output_path: Path) -> None:
|
||||
module = load_target_module(spec)
|
||||
writer = getattr(module, "write", None)
|
||||
if writer is None:
|
||||
raise NotImplementedError(
|
||||
f"target {spec.name!r} is not implemented yet; "
|
||||
f"expected write(processed, output_path) in {spec.module_name}"
|
||||
)
|
||||
writer(processed, output_path)
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Process TensorRT layer-info JSON and write one or more visualization targets."
|
||||
)
|
||||
parser.add_argument("input", type=Path, nargs="?", help="Path to TensorRT layer-info JSON")
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
"--target",
|
||||
action="append",
|
||||
dest="targets",
|
||||
metavar="TARGET",
|
||||
help=f"Output target; repeat for multiple outputs. Default: html. Available: {target_names_help()}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
type=Path,
|
||||
help="Output path. Only valid when writing one target.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
help="Directory for target-specific default filenames.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list-targets",
|
||||
action="store_true",
|
||||
help="List supported target names and exit.",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def list_targets() -> None:
|
||||
seen: set[str] = set()
|
||||
for spec in TARGET_SPECS:
|
||||
aliases = f" (aliases: {', '.join(spec.aliases)})" if spec.aliases else ""
|
||||
print(f"{spec.name}{aliases}: {spec.description}")
|
||||
seen.add(spec.name)
|
||||
|
||||
alias_only = sorted(name for name, spec in TARGETS_BY_NAME.items() if spec.name not in seen)
|
||||
if alias_only:
|
||||
print(f"aliases: {', '.join(alias_only)}")
|
||||
|
||||
|
||||
def resolve_requested_targets(targets: list[str] | None) -> list[TargetSpec]:
|
||||
requested = targets or ["html"]
|
||||
specs: list[TargetSpec] = []
|
||||
seen: set[str] = set()
|
||||
for target in requested:
|
||||
spec = resolve_target_spec(target)
|
||||
if spec.name in seen:
|
||||
continue
|
||||
specs.append(spec)
|
||||
seen.add(spec.name)
|
||||
return specs
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(sys.argv[1:] if argv is None else argv)
|
||||
if args.list_targets:
|
||||
list_targets()
|
||||
return 0
|
||||
if args.input is None:
|
||||
print("error: input is required unless --list-targets is used", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
target_specs = resolve_requested_targets(args.targets)
|
||||
if args.output is not None and len(target_specs) != 1:
|
||||
raise ValueError("--output can only be used when writing one target")
|
||||
|
||||
processed = load_and_process_layer_info(args.input)
|
||||
written: list[Path] = []
|
||||
for spec in target_specs:
|
||||
if args.output is not None:
|
||||
output_path = args.output
|
||||
elif args.output_dir is not None:
|
||||
output_path = args.output_dir / f"{args.input.stem}{spec.default_suffix}"
|
||||
else:
|
||||
output_path = default_output_path(args.input, spec.name)
|
||||
|
||||
write_target(processed, spec, output_path)
|
||||
written.append(output_path)
|
||||
except Exception as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
for output_path in written:
|
||||
print(f"wrote {output_path}")
|
||||
return 0
|
||||
@@ -0,0 +1,220 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Write a standalone HTML visualization target for TensorRT layer-info data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def escape(value: Any) -> str:
|
||||
return html.escape(str(value), quote=True)
|
||||
|
||||
|
||||
def compact_json(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def short(value: Any, limit: int = 180) -> str:
|
||||
text = str(value)
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return f"{text[: limit - 1]}..."
|
||||
|
||||
|
||||
def table_rows(items: list[tuple[str, Any]]) -> str:
|
||||
rows = []
|
||||
for key, value in items:
|
||||
rows.append(
|
||||
"<tr>"
|
||||
f"<th>{escape(key)}</th>"
|
||||
f"<td><code>{escape(value)}</code></td>"
|
||||
"</tr>"
|
||||
)
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def write(processed: Any, output_path: Path) -> None:
|
||||
layer_rows = []
|
||||
for node in processed.nodes:
|
||||
attrs = node.attrs
|
||||
layer_rows.append(
|
||||
"<tr>"
|
||||
f"<td>{escape(attrs.get('trt_layer_index', ''))}</td>"
|
||||
f"<td><code>{escape(short(node.name, 140))}</code></td>"
|
||||
f"<td>{escape(attrs.get('trt_layer_type', node.op_type))}</td>"
|
||||
f"<td>{escape(node.op_type)}</td>"
|
||||
f"<td>{escape(len(node.inputs))}</td>"
|
||||
f"<td>{escape(len(node.outputs))}</td>"
|
||||
f"<td><code>{escape(short(attrs.get('trt_tactic_name', ''), 160))}</code></td>"
|
||||
"</tr>"
|
||||
)
|
||||
|
||||
summary_rows = table_rows(
|
||||
[
|
||||
("source", processed.source_path),
|
||||
("layers", len(processed.layers)),
|
||||
("nodes", len(processed.nodes)),
|
||||
("graph inputs", len(processed.graph_inputs)),
|
||||
("graph outputs", len(processed.graph_outputs)),
|
||||
("initializers", len(processed.initializers)),
|
||||
("layer types", compact_json(dict(processed.layer_type_counts))),
|
||||
("node op types", compact_json(dict(processed.node_op_type_counts))),
|
||||
]
|
||||
)
|
||||
|
||||
graph_inputs = "\n".join(f"<li><code>{escape(name)}</code></li>" for name in processed.graph_inputs)
|
||||
graph_outputs = "\n".join(f"<li><code>{escape(name)}</code></li>" for name in processed.graph_outputs)
|
||||
|
||||
document = f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>TensorRT Layer Info: {escape(processed.source_path.name)}</title>
|
||||
<style>
|
||||
:root {{
|
||||
color-scheme: light dark;
|
||||
--bg: #f8f9fb;
|
||||
--fg: #17202a;
|
||||
--muted: #586476;
|
||||
--line: #d7dce5;
|
||||
--panel: #ffffff;
|
||||
--accent: #0f766e;
|
||||
--code: #0b4a64;
|
||||
}}
|
||||
@media (prefers-color-scheme: dark) {{
|
||||
:root {{
|
||||
--bg: #111417;
|
||||
--fg: #edf1f5;
|
||||
--muted: #aab4c0;
|
||||
--line: #303942;
|
||||
--panel: #181d22;
|
||||
--accent: #5eead4;
|
||||
--code: #93c5fd;
|
||||
}}
|
||||
}}
|
||||
* {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--fg);
|
||||
font: 14px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
}}
|
||||
main {{
|
||||
width: min(1400px, calc(100vw - 32px));
|
||||
margin: 0 auto;
|
||||
padding: 28px 0 48px;
|
||||
}}
|
||||
h1 {{
|
||||
margin: 0 0 6px;
|
||||
font-size: 24px;
|
||||
letter-spacing: 0;
|
||||
}}
|
||||
h2 {{
|
||||
margin: 28px 0 10px;
|
||||
font-size: 17px;
|
||||
letter-spacing: 0;
|
||||
}}
|
||||
p {{ margin: 0 0 16px; color: var(--muted); }}
|
||||
code {{
|
||||
color: var(--code);
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
word-break: break-word;
|
||||
}}
|
||||
table {{
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--line);
|
||||
}}
|
||||
th, td {{
|
||||
border-bottom: 1px solid var(--line);
|
||||
padding: 8px 10px;
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}}
|
||||
th {{
|
||||
color: var(--muted);
|
||||
font-weight: 600;
|
||||
white-space: nowrap;
|
||||
}}
|
||||
thead th {{
|
||||
position: sticky;
|
||||
top: 0;
|
||||
background: var(--panel);
|
||||
z-index: 1;
|
||||
}}
|
||||
tbody tr:last-child td, tbody tr:last-child th {{ border-bottom: 0; }}
|
||||
.summary th {{ width: 160px; }}
|
||||
.split {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
}}
|
||||
.panel {{
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--line);
|
||||
padding: 12px 14px;
|
||||
}}
|
||||
ul {{
|
||||
margin: 0;
|
||||
padding-left: 18px;
|
||||
}}
|
||||
@media (max-width: 780px) {{
|
||||
main {{ width: min(100vw - 20px, 1400px); padding-top: 18px; }}
|
||||
.split {{ grid-template-columns: 1fr; }}
|
||||
th, td {{ padding: 7px 8px; }}
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>TensorRT Layer Info</h1>
|
||||
<p>{escape(processed.source_path)}</p>
|
||||
|
||||
<h2>Summary</h2>
|
||||
<table class="summary"><tbody>
|
||||
{summary_rows}
|
||||
</tbody></table>
|
||||
|
||||
<h2>Graph Endpoints</h2>
|
||||
<div class="split">
|
||||
<section class="panel">
|
||||
<h2>Inputs</h2>
|
||||
<ul>{graph_inputs}</ul>
|
||||
</section>
|
||||
<section class="panel">
|
||||
<h2>Outputs</h2>
|
||||
<ul>{graph_outputs}</ul>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<h2>Layers</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th>Name</th>
|
||||
<th>Layer Type</th>
|
||||
<th>Op Type</th>
|
||||
<th>Inputs</th>
|
||||
<th>Outputs</th>
|
||||
<th>Tactic</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{chr(10).join(layer_rows)}
|
||||
</tbody>
|
||||
</table>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(document, encoding="utf-8")
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Serialize extracted TensorRT performance data as JSON."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
def json_ready(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {str(key): json_ready(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [json_ready(item) for item in value]
|
||||
if isinstance(value, (str, int, float, bool)) or value is None:
|
||||
return value
|
||||
return str(value)
|
||||
|
||||
|
||||
def render_results_json(data: Dict[str, Any], indent: int = 2) -> str:
|
||||
return json.dumps(json_ready(data), indent=indent, sort_keys=True) + "\n"
|
||||
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Validate analyzer JSON output against schemas/analyze-data.schema.json.
|
||||
|
||||
This intentionally implements only the JSON Schema keywords used by the local
|
||||
schema so it can run without third-party dependencies in benchmark environments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Any, List, Optional, Sequence
|
||||
|
||||
|
||||
class ValidationError(Exception):
|
||||
"""Raised for schema loading or schema reference errors."""
|
||||
|
||||
|
||||
def default_schema_path() -> str:
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
return os.path.abspath(os.path.join(script_dir, "..", "schemas", "analyze-data.schema.json"))
|
||||
|
||||
|
||||
def load_json(path: str) -> Any:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
except FileNotFoundError as exc:
|
||||
raise ValidationError(f"file not found: {path}") from exc
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValidationError(f"invalid JSON in {path}: line {exc.lineno}, column {exc.colno}: {exc.msg}") from exc
|
||||
except OSError as exc:
|
||||
raise ValidationError(f"unable to read {path}: {exc}") from exc
|
||||
|
||||
|
||||
def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate TensorRT analysis JSON generated by analyze_trt_perf.py."
|
||||
)
|
||||
parser.add_argument("input", help="Path to generated analysis JSON.")
|
||||
parser.add_argument(
|
||||
"--schema",
|
||||
default=default_schema_path(),
|
||||
help="Schema path. Defaults to schemas/analyze-data.schema.json.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-errors",
|
||||
type=int,
|
||||
default=40,
|
||||
help="Maximum validation errors to print. Defaults to 40.",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def decode_ref_token(token: str) -> str:
|
||||
return token.replace("~1", "/").replace("~0", "~")
|
||||
|
||||
|
||||
def resolve_ref(ref: str, root_schema: Any) -> Any:
|
||||
if not ref.startswith("#/"):
|
||||
raise ValidationError(f"only local JSON Schema refs are supported: {ref}")
|
||||
|
||||
current = root_schema
|
||||
for token in ref[2:].split("/"):
|
||||
key = decode_ref_token(token)
|
||||
if not isinstance(current, dict) or key not in current:
|
||||
raise ValidationError(f"unresolved schema ref: {ref}")
|
||||
current = current[key]
|
||||
return current
|
||||
|
||||
|
||||
def type_label(instance: Any) -> str:
|
||||
if instance is None:
|
||||
return "null"
|
||||
if isinstance(instance, bool):
|
||||
return "boolean"
|
||||
if isinstance(instance, int):
|
||||
return "integer"
|
||||
if isinstance(instance, float):
|
||||
return "number"
|
||||
if isinstance(instance, str):
|
||||
return "string"
|
||||
if isinstance(instance, list):
|
||||
return "array"
|
||||
if isinstance(instance, dict):
|
||||
return "object"
|
||||
return type(instance).__name__
|
||||
|
||||
|
||||
def matches_type(instance: Any, expected: str) -> bool:
|
||||
if expected == "null":
|
||||
return instance is None
|
||||
if expected == "boolean":
|
||||
return isinstance(instance, bool)
|
||||
if expected == "integer":
|
||||
return isinstance(instance, int) and not isinstance(instance, bool)
|
||||
if expected == "number":
|
||||
return isinstance(instance, (int, float)) and not isinstance(instance, bool)
|
||||
if expected == "string":
|
||||
return isinstance(instance, str)
|
||||
if expected == "array":
|
||||
return isinstance(instance, list)
|
||||
if expected == "object":
|
||||
return isinstance(instance, dict)
|
||||
raise ValidationError(f"unsupported schema type: {expected}")
|
||||
|
||||
|
||||
def matches_any_type(instance: Any, expected: Any) -> bool:
|
||||
if isinstance(expected, str):
|
||||
return matches_type(instance, expected)
|
||||
if isinstance(expected, list):
|
||||
return any(matches_type(instance, item) for item in expected)
|
||||
raise ValidationError(f"invalid schema type declaration: {expected!r}")
|
||||
|
||||
|
||||
def append_error(errors: List[str], path: str, message: str) -> None:
|
||||
errors.append(f"{path}: {message}")
|
||||
|
||||
|
||||
def append_nearest_errors(errors: List[str], matches: List[List[str]], limit: int = 5) -> None:
|
||||
if not matches:
|
||||
return
|
||||
nearest = min(matches, key=lambda item: (sum("expected constant" in error for error in item), len(item)))
|
||||
for error in nearest[:limit]:
|
||||
errors.append(f" nearest schema: {error}")
|
||||
|
||||
|
||||
def validate(instance: Any, schema: Any, path: str, root_schema: Any) -> List[str]:
|
||||
if not isinstance(schema, dict):
|
||||
raise ValidationError(f"{path}: schema node is not an object")
|
||||
|
||||
if "$ref" in schema:
|
||||
return validate(instance, resolve_ref(str(schema["$ref"]), root_schema), path, root_schema)
|
||||
|
||||
errors: List[str] = []
|
||||
|
||||
if "allOf" in schema:
|
||||
for subschema in schema["allOf"]:
|
||||
errors.extend(validate(instance, subschema, path, root_schema))
|
||||
|
||||
if "anyOf" in schema:
|
||||
matches = [validate(instance, subschema, path, root_schema) for subschema in schema["anyOf"]]
|
||||
if not any(not match_errors for match_errors in matches):
|
||||
append_error(errors, path, "does not match any allowed schema")
|
||||
append_nearest_errors(errors, matches)
|
||||
|
||||
if "oneOf" in schema:
|
||||
matches = [validate(instance, subschema, path, root_schema) for subschema in schema["oneOf"]]
|
||||
match_count = sum(1 for match_errors in matches if not match_errors)
|
||||
if match_count != 1:
|
||||
append_error(errors, path, f"matches {match_count} schemas, expected exactly one")
|
||||
if match_count == 0:
|
||||
append_nearest_errors(errors, matches)
|
||||
|
||||
if "const" in schema and instance != schema["const"]:
|
||||
append_error(errors, path, f"expected constant {schema['const']!r}, got {instance!r}")
|
||||
|
||||
if "enum" in schema and instance not in schema["enum"]:
|
||||
append_error(errors, path, f"expected one of {schema['enum']!r}, got {instance!r}")
|
||||
|
||||
expected_type = schema.get("type")
|
||||
if expected_type is not None and not matches_any_type(instance, expected_type):
|
||||
append_error(errors, path, f"expected type {expected_type!r}, got {type_label(instance)}")
|
||||
return errors
|
||||
|
||||
if isinstance(instance, dict):
|
||||
validate_object(instance, schema, path, root_schema, errors)
|
||||
elif isinstance(instance, list):
|
||||
validate_array(instance, schema, path, root_schema, errors)
|
||||
elif isinstance(instance, (int, float)) and not isinstance(instance, bool):
|
||||
validate_number(instance, schema, path, errors)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate_object(instance: dict, schema: dict, path: str, root_schema: Any, errors: List[str]) -> None:
|
||||
properties = schema.get("properties", {})
|
||||
required = schema.get("required", [])
|
||||
additional = schema.get("additionalProperties", True)
|
||||
|
||||
for key in required:
|
||||
if key not in instance:
|
||||
append_error(errors, path, f"missing required property {key!r}")
|
||||
|
||||
for key, value in instance.items():
|
||||
child_path = f"{path}.{key}"
|
||||
if key in properties:
|
||||
errors.extend(validate(value, properties[key], child_path, root_schema))
|
||||
elif additional is False:
|
||||
append_error(errors, child_path, "additional property is not allowed")
|
||||
elif isinstance(additional, dict):
|
||||
errors.extend(validate(value, additional, child_path, root_schema))
|
||||
|
||||
|
||||
def validate_array(instance: list, schema: dict, path: str, root_schema: Any, errors: List[str]) -> None:
|
||||
min_items = schema.get("minItems")
|
||||
if isinstance(min_items, int) and len(instance) < min_items:
|
||||
append_error(errors, path, f"expected at least {min_items} items, got {len(instance)}")
|
||||
|
||||
max_items = schema.get("maxItems")
|
||||
if isinstance(max_items, int) and len(instance) > max_items:
|
||||
append_error(errors, path, f"expected at most {max_items} items, got {len(instance)}")
|
||||
|
||||
items_schema = schema.get("items")
|
||||
if isinstance(items_schema, dict):
|
||||
for index, value in enumerate(instance):
|
||||
errors.extend(validate(value, items_schema, f"{path}[{index}]", root_schema))
|
||||
|
||||
|
||||
def validate_number(instance: float, schema: dict, path: str, errors: List[str]) -> None:
|
||||
if "minimum" in schema and instance < schema["minimum"]:
|
||||
append_error(errors, path, f"expected >= {schema['minimum']}, got {instance}")
|
||||
if "maximum" in schema and instance > schema["maximum"]:
|
||||
append_error(errors, path, f"expected <= {schema['maximum']}, got {instance}")
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
args = parse_args(argv)
|
||||
|
||||
try:
|
||||
data = load_json(args.input)
|
||||
schema = load_json(args.schema)
|
||||
errors = validate(data, schema, "$", schema)
|
||||
except ValidationError as exc:
|
||||
sys.stderr.write(f"error: {exc}\n")
|
||||
return 2
|
||||
|
||||
if errors:
|
||||
sys.stderr.write(f"{args.input}: FAIL ({len(errors)} schema error(s))\n")
|
||||
for error in errors[: args.max_errors]:
|
||||
sys.stderr.write(f"- {error}\n")
|
||||
remaining = len(errors) - args.max_errors
|
||||
if remaining > 0:
|
||||
sys.stderr.write(f"... {remaining} more error(s) omitted\n")
|
||||
return 1
|
||||
|
||||
sys.stdout.write(f"{args.input}: PASS\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Standalone TensorRT layer-info HTML visualizer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from trt_perf.layer_info import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,418 @@
|
||||
---
|
||||
name: trt-strong-typing-migration
|
||||
description: >-
|
||||
Migrate a TensorRT build from weak typing (deprecated 10.12, removed 11.0) to
|
||||
strong typing — across Python INetworkDefinition builders, the trtexec CLI, and
|
||||
C++ builder code. Use when a TRT 11 upgrade breaks a weakly-typed build. Triggers: weakly typed to strongly typed,
|
||||
kSTRONGLY_TYPED, weak typing deprecated, kFP16/kINT8 removed, setPrecision rejected,
|
||||
setComputePrecision deprecated, do I still need --stronglyTyped, how to add the
|
||||
kSTRONGLY_TYPED flag, ModelOpt autocast, INT8 on TRT 11. NOT for ONNX import
|
||||
(`trt-onnx-quickstart`), Torch-TRT (`trt-torch-quickstart`), or C++ deploy
|
||||
(`trt-cpp-runtime-quickstart`).
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: NVIDIA Corporation
|
||||
version: "1.0"
|
||||
tags:
|
||||
- migration
|
||||
- strong-typing
|
||||
- autocast
|
||||
- modelopt
|
||||
- builder
|
||||
---
|
||||
|
||||
# Migrate Weakly-Typed to Strongly-Typed TensorRT Build
|
||||
|
||||
Strong typing is the recommended build mode for TensorRT 11.x; weak typing was
|
||||
deprecated in 10.12 and is scheduled for removal. This skill migrates a weakly-typed
|
||||
build flow to a strongly-typed equivalent, one path per entry point (Python network,
|
||||
`trtexec`, C++ builder), plus the ModelOpt AutoCast pre-step needed when the source
|
||||
is an ONNX file that must run in mixed precision.
|
||||
|
||||
**Version behavior.** On **TensorRT 11.x strong typing is unconditional** — every
|
||||
network is strongly typed, the `kSTRONGLY_TYPED` creation flag is accepted but
|
||||
ignored, and `trtexec --stronglyTyped` is a no-op (default-on). On **10.12–10.x**
|
||||
you opt in explicitly via that flag / `--stronglyTyped`. Setting it is therefore
|
||||
always safe: required on 10.x, harmless on 11.x. Either way the migration is the
|
||||
same — remove the precision *hints* (gone in 11.x) and move precision into the graph.
|
||||
|
||||
## When to Use
|
||||
|
||||
| Scenario | Use this skill? |
|
||||
|----------|-----------------|
|
||||
| Existing Python build calls `config.set_flag(trt.BuilderFlag.FP16 / INT8 / TF32)` | Yes |
|
||||
| Existing C++ build calls `config->setFlag(BuilderFlag::kFP16 / kINT8 / kTF32)` | Yes |
|
||||
| Existing `trtexec` line uses `--fp16`, `--int8`, `--best`, `--bf16` and the developer wants strong typing | Yes |
|
||||
| Build emits "weak typing is deprecated, use STRONGLY_TYPED" warning at runtime | Yes |
|
||||
| Build fails with "flag X is incompatible with strongly typed networks" | Yes |
|
||||
| Developer has a FP32 ONNX file and wants a strongly-typed mixed FP16 engine | Yes — start at Step 1 (AutoCast) |
|
||||
| Developer is importing a brand-new ONNX file with no existing build flow | No — use `trt-onnx-quickstart` first; come back here to switch the flag |
|
||||
| Developer's engine produces wrong outputs *after* migration | No — use `trt-quantization-accuracy` |
|
||||
| Developer is on TensorRT 10.11 or earlier | No — strong typing is fully available from 10.12 onward; upgrade before migrating |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **TensorRT 10.12 or later (11.x recommended).** Check:
|
||||
```bash
|
||||
python3 -c "import tensorrt; print(tensorrt.__version__)"
|
||||
trtexec --help 2>&1 | grep -i stronglyTyped
|
||||
```
|
||||
The version check is the determinant — strong typing is available from 10.12
|
||||
on. The `trtexec` probe just confirms the CLI flag for the trtexec path; a
|
||||
missing `--stronglyTyped` means a pre-10.12 install.
|
||||
|
||||
2. **ModelOpt installed when the source is an ONNX file requiring mixed precision.**
|
||||
```bash
|
||||
pip install nvidia-modelopt onnx onnxruntime
|
||||
```
|
||||
If the ONNX file is already strongly typed (contains explicit `Cast` nodes
|
||||
and FP16 initializers), AutoCast is not needed — skip directly to Step 2.
|
||||
|
||||
3. **A working baseline.** Capture the weakly-typed engine's outputs on a known
|
||||
input set before migrating. Strong typing is stricter, and any silent precision
|
||||
substitutions weak typing was making will become observable.
|
||||
|
||||
4. **The verifier script.** `scripts/verify.sh` runs `migrate.py` against a
|
||||
committed sample and asserts the rewrite is well-formed. Run it before editing
|
||||
anything to confirm the helper works.
|
||||
|
||||
## Migration paths
|
||||
|
||||
There are three migration paths — Python builder, `trtexec`, C++ builder.
|
||||
**Infer the path from the query when signals are clear; ask only if genuinely
|
||||
uncertain.** Match the strongest signal:
|
||||
|
||||
| Signal in the request | Path | Entry point |
|
||||
|------------------------|------|-------------|
|
||||
| Python tokens: `trt.BuilderFlag`, `set_flag(`, `create_network(`, `.py` | **A** | Python `INetworkDefinition` builder |
|
||||
| Standalone `trtexec --fp16` / `--int8` / `--best` / `--bf16` | **B** | `trtexec` CLI |
|
||||
| C++ tokens: `BuilderFlag::`, `createNetworkV2`, `->setFlag`, `.cpp` / `.h` | **C** | C++ builder |
|
||||
| Multiple signals | — | do each, most-specific first |
|
||||
| No usable signal (generic "how do I migrate") | — | ask which entry point, or proceed with Python and say you assumed it |
|
||||
|
||||
The migration preserves existing precisions (kFP16 → FP16 cast, kINT8 → Q/DQ) and is
|
||||
identical whether the network came from a parser or was hand-built — only Step 1
|
||||
(AutoCast) is ONNX-specific, so source and precision goal rarely need a question.
|
||||
|
||||
Inferring the path is a low-stakes default — it does **not** extend to destructive
|
||||
operations, which always need confirmation. `migrate.py` defaults to dry-run
|
||||
(prints a diff); the `--write` flag is the human-in-the-loop confirmation. Treat
|
||||
any other destructive step (overwriting a `.plan`, replacing an ONNX) the same way:
|
||||
show, confirm, then act.
|
||||
|
||||
For a strongly-typed INT8 model straight from an ONNX file (not a migration of
|
||||
existing builder code), jump to Step 1's ModelOpt quantization recipe.
|
||||
|
||||
## Step 1 — (ONNX source only) AutoCast the model to mixed precision
|
||||
|
||||
Skip if the source is not ONNX, or if the ONNX file is already strongly typed.
|
||||
|
||||
A FP32 ONNX file built for a weakly-typed `--fp16` flow carries no precision
|
||||
information — TensorRT was free to pick FP16 or FP32 per layer. Strong typing
|
||||
requires the graph itself to declare precision via `Cast` operators and typed
|
||||
initializers. ModelOpt's AutoCast inserts these.
|
||||
|
||||
Use the ModelOpt CLI — it's the supported entry point and takes `.npz` calibration
|
||||
directly. Use `--calibration_data` for both tools (autocast's canonical flag;
|
||||
quantization's abbreviation of `--calibration_data_path`):
|
||||
|
||||
```bash
|
||||
# FP16 / BF16 mixed-precision AutoCast
|
||||
python -m modelopt.onnx.autocast \
|
||||
--onnx_path model.fp32.onnx \
|
||||
--output_path model.mixed.onnx \
|
||||
--low_precision_type fp16 \
|
||||
--calibration_data calib.npz
|
||||
|
||||
# INT8 / FP8 strongly-typed quantization (produces ONNX with explicit Q/DQ)
|
||||
python -m modelopt.onnx.quantization \
|
||||
--onnx_path model.fp32.onnx \
|
||||
--output_path model.int8.onnx \
|
||||
--quantize_mode int8 \
|
||||
--high_precision_dtype fp16 \
|
||||
--calibration_data calib.npz
|
||||
```
|
||||
|
||||
Keep accuracy-sensitive ops in FP32 via AutoCast's node/op-type exclusion lists
|
||||
(e.g. exclude `MatMul`/`LayerNormalization` on transformer-heavy networks), and keep
|
||||
FP32 graph I/O when downstream stages expect it. See the
|
||||
[Model Optimizer docs](https://nvidia.github.io/Model-Optimizer) for the exact
|
||||
module and flags for your ModelOpt version.
|
||||
|
||||
Verify the converted model in ONNX Runtime before passing it to TRT:
|
||||
|
||||
```python
|
||||
import onnxruntime, numpy as np
|
||||
sess = onnxruntime.InferenceSession("model.mixed.onnx", providers=["CPUExecutionProvider"])
|
||||
# Compare a few outputs against the FP32 model with np.allclose(..., rtol=5e-3, atol=5e-3).
|
||||
```
|
||||
|
||||
If parity fails here, **stop** — the conversion is wrong and no amount of TRT
|
||||
engine work will recover it. Re-run with tighter exclusion lists.
|
||||
|
||||
**Next step — build the strongly-typed plan from the converted ONNX:**
|
||||
|
||||
```bash
|
||||
# For the INT8 case (modelopt.onnx.quantization output)
|
||||
trtexec --onnx=model.int8.onnx --stronglyTyped --saveEngine=model.int8.plan
|
||||
|
||||
# For the mixed-precision case (modelopt.onnx.autocast output)
|
||||
trtexec --onnx=model.mixed.onnx --stronglyTyped --saveEngine=model.fp16.plan
|
||||
```
|
||||
|
||||
**Critical**: do NOT pass `--int8` or `--fp16` alongside `--stronglyTyped`. Those flags are removed in TRT 11, and on TRT 10.12+ they conflict with strong typing. The precision is already encoded in the ONNX (via explicit Cast / Q / DQ nodes); the builder honors it.
|
||||
|
||||
The reference recipe for the end-to-end INT8 flow is `samples/python/strongly_type_autocast/sample.py` in `tensorrt-oss`.
|
||||
|
||||
## Step 2A — Migrate a Python `INetworkDefinition` builder
|
||||
|
||||
Mechanical replacements:
|
||||
|
||||
| Weakly typed | Strongly typed |
|
||||
|--------------|----------------|
|
||||
| `builder.create_network(0)` or `builder.create_network(1 << int(NetworkDefinitionCreationFlag.EXPLICIT_BATCH))` | `builder.create_network(1 << int(NetworkDefinitionCreationFlag.STRONGLY_TYPED))` |
|
||||
| `config.set_flag(trt.BuilderFlag.FP16)` | **Remove.** Types come from the network. |
|
||||
| `config.set_flag(trt.BuilderFlag.BF16)` | **Remove.** |
|
||||
| `config.set_flag(trt.BuilderFlag.INT8)` | **Remove.** Quantization expressed via Q/DQ nodes in the graph. |
|
||||
| `config.set_flag(trt.BuilderFlag.TF32)` | **Keep.** `kTF32` is retained in 11.x — it allows (not requires) TF32 for FP32 tensors and is orthogonal to typing. |
|
||||
| `layer.precision = trt.float16` | **Remove.** Cast the input tensor instead (`network.add_cast`). |
|
||||
| `layer.set_output_type(0, trt.float16)` | **Remove.** Output type follows the network; for cast/quantize/dequantize/fill layers use `set_to_type` instead. |
|
||||
| `builder.platform_has_fast_int8 / has_fast_fp16` checks gating the above | **Remove the gating logic** along with the flag. |
|
||||
|
||||
Worked rewrite (the MNIST builder from `samples/python/network_api_pytorch_mnist/sample.py`
|
||||
already uses the strongly-typed form — note `NetworkDefinitionCreationFlag.STRONGLY_TYPED`
|
||||
and the absence of any `BuilderFlag.FP16` call):
|
||||
|
||||
```python
|
||||
builder = trt.Builder(TRT_LOGGER)
|
||||
network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED))
|
||||
config = builder.create_builder_config()
|
||||
config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, common.GiB(1))
|
||||
# ... populate network ...
|
||||
plan = builder.build_serialized_network(network, config)
|
||||
```
|
||||
|
||||
If the network is built via the ONNX parser, the parser respects strong typing
|
||||
automatically — no code change is needed beyond the flag.
|
||||
|
||||
Where a removed `precision` / `set_output_type` hint actually mattered, encode it
|
||||
in the graph: cast a tensor's dtype with `add_cast`, or insert an explicit
|
||||
Quantize/Dequantize pair for INT8 (the scale is a build-time constant; ModelOpt
|
||||
derives it from calibration, or supply it by hand):
|
||||
|
||||
```python
|
||||
# kFP16 hint -> explicit cast on the tensor:
|
||||
cast = network.add_cast(some_tensor, trt.float16)
|
||||
|
||||
# kINT8 hint -> explicit Q/DQ pair (per-tensor scalar scale). NOTE: trt.Weights does
|
||||
# not copy the numpy buffer, so q_scale must stay alive until the engine is built.
|
||||
q_scale = np.array(1.0 / 127.0, dtype=np.float32).reshape(()) # rank-0 scalar
|
||||
scale = network.add_constant(shape=(), weights=trt.Weights(q_scale)).get_output(0)
|
||||
q = network.add_quantize(some_tensor, scale, trt.int8).get_output(0)
|
||||
dq = network.add_dequantize(q, scale, trt.float32).get_output(0)
|
||||
# Feed dq into the consumer; TensorRT fuses the Q/DQ into it so it runs in INT8.
|
||||
```
|
||||
|
||||
Drive the rewrite from `scripts/migrate.py`:
|
||||
|
||||
```bash
|
||||
python3 scripts/migrate.py path/to/build.py # dry-run diff
|
||||
python3 scripts/migrate.py path/to/build.py --write # rewrite in place
|
||||
```
|
||||
|
||||
The script edits AST nodes (not regexes) — it understands `create_network` and
|
||||
`set_flag` call shapes, so it is safe on files that mix conventions.
|
||||
|
||||
## Step 2B — Migrate a `trtexec` command line
|
||||
|
||||
Replacements:
|
||||
|
||||
| Weakly typed flag | Strongly typed equivalent |
|
||||
|-------------------|---------------------------|
|
||||
| `--fp16` | **Remove.** Precision comes from the ONNX. See the `--stronglyTyped` note below. |
|
||||
| `--bf16` | **Remove.** Same as `--fp16`. |
|
||||
| `--int8` | **Remove.** Q/DQ must be present in the ONNX. |
|
||||
| `--best` | **Remove.** Not allowed with `--stronglyTyped` on 10.12–10.x (see Common Errors). |
|
||||
| `--stronglyTyped` | **Version-gated.** Required on 10.12–10.x to opt into strong typing; a no-op (default-on) on 11.x — drop it or leave it, your choice. |
|
||||
| `--noTF32` | **Keep.** `kTF32` is retained in 11.x; `--noTF32` still disables TF32 and is orthogonal to typing. |
|
||||
| `--precisionConstraints=obey` / `prefer` | **Remove.** Constraints are expressed in the graph. |
|
||||
| `--layerPrecisions=...` | **Remove.** Use ONNX `Cast` nodes / AutoCast. |
|
||||
| `--layerOutputTypes=...` | **Remove.** |
|
||||
|
||||
Worked rewrite:
|
||||
|
||||
```bash
|
||||
# Before (weakly typed)
|
||||
trtexec --onnx=model.onnx --fp16 --saveEngine=model.fp16.plan \
|
||||
--minShapes=input:1x3x224x224 --optShapes=input:8x3x224x224 \
|
||||
--maxShapes=input:16x3x224x224
|
||||
|
||||
# After (strongly typed; ONNX has been AutoCast'd to mixed FP16)
|
||||
# On 11.x --stronglyTyped is the default and a no-op — the line below is equivalent
|
||||
# with or without it. On 10.12–10.x it is REQUIRED, or the build stays weakly typed.
|
||||
trtexec --onnx=model.mixed.onnx --stronglyTyped --saveEngine=model.fp16.plan \
|
||||
--minShapes=input:1x3x224x224 --optShapes=input:8x3x224x224 \
|
||||
--maxShapes=input:16x3x224x224
|
||||
```
|
||||
|
||||
The `trtexec` README's [Example 6](https://github.com/NVIDIA/TensorRT/blob/main/samples/trtexec/README.md#example-6-create-a-strongly-typed-plan-file)
|
||||
is the canonical reference.
|
||||
|
||||
## Step 2C — Migrate a C++ builder
|
||||
|
||||
Replacements mirror the Python path:
|
||||
|
||||
| Weakly typed (C++) | Strongly typed (C++) |
|
||||
|---|---|
|
||||
| `builder->createNetworkV2(0)` | `builder->createNetworkV2(1U << static_cast<uint32_t>(NetworkDefinitionCreationFlag::kSTRONGLY_TYPED))` |
|
||||
| `config->setFlag(BuilderFlag::kFP16)` | **Remove.** |
|
||||
| `config->setFlag(BuilderFlag::kBF16)` | **Remove.** |
|
||||
| `config->setFlag(BuilderFlag::kINT8)` | **Remove.** |
|
||||
| `config->setFlag(BuilderFlag::kTF32)` | **Keep.** `kTF32` is retained in 11.x — orthogonal to typing. |
|
||||
| `config->setFlag(BuilderFlag::kOBEY_PRECISION_CONSTRAINTS / kPREFER_PRECISION_CONSTRAINTS)` | **Remove.** No per-layer precision exists to constrain. |
|
||||
| `layer->setPrecision(DataType::kHALF)` | **Remove.** Cast the input tensor instead (`addCast`). |
|
||||
| `layer->setComputePrecision(...)` | **Remove.** Deprecated 10.16, removed; compute precision follows input dtypes. |
|
||||
| `layer->setOutputType(0, DataType::kHALF)` | **Remove.** For cast/quantize/dequantize/fill layers use `setToType()` instead. |
|
||||
|
||||
Worked rewrite:
|
||||
|
||||
```cpp
|
||||
// Before (10.x weakly typed) — the builder picks precision from these hints:
|
||||
auto network = std::unique_ptr<INetworkDefinition>(
|
||||
builder->createNetworkV2(1U << static_cast<uint32_t>(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH)));
|
||||
auto config = std::unique_ptr<IBuilderConfig>(builder->createBuilderConfig());
|
||||
config->setFlag(BuilderFlag::kFP16); // hint: builder may use FP16
|
||||
config->setFlag(BuilderFlag::kINT8); // hint: builder may use INT8 (+ a calibrator)
|
||||
config->setFlag(BuilderFlag::kOBEY_PRECISION_CONSTRAINTS); // force the per-layer overrides below
|
||||
someLayer->setPrecision(DataType::kHALF); // per-layer override
|
||||
someLayer->setOutputType(0, DataType::kHALF); // per-layer override
|
||||
```
|
||||
|
||||
Each hint above maps to a concrete graph edit — there is no longer a "tell the
|
||||
builder to prefer X" knob, so state the dtype directly:
|
||||
|
||||
```cpp
|
||||
// After (11.x strongly typed) — precision lives in the graph; every line above is gone.
|
||||
auto network = std::unique_ptr<INetworkDefinition>(
|
||||
builder->createNetworkV2(1U << static_cast<uint32_t>(NetworkDefinitionCreationFlag::kSTRONGLY_TYPED)));
|
||||
// kSTRONGLY_TYPED: required on 10.12–10.x, accepted-but-ignored on 11.x (0 works too).
|
||||
auto config = std::unique_ptr<IBuilderConfig>(builder->createBuilderConfig());
|
||||
// No precision setFlag, no setPrecision / setOutputType.
|
||||
|
||||
// kFP16 hint -> cast the tensor's dtype explicitly:
|
||||
auto* cast = network->addCast(*someTensor, DataType::kHALF);
|
||||
// feed cast->getOutput(0) into the consumer layer
|
||||
|
||||
// kINT8 hint -> insert an explicit Quantize/Dequantize pair. The scale is a
|
||||
// build-time constant (per-tensor scalar here); ModelOpt derives it from calibration,
|
||||
// or supply it by hand. NOTE: TensorRT does not copy Weights, so qScale must stay
|
||||
// alive until the engine is built.
|
||||
float qScale = 1.0f / 127.0f;
|
||||
auto* scale = network->addConstant(Dims{0, {}},
|
||||
Weights{DataType::kFLOAT, &qScale, 1})->getOutput(0);
|
||||
auto* q = network->addQuantize(*someTensor, *scale, DataType::kINT8)->getOutput(0);
|
||||
auto* dq = network->addDequantize(*q, *scale, DataType::kFLOAT)->getOutput(0);
|
||||
// feed dq into the consumer; TensorRT fuses the Q/DQ into it so it runs in INT8.
|
||||
|
||||
// kOBEY_PRECISION_CONSTRAINTS + setPrecision/setOutputType -> nothing to port:
|
||||
// the dtype is now fixed by the cast / Q-DQ above, so there is no constraint to obey.
|
||||
```
|
||||
|
||||
`setToType()` is the only surviving per-layer type control, and is valid **only**
|
||||
on `ICastLayer`, `IDequantizeLayer`, `IDynamicQuantizeLayer`, `IFillLayer`, and
|
||||
`IQuantizeLayer` — not on arbitrary layers.
|
||||
|
||||
If the build pipeline parses ONNX via `nvonnxparser::IParser`, no parser change is
|
||||
required — the parser honors the strong typing flag set on the network.
|
||||
|
||||
## Step 3 — Rebuild and validate
|
||||
|
||||
1. Rebuild the engine with the migrated flow.
|
||||
2. Run the captured baseline inputs through the new engine.
|
||||
3. Compare against the weakly-typed baseline:
|
||||
- **Outputs match within `atol=5e-3, rtol=5e-3`**: migration is complete.
|
||||
- **Outputs diverge but stay within model accuracy tolerance**: weak typing
|
||||
was opportunistically using FP16/INT8 where the new graph keeps FP32 (or vice
|
||||
versa). Acceptable if the downstream metric (top-1, BLEU, WER) is unchanged;
|
||||
confirm with the original accuracy harness.
|
||||
- **Outputs diverge beyond tolerance**: see Common Errors below.
|
||||
|
||||
If the source was ONNX, also run the AutoCast'd ONNX through ONNX Runtime and
|
||||
compare against the FP32 ONNX before blaming TRT — graph-level conversion errors
|
||||
should be caught there.
|
||||
|
||||
## Migration Patterns Reference
|
||||
|
||||
| Pattern | Action |
|
||||
|---------|--------|
|
||||
| Network created with no flags (legacy implicit batch) | First migrate to explicit batch; only then add `STRONGLY_TYPED` |
|
||||
| Multiple `set_flag` calls (FP16 + INT8 + TF32 + REFIT) | Keep `REFIT`, `SPARSE_WEIGHTS`, `kTF32`; drop the precision-hint flags (FP16/BF16/INT8/FP8) and the removed `kOBEY`/`kPREFER_PRECISION_CONSTRAINTS` |
|
||||
| `set_calibration_profile` / `IInt8Calibrator` | **Remove.** PTQ calibration is replaced by Q/DQ in the ONNX. Use ModelOpt's PTQ for the model first. |
|
||||
| Mixed FP16/FP32 per-layer overrides via `layer.precision` | Express the same intent with `Cast` ops in the ONNX, or in the Python network construction code |
|
||||
| Plugin layers with explicit precision | Plugins must implement `IPluginV3OneBuild::getOutputDataTypes`; the network reads types from the plugin |
|
||||
| Engine consumed by downstream framework expecting FP32 I/O | Set `keep_io_types=True` in AutoCast, or add explicit `Cast` to FP32 at the network outputs |
|
||||
|
||||
## Common Errors
|
||||
|
||||
- **`"BuilderFlag::kFP16 is not compatible with a strongly typed network"`** —
|
||||
a `setFlag`/`set_flag` call survived the migration. Re-run `migrate.py` or grep
|
||||
for `BuilderFlag\.` / `BuilderFlag::` in the migrated source.
|
||||
- **`"--best is incompatible with --stronglyTyped"`** — drop `--best` from the
|
||||
`trtexec` line; precision choices live in the graph now.
|
||||
- **`"Network has no marked outputs"` immediately after switching the flag** —
|
||||
some builders only call `mark_output` on the path conditioned by
|
||||
`platform_has_fast_fp16`. Remove the conditional.
|
||||
- **AutoCast'd ONNX fails the parser with "If subgraph type mismatch"** — AutoCast
|
||||
can corrupt certain `If` node subgraphs. Add the affected `If` node names to
|
||||
`nodes_to_exclude` so they stay FP32.
|
||||
- **Engine output is FP32 zeros after migration** — `keep_io_types=False` was used
|
||||
but the host code still feeds FP32 buffers. Either set `keep_io_types=True` or
|
||||
cast the host input before binding.
|
||||
- **Accuracy regression of ~1-3% after migration** — known case for
|
||||
transformer-heavy networks: leftover `Cast(to=FLOAT32)` from AutoCast combined
|
||||
with FP16 weights causes thrash. Work around by excluding the residual cast op
|
||||
types in `op_types_to_exclude`.
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **Do not partially migrate.** Half-migrated builds (some `set_flag` calls
|
||||
removed, `STRONGLY_TYPED` not set) silently fall back to weak typing and the
|
||||
warning is easy to miss. Check the engine build log for `"Strongly typed network
|
||||
detected"` to confirm the flag took effect.
|
||||
- **Do not migrate without a baseline.** Weak typing's per-layer precision choices
|
||||
are non-deterministic across TRT versions; without the old outputs you have no
|
||||
ground truth and cannot tell whether the new engine is correct.
|
||||
- **Do not run `--stronglyTyped` against a FP32 ONNX expecting FP16 speedup.**
|
||||
Strong typing builds exactly what the graph specifies — a FP32 ONNX produces a
|
||||
FP32 engine. Run AutoCast first.
|
||||
- **Plugins must declare types.** Old plugins that left output types as "follow
|
||||
input" will fail to build under strong typing. Update them to the
|
||||
`IPluginV3OneBuild` interface, the only plugin API that supports strong typing
|
||||
in 11.x.
|
||||
- **Refit + strong typing**: refittable strongly-typed engines work, but the refit
|
||||
weights must match the precision declared in the network. A FP16 layer cannot be
|
||||
refit with FP32 weights — convert host weights before calling `setNamedWeights`.
|
||||
- **Don't trust grep alone for completeness.** `migrate.py` uses AST matching, so
|
||||
it catches the precision flags regardless of how `tensorrt` is imported or
|
||||
aliased via `as` — something string-level grep misses. It matches direct
|
||||
attribute access (`trt.BuilderFlag.FP16`); dynamic forms such as
|
||||
`getattr(trt.BuilderFlag, 'FP16')` are *not* rewritten and need manual review.
|
||||
Run the AST helper as the source of truth.
|
||||
|
||||
## References
|
||||
|
||||
- **Migration guide:** [TensorRT 10.x → 11.x](https://docs.nvidia.com/deeplearning/tensorrt/latest/api/migration/tensorrt-10x-to-11x.html)
|
||||
— the authoritative description of the weak → strong typing change.
|
||||
- **Operator type rules:** the [TensorRT Operators Reference](https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/operators/)
|
||||
— each operator's page lists its supported input/output types, i.e. how its output
|
||||
dtype follows its inputs in a strongly-typed graph.
|
||||
- **API reference:** the TensorRT C++/Python API docs for `NetworkDefinitionCreationFlag`,
|
||||
`BuilderFlag`, `ILayer::setToType`, and `INetworkDefinition::addCast` / `addQuantize` /
|
||||
`addDequantize`.
|
||||
- **TensorRT Model Optimizer:** <https://nvidia.github.io/Model-Optimizer> — AutoCast and
|
||||
quantization tooling for getting precision into the model.
|
||||
- **Sample code** (TensorRT OSS, <https://github.com/NVIDIA/TensorRT>):
|
||||
[`samples/trtexec/README.md` Example 6](https://github.com/NVIDIA/TensorRT/blob/main/samples/trtexec/README.md#example-6-create-a-strongly-typed-plan-file)
|
||||
(strongly-typed plan), `samples/python/strongly_type_autocast/sample.py`
|
||||
(AutoCast → strongly-typed INT8), and `samples/python/network_api_pytorch_mnist/sample.py`
|
||||
(strongly-typed Python builder).
|
||||
@@ -0,0 +1,65 @@
|
||||
# trt-strong-typing-migration helper scripts
|
||||
|
||||
Scripts that automate the mechanical parts of the SKILL.md walkthrough.
|
||||
|
||||
## `migrate.py`
|
||||
|
||||
AST-based rewriter for Python TensorRT builder code. Detects the
|
||||
weakly-typed builder pattern (`create_network` + `set_flag(BuilderFlag.FP16/...)`)
|
||||
and rewrites it to the strongly-typed form.
|
||||
|
||||
```bash
|
||||
# Show what would change (dry-run): prints a unified diff; exits 1 if changes are pending, 0 if nothing to do.
|
||||
python3 migrate.py path/to/build.py
|
||||
|
||||
# Rewrite the file in place:
|
||||
python3 migrate.py path/to/build.py --write
|
||||
|
||||
# Process every .py file in a directory tree:
|
||||
python3 migrate.py path/to/project/ --write
|
||||
```
|
||||
|
||||
The script uses `ast.NodeTransformer` so it understands call shapes, not just
|
||||
text — it correctly handles `set_flag` accessed via aliased imports, multi-flag
|
||||
construction in `create_network`, and per-layer `precision`/`set_output_type`
|
||||
assignments. Unrelated logic and docstrings are left intact, but note: because
|
||||
the rewrite round-trips through `ast.unparse`, **regular `#` comments and the
|
||||
original formatting are not preserved**. Review the dry-run diff before
|
||||
`--write`, and re-add any comments you need afterward.
|
||||
|
||||
### What it changes
|
||||
|
||||
| Before | After |
|
||||
|--------|-------|
|
||||
| `builder.create_network(0)` | `builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED))` |
|
||||
| `builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))` | `builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED))` |
|
||||
| `config.set_flag(trt.BuilderFlag.FP16 / BF16 / INT8 / FP8)` | (line removed) |
|
||||
| `layer.precision = trt.float16` | (statement removed) |
|
||||
| `layer.set_output_type(0, trt.float16)` | (statement removed) |
|
||||
|
||||
### What it does NOT change
|
||||
|
||||
- `set_flag` calls for non-precision-hint flags (`REFIT`, `SPARSE_WEIGHTS`,
|
||||
`DISABLE_TIMING_CACHE`, and `TF32`). These are orthogonal to typing — `kTF32`
|
||||
is kept in TRT 11.
|
||||
- Calibration setup (`set_calibration_profile`, `IInt8Calibrator`). These must
|
||||
be removed manually because the correct replacement (Q/DQ in the ONNX) lives
|
||||
outside the build script.
|
||||
- Conditional logic gating on `platform_has_fast_fp16` / `platform_has_fast_int8`
|
||||
— the body of those branches becomes a no-op after the flag is removed, but
|
||||
the conditional remains. Clean up manually if desired.
|
||||
|
||||
## `verify.sh`
|
||||
|
||||
End-to-end check that `migrate.py` is doing the right thing. Writes a
|
||||
representative weakly-typed sample to a temp directory, runs `migrate.py
|
||||
--write` on it, then asserts the rewritten file matches the strongly-typed
|
||||
contract (flag present, precision flags absent, valid Python).
|
||||
|
||||
```bash
|
||||
bash verify.sh # run and clean up
|
||||
bash verify.sh --keep # keep the temp workspace for inspection
|
||||
```
|
||||
|
||||
Exits 0 on success, 1 on any assertion failure. Run this before applying
|
||||
`migrate.py` to a real codebase.
|
||||
@@ -0,0 +1,305 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""AST-based migrator for weakly-typed TensorRT Python builders.
|
||||
|
||||
Rewrites:
|
||||
|
||||
- `builder.create_network(...)` -> uses NetworkDefinitionCreationFlag.STRONGLY_TYPED
|
||||
- `config.set_flag(trt.BuilderFlag.{FP16,BF16,INT8,FP8})` -> removed
|
||||
- `layer.precision = ...` and `layer.set_output_type(...)` -> removed
|
||||
|
||||
The rewrites are intentionally conservative: ambiguous patterns are skipped and
|
||||
reported so the user can resolve them manually. Non-precision flags (REFIT,
|
||||
SPARSE_WEIGHTS, TF32, etc.) are preserved — kTF32 is kept in TRT 11 and is
|
||||
orthogonal to typing.
|
||||
|
||||
Usage:
|
||||
python3 migrate.py path/to/file.py # dry-run, print unified diff
|
||||
python3 migrate.py path/to/file.py --write # rewrite in place
|
||||
python3 migrate.py path/to/dir/ --write # recurse over .py files
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import difflib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
# Precision-hint BuilderFlag attribute names that must be removed. NOTE: TF32 is
|
||||
# deliberately NOT here — kTF32 is kept in TRT 11 (orthogonal to typing), like REFIT.
|
||||
PRECISION_FLAGS = frozenset({"FP16", "BF16", "INT8", "FP8"})
|
||||
|
||||
# NetworkDefinitionCreationFlag attribute names that map to STRONGLY_TYPED.
|
||||
WEAK_NETWORK_FLAGS = frozenset({"EXPLICIT_BATCH"})
|
||||
|
||||
|
||||
def _is_attr(node: ast.AST, *, attr_chain: tuple[str, ...]) -> bool:
|
||||
"""Return True if `node` is an attribute access matching the trailing chain.
|
||||
|
||||
Walks `node.attr -> node.value.attr -> ...` and compares to attr_chain
|
||||
in reverse. The leading element (e.g. module / object name) is not checked,
|
||||
so this matches `trt.BuilderFlag.FP16`, `tensorrt.BuilderFlag.FP16`, or any
|
||||
aliased import.
|
||||
"""
|
||||
cur = node
|
||||
for expected in reversed(attr_chain):
|
||||
if not isinstance(cur, ast.Attribute):
|
||||
return False
|
||||
if cur.attr != expected:
|
||||
return False
|
||||
cur = cur.value
|
||||
return True
|
||||
|
||||
|
||||
def _builder_flag_name(node: ast.AST) -> str | None:
|
||||
"""If `node` is `<anything>.BuilderFlag.<NAME>`, return NAME."""
|
||||
if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Attribute):
|
||||
if node.value.attr == "BuilderFlag":
|
||||
return node.attr
|
||||
return None
|
||||
|
||||
|
||||
def _network_flag_name(node: ast.AST) -> str | None:
|
||||
"""If `node` is `<anything>.NetworkDefinitionCreationFlag.<NAME>`, return NAME."""
|
||||
if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Attribute):
|
||||
if node.value.attr == "NetworkDefinitionCreationFlag":
|
||||
return node.attr
|
||||
return None
|
||||
|
||||
|
||||
def _strongly_typed_arg() -> ast.expr:
|
||||
"""Build the AST for `1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED)`.
|
||||
|
||||
Uses bare `trt.` qualifier — matches the canonical import alias. Callers
|
||||
whose code uses `tensorrt.` directly will get a working but visually
|
||||
inconsistent line; that is acceptable as a one-line manual cleanup.
|
||||
"""
|
||||
return ast.BinOp(
|
||||
left=ast.Constant(value=1),
|
||||
op=ast.LShift(),
|
||||
right=ast.Call(
|
||||
func=ast.Name(id="int", ctx=ast.Load()),
|
||||
args=[
|
||||
ast.Attribute(
|
||||
value=ast.Attribute(
|
||||
value=ast.Name(id="trt", ctx=ast.Load()),
|
||||
attr="NetworkDefinitionCreationFlag",
|
||||
ctx=ast.Load(),
|
||||
),
|
||||
attr="STRONGLY_TYPED",
|
||||
ctx=ast.Load(),
|
||||
)
|
||||
],
|
||||
keywords=[],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class Migrator(ast.NodeTransformer):
|
||||
def __init__(self) -> None:
|
||||
self.removed_flag_calls = 0
|
||||
self.removed_precision_assigns = 0
|
||||
self.removed_set_output_type = 0
|
||||
self.rewrote_create_network = 0
|
||||
self.skipped: list[str] = []
|
||||
|
||||
# ---- create_network rewrites ----------------------------------------
|
||||
|
||||
def visit_Call(self, node: ast.Call) -> ast.AST:
|
||||
self.generic_visit(node)
|
||||
func = node.func
|
||||
if isinstance(func, ast.Attribute) and func.attr == "create_network":
|
||||
# Replace the single positional argument with the STRONGLY_TYPED flag.
|
||||
if len(node.args) == 0:
|
||||
# No positional arg: the flag may be passed as `flags=` keyword.
|
||||
# Blindly appending a positional arg would collide with it
|
||||
# (TypeError: got multiple values for argument 'flags').
|
||||
flags_kw = next((kw for kw in node.keywords if kw.arg == "flags"), None)
|
||||
if flags_kw is not None:
|
||||
if self._is_weak_network_arg(flags_kw.value):
|
||||
flags_kw.value = _strongly_typed_arg()
|
||||
self.rewrote_create_network += 1
|
||||
elif self._already_strongly_typed(flags_kw.value):
|
||||
pass # idempotent
|
||||
else:
|
||||
self.skipped.append(
|
||||
f"create_network at line {node.lineno}: unrecognized flags= "
|
||||
"argument shape, review manually"
|
||||
)
|
||||
elif not node.keywords:
|
||||
# Truly empty call create_network(): defaults to weak typing.
|
||||
node.args = [_strongly_typed_arg()]
|
||||
self.rewrote_create_network += 1
|
||||
else:
|
||||
self.skipped.append(
|
||||
f"create_network at line {node.lineno}: keyword args present "
|
||||
"without flags=, review manually"
|
||||
)
|
||||
elif len(node.args) == 1:
|
||||
arg = node.args[0]
|
||||
if self._is_weak_network_arg(arg):
|
||||
node.args[0] = _strongly_typed_arg()
|
||||
self.rewrote_create_network += 1
|
||||
elif self._already_strongly_typed(arg):
|
||||
pass # idempotent
|
||||
else:
|
||||
self.skipped.append(
|
||||
f"create_network at line {node.lineno}: unrecognized argument shape, "
|
||||
"review manually"
|
||||
)
|
||||
return node
|
||||
|
||||
@staticmethod
|
||||
def _is_weak_network_arg(arg: ast.expr) -> bool:
|
||||
# Match `0`
|
||||
if isinstance(arg, ast.Constant) and arg.value == 0:
|
||||
return True
|
||||
# Match `1 << int(...EXPLICIT_BATCH)` (any weak-network flag name)
|
||||
if isinstance(arg, ast.BinOp) and isinstance(arg.op, ast.LShift):
|
||||
inner = arg.right
|
||||
if isinstance(inner, ast.Call) and isinstance(inner.func, ast.Name) and inner.func.id == "int":
|
||||
if inner.args:
|
||||
name = _network_flag_name(inner.args[0])
|
||||
if name in WEAK_NETWORK_FLAGS:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _already_strongly_typed(arg: ast.expr) -> bool:
|
||||
if isinstance(arg, ast.BinOp) and isinstance(arg.op, ast.LShift):
|
||||
inner = arg.right
|
||||
if isinstance(inner, ast.Call) and isinstance(inner.func, ast.Name) and inner.func.id == "int":
|
||||
if inner.args and _network_flag_name(inner.args[0]) == "STRONGLY_TYPED":
|
||||
return True
|
||||
return False
|
||||
|
||||
# ---- set_flag / precision assignment removals -----------------------
|
||||
|
||||
def visit_Expr(self, node: ast.Expr) -> ast.AST | None:
|
||||
# `config.set_flag(trt.BuilderFlag.FP16)` as an Expr statement.
|
||||
self.generic_visit(node)
|
||||
if isinstance(node.value, ast.Call):
|
||||
call = node.value
|
||||
if isinstance(call.func, ast.Attribute) and call.func.attr == "set_flag":
|
||||
if call.args:
|
||||
flag = _builder_flag_name(call.args[0])
|
||||
if flag in PRECISION_FLAGS:
|
||||
self.removed_flag_calls += 1
|
||||
return None
|
||||
# `layer.set_output_type(0, trt.float16)` — drop unconditionally.
|
||||
if isinstance(call.func, ast.Attribute) and call.func.attr == "set_output_type":
|
||||
self.removed_set_output_type += 1
|
||||
return None
|
||||
return node
|
||||
|
||||
def visit_Assign(self, node: ast.Assign) -> ast.AST | None:
|
||||
# `something.precision = trt.float16` (or any value) — drop.
|
||||
self.generic_visit(node)
|
||||
if len(node.targets) == 1:
|
||||
tgt = node.targets[0]
|
||||
if isinstance(tgt, ast.Attribute) and tgt.attr == "precision":
|
||||
self.removed_precision_assigns += 1
|
||||
return None
|
||||
return node
|
||||
|
||||
# ---- if-statement cleanup ------------------------------------------
|
||||
|
||||
def visit_If(self, node: ast.If) -> ast.AST | None:
|
||||
"""Drop `if builder.platform_has_fast_{fp16,int8,bf16}:` blocks that become empty."""
|
||||
self.generic_visit(node)
|
||||
test = node.test
|
||||
gating_attrs = {"platform_has_fast_fp16", "platform_has_fast_int8",
|
||||
"platform_has_fast_bf16", "platform_has_fast_fp8"}
|
||||
if isinstance(test, ast.Attribute) and test.attr in gating_attrs:
|
||||
# If body becomes empty (because set_flag was the only statement) and
|
||||
# there is no else clause, the whole if is dead.
|
||||
if not node.body and not node.orelse:
|
||||
return None
|
||||
if not node.body:
|
||||
# Body emptied but orelse remains — replace with the orelse block.
|
||||
return node.orelse if len(node.orelse) > 1 else (
|
||||
node.orelse[0] if node.orelse else None
|
||||
)
|
||||
return node
|
||||
|
||||
|
||||
def _process_source(src: str) -> tuple[str, Migrator]:
|
||||
tree = ast.parse(src)
|
||||
m = Migrator()
|
||||
new_tree = m.visit(tree)
|
||||
ast.fix_missing_locations(new_tree)
|
||||
return ast.unparse(new_tree), m
|
||||
|
||||
|
||||
def _iter_files(paths: Iterable[Path]) -> Iterable[Path]:
|
||||
for p in paths:
|
||||
if p.is_dir():
|
||||
yield from sorted(p.rglob("*.py"))
|
||||
elif p.suffix == ".py":
|
||||
yield p
|
||||
|
||||
|
||||
def _process_file(path: Path, *, write: bool) -> bool:
|
||||
"""Return True iff the file needs (or got) a migration.
|
||||
|
||||
Keyed on the migrator's transform counts, not on text equality: `ast.unparse`
|
||||
reformats and drops comments on every round-trip, so a file that needs no
|
||||
migration would otherwise look "changed" and have its comments stripped.
|
||||
"""
|
||||
original = path.read_text()
|
||||
try:
|
||||
new_src, m = _process_source(original)
|
||||
except SyntaxError as e:
|
||||
print(f"[skip] {path}: syntax error: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
migrated = (m.rewrote_create_network + m.removed_flag_calls
|
||||
+ m.removed_precision_assigns + m.removed_set_output_type) > 0
|
||||
if not migrated:
|
||||
for note in m.skipped:
|
||||
print(f"[note] {path}: {note}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
if write:
|
||||
path.write_text(new_src)
|
||||
print(
|
||||
f"[write] {path}: create_network={m.rewrote_create_network} "
|
||||
f"set_flag={m.removed_flag_calls} "
|
||||
f"precision_assign={m.removed_precision_assigns} "
|
||||
f"set_output_type={m.removed_set_output_type}"
|
||||
)
|
||||
else:
|
||||
diff = difflib.unified_diff(
|
||||
original.splitlines(keepends=True),
|
||||
new_src.splitlines(keepends=True),
|
||||
fromfile=f"{path} (weakly typed)",
|
||||
tofile=f"{path} (strongly typed)",
|
||||
)
|
||||
sys.stdout.writelines(diff)
|
||||
for note in m.skipped:
|
||||
print(f"[note] {path}: {note}", file=sys.stderr)
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("paths", nargs="+", type=Path, help="Files or directories to process.")
|
||||
parser.add_argument("--write", action="store_true",
|
||||
help="Rewrite files in place. Default is dry-run with unified diff.")
|
||||
args = parser.parse_args()
|
||||
|
||||
any_changed = False
|
||||
for path in _iter_files(args.paths):
|
||||
if _process_file(path, write=args.write):
|
||||
any_changed = True
|
||||
|
||||
# In dry-run, exit non-zero when changes are pending so callers/CI can gate
|
||||
# on it (like `black --check`). `--write` applies the changes and exits 0.
|
||||
return 1 if any_changed and not args.write else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Verify the `migrate.py` helper on an inline weakly-typed sample.
|
||||
# Run from anywhere; resolves paths relative to this script.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/verify.sh # run the verification end-to-end
|
||||
# bash scripts/verify.sh --keep # keep the temp workspace for inspection
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MIGRATE="${SCRIPT_DIR}/migrate.py"
|
||||
|
||||
if [[ ! -f "${MIGRATE}" ]]; then
|
||||
echo "ERROR: migrate.py not found at ${MIGRATE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
KEEP=0
|
||||
if [[ "${1:-}" == "--keep" ]]; then
|
||||
KEEP=1
|
||||
fi
|
||||
|
||||
WORK="$(mktemp -d -t trt-w2s-verify-XXXXXX)"
|
||||
|
||||
cleanup_workspace() {
|
||||
if [[ ${KEEP} -eq 1 ]]; then
|
||||
echo "Kept workspace at ${WORK}"
|
||||
else
|
||||
rm -rf "${WORK}"
|
||||
fi
|
||||
}
|
||||
trap cleanup_workspace EXIT
|
||||
|
||||
SAMPLE="${WORK}/sample_build.py"
|
||||
|
||||
# A representative weakly-typed Python builder. Contains every transform path
|
||||
# migrate.py is expected to handle: EXPLICIT_BATCH network flag, set_flag for
|
||||
# FP16/INT8 (removed) plus TF32/REFIT (preserved), per-layer precision overrides,
|
||||
# and the platform_has_fast_* gating that becomes dead code after migration.
|
||||
cat > "${SAMPLE}" <<'PY'
|
||||
import tensorrt as trt
|
||||
|
||||
TRT_LOGGER = trt.Logger(trt.Logger.WARNING)
|
||||
|
||||
def build():
|
||||
builder = trt.Builder(TRT_LOGGER)
|
||||
network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
|
||||
config = builder.create_builder_config()
|
||||
|
||||
if builder.platform_has_fast_fp16:
|
||||
config.set_flag(trt.BuilderFlag.FP16)
|
||||
config.set_flag(trt.BuilderFlag.TF32) # kept in TRT 11 (orthogonal to typing); must survive
|
||||
config.set_flag(trt.BuilderFlag.INT8)
|
||||
config.set_flag(trt.BuilderFlag.REFIT) # not a precision flag; must survive
|
||||
|
||||
parser = trt.OnnxParser(network, TRT_LOGGER)
|
||||
with open("model.onnx", "rb") as f:
|
||||
parser.parse(f.read())
|
||||
|
||||
last = network.get_layer(network.num_layers - 1)
|
||||
last.precision = trt.float16
|
||||
last.set_output_type(0, trt.float16)
|
||||
|
||||
plan = builder.build_serialized_network(network, config)
|
||||
return plan
|
||||
PY
|
||||
|
||||
echo "[verify] sample written to ${SAMPLE}"
|
||||
|
||||
echo "[verify] dry-run diff (exit 1 = changes pending, expected here):"
|
||||
python3 "${MIGRATE}" "${SAMPLE}" || true
|
||||
|
||||
echo "[verify] writing rewrite in place:"
|
||||
python3 "${MIGRATE}" "${SAMPLE}" --write
|
||||
|
||||
# Assertions on the rewritten file.
|
||||
fail=0
|
||||
assert_present() {
|
||||
local needle="$1"
|
||||
if ! grep -qF "${needle}" "${SAMPLE}"; then
|
||||
echo "ASSERT FAIL: expected to find: ${needle}" >&2
|
||||
fail=1
|
||||
fi
|
||||
}
|
||||
assert_absent() {
|
||||
local needle="$1"
|
||||
if grep -qF "${needle}" "${SAMPLE}"; then
|
||||
echo "ASSERT FAIL: expected to be removed: ${needle}" >&2
|
||||
fail=1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_present "NetworkDefinitionCreationFlag.STRONGLY_TYPED"
|
||||
assert_absent "NetworkDefinitionCreationFlag.EXPLICIT_BATCH"
|
||||
assert_absent "BuilderFlag.FP16"
|
||||
assert_absent "BuilderFlag.INT8"
|
||||
assert_present "BuilderFlag.TF32" # kTF32 is kept in TRT 11; must NOT be stripped
|
||||
assert_present "BuilderFlag.REFIT" # non-precision flag must remain
|
||||
assert_absent ".precision = trt.float16"
|
||||
assert_absent "set_output_type(0, trt.float16)"
|
||||
|
||||
# Syntactic validity.
|
||||
python3 -c "import ast, sys; ast.parse(open('${SAMPLE}').read())" || {
|
||||
echo "ASSERT FAIL: rewritten file is not valid Python" >&2
|
||||
fail=1
|
||||
}
|
||||
|
||||
if [[ ${fail} -ne 0 ]]; then
|
||||
echo "[verify] FAIL — see assertions above. Workspace: ${WORK}"
|
||||
KEEP=1
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[verify] OK — migrate.py produced a well-formed strongly-typed rewrite"
|
||||
@@ -0,0 +1,266 @@
|
||||
---
|
||||
name: trt-torch-quickstart
|
||||
description: >
|
||||
Compile a PyTorch model to a TensorRT engine via Torch-TensorRT — AOT or
|
||||
JIT — under the new strong-typing default. Use when the user
|
||||
compiles PyTorch to TensorRT without ONNX, hits "enabled_precisions
|
||||
should not be used when use_explicit_typing=True", sees Dynamo graph
|
||||
breaks or PyTorch fallback, debugs ABI errors at import torch_tensorrt, or
|
||||
needs the compatible torch / torch_tensorrt / tensorrt-cu13 version pins for
|
||||
TensorRT 11. Triggers: torch_tensorrt, torch_tensorrt.dynamo.compile,
|
||||
torch.compile backend torch_tensorrt, pytorch to tensorrt, ExportedProgram,
|
||||
Dynamo graph break, use_explicit_typing, enabled_precisions,
|
||||
torch_tensorrt.Input, min_block_size, truncate_double, tensorrt-cu13,
|
||||
version pinning, version compatibility. Adjacent skills: `trt-onnx-quickstart`,
|
||||
`trt-cpp-runtime-quickstart`. LLM token generation belongs in TensorRT-LLM.
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
author: NVIDIA Corporation
|
||||
version: "1.0"
|
||||
tags:
|
||||
- torch-tensorrt
|
||||
- dynamo
|
||||
- pytorch
|
||||
- quickstart
|
||||
- fp16
|
||||
---
|
||||
|
||||
# Torch-TensorRT Quickstart
|
||||
|
||||
Convert a PyTorch `nn.Module` to a TensorRT engine using `torch_tensorrt` (the [Torch-TensorRT](https://github.com/pytorch/TensorRT) frontend). Covers the AOT path (for production and C++ deploy) and the JIT path (for Python-only inference).
|
||||
|
||||
## When to Use
|
||||
|
||||
| Scenario | Use this skill? |
|
||||
|----------|-----------------|
|
||||
| PyTorch `nn.Module`; want a TensorRT engine without writing an ONNX intermediate | Yes |
|
||||
| Model uses ops that don't export cleanly to ONNX (custom autograd, dynamic control flow) | Yes |
|
||||
| Want Torch-TensorRT's automatic fallback of unsupported subgraphs to PyTorch | Yes |
|
||||
| Need a serialized engine for C++ deploy from a PyTorch source | Yes — use AOT path here, then `trt-cpp-runtime-quickstart` for the C++ load |
|
||||
| Have a working ONNX file already | No — use `trt-onnx-quickstart` |
|
||||
| LLM token generation (Llama, Mistral, Qwen text gen) | **No** — route to [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) |
|
||||
| Have a `.plan` file already and want to run it from C++ | No — use `trt-cpp-runtime-quickstart` |
|
||||
| Migrating an existing weakly-typed TRT network to strongly-typed | No — use `trt-strong-typing-migration` |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Torch-TensorRT versions are tightly coupled to a (torch, tensorrt-cu13, CUDA) triple; mismatches produce ABI errors at `import torch_tensorrt`. **Pin matrix source of truth:** the [pytorch/TensorRT releases page](https://github.com/pytorch/TensorRT/releases) — each release lists the exact `torch` / `tensorrt` / CUDA versions it was built against. Do not invent pins from memory.
|
||||
|
||||
For TensorRT 11.0:
|
||||
|
||||
| Component | Required | Notes |
|
||||
|-----------|----------|-------|
|
||||
| TensorRT | 11.x (`tensorrt-cu13` wheel) | Mixing `-cu12` and `-cu13` wheels breaks imports. |
|
||||
| CUDA toolkit | 13.x | Driver R590+. |
|
||||
| Python | ≥ 3.10 | TRT 11 dropped 3.9 and earlier. |
|
||||
| `torch` | Match the `torch_tensorrt` release notes for the chosen `torch_tensorrt` version. | |
|
||||
| `torch_tensorrt` | The release matching the TRT 11.0 RC — see [pytorch/TensorRT releases](https://github.com/pytorch/TensorRT/releases). | |
|
||||
|
||||
**Recommended environment**: NGC TensorRT container with PyTorch (`nvcr.io/nvidia/pytorch:<tag>` or `nvcr.io/nvidia/tensorrt:<tag>` with `torch_tensorrt` pip-installed on top). Run with `--gpus all`.
|
||||
|
||||
Verification:
|
||||
```bash
|
||||
python3 -c "import torch, torch_tensorrt, tensorrt; print(torch.__version__, torch_tensorrt.__version__, tensorrt.__version__)"
|
||||
```
|
||||
|
||||
## Step 1 — Load and prepare the model
|
||||
|
||||
Use a `torch.nn.Module` in eval mode. Torch-TensorRT will trace through it.
|
||||
|
||||
```python
|
||||
import torch
|
||||
import torchvision.models as models
|
||||
|
||||
model = models.resnet50(weights=None).eval().cuda()
|
||||
example = torch.randn(1, 3, 224, 224, device="cuda")
|
||||
```
|
||||
|
||||
Notes:
|
||||
- `weights=None` skips the download — fine for shape/perf testing. For accuracy work, load real weights.
|
||||
- Model must be on CUDA before compile. Torch-TensorRT does not move it for you.
|
||||
- `.eval()` matters: BatchNorm and Dropout behave differently in train mode and can produce different engines.
|
||||
|
||||
## Step 2 — Compile to a TensorRT engine (AOT)
|
||||
|
||||
Two compile paths — pick by deployment target:
|
||||
|
||||
| Goal | Path | API |
|
||||
|------|------|-----|
|
||||
| Serialized engine (production, C++ deploy, reuse) | **AOT** — this section | `torch_tensorrt.dynamo.compile` → `torch_tensorrt.save` |
|
||||
| In-process Python callable only (no serialized engine) | **JIT** — end of this section | `torch.compile(backend="torch_tensorrt")` |
|
||||
|
||||
The AOT path uses `torch_tensorrt.dynamo.compile` and returns a serializable `ExportedProgram`.
|
||||
|
||||
**Important — strong typing is the default in torch_tensorrt ≥ 2.12.** With `use_explicit_typing=True` (the default), engine precision is **inferred from the exported model's dtype** — passing `enabled_precisions={torch.float16}` raises `AssertionError`. To compile in FP16, cast the model and example tensor to FP16 before exporting:
|
||||
|
||||
```python
|
||||
import torch
|
||||
import torch_tensorrt
|
||||
|
||||
model = MyModel().eval().cuda().half() # cast to FP16
|
||||
example = torch.randn(1, 3, 224, 224, device="cuda", dtype=torch.float16)
|
||||
|
||||
trt_gm = torch_tensorrt.dynamo.compile(
|
||||
torch.export.export(model, (example,)),
|
||||
inputs=[example],
|
||||
# No enabled_precisions — precision comes from the model dtype.
|
||||
truncate_double=True,
|
||||
min_block_size=1,
|
||||
)
|
||||
|
||||
# Serialize for later / C++ loading — use torch_tensorrt.save, NOT raw bytes.
|
||||
torch_tensorrt.save(trt_gm, "resnet50_trt.ep", inputs=[example])
|
||||
```
|
||||
|
||||
**Do not extract raw engine bytes via `submod.engine` and write them with `open(...).write(...)`.** That bypasses Torch-TensorRT's metadata wrapper: the blob will not round-trip through `torch_tensorrt.load(...)` and silently drops the dispatch graph that handles partial-fallback subgraphs. `torch_tensorrt.save(trt_gm, path, inputs=...)` is the only supported serialization path.
|
||||
|
||||
For mixed precision or to override the model dtype, set `use_explicit_typing=False` and `enabled_precisions` applies as before (the weakly-typed path, deprecated in TRT 11):
|
||||
|
||||
```python
|
||||
trt_gm = torch_tensorrt.dynamo.compile(
|
||||
torch.export.export(model, (example,)),
|
||||
inputs=[example],
|
||||
use_explicit_typing=False, # weakly-typed (deprecated)
|
||||
enabled_precisions={torch.float16, torch.float32},
|
||||
truncate_double=True,
|
||||
)
|
||||
```
|
||||
|
||||
Key arguments:
|
||||
- `truncate_double=True`: silently downcasts FP64 constants. Without it, any FP64 op forces a partition boundary.
|
||||
- `min_block_size`: smallest subgraph (in node count) worth handing to TRT. `1` is aggressive; default `5` avoids tiny TRT subgraphs that don't pay back their launch overhead.
|
||||
- `workspace_size`: bytes of scratch memory the TRT builder can use. Leave unset to let TRT decide.
|
||||
|
||||
**JIT alternative (Python-only inference):** if you just want a callable and not a serialized engine, use `torch.compile` with the Torch-TensorRT backend. Under the strong-typing default, precision comes from the model dtype — `.half()` the model for FP16 rather than passing `enabled_precisions`:
|
||||
```python
|
||||
model = model.half() # FP16 inferred from model dtype (strong typing default)
|
||||
example = example.half()
|
||||
trt_model = torch.compile(model, backend="torch_tensorrt")
|
||||
out = trt_model(example) # compiles lazily on the first call, then runs through TRT in-process
|
||||
```
|
||||
This skips `torch.export` and gives a `torch.compile`-wrapped callable. It cannot be loaded from C++ — use the AOT path above if you need that.
|
||||
|
||||
## Step 3 — Dynamic shapes
|
||||
|
||||
Wrap inputs in `torch_tensorrt.Input` with `min_shape`/`opt_shape`/`max_shape`:
|
||||
|
||||
```python
|
||||
dynamic_input = torch_tensorrt.Input(
|
||||
min_shape=(1, 3, 224, 224),
|
||||
opt_shape=(8, 3, 224, 224),
|
||||
max_shape=(32, 3, 224, 224),
|
||||
dtype=torch.float16,
|
||||
)
|
||||
trt_gm = torch_tensorrt.dynamo.compile(
|
||||
torch.export.export(model, (example,), dynamic_shapes={"x": {0: torch.export.Dim("batch", min=1, max=32)}}),
|
||||
inputs=[dynamic_input],
|
||||
# Strong typing: FP16 comes from the input/model dtype, not enabled_precisions.
|
||||
)
|
||||
```
|
||||
|
||||
Pitfalls:
|
||||
- The `dynamic_shapes=` argument to `torch.export.export` and the `Input` ranges must agree. A mismatch produces a builder error 2–3 minutes into compilation, not at export time.
|
||||
- `opt_shape` is what TRT tunes kernels for. Set it to the most common runtime shape, not the midpoint.
|
||||
- Only mark dimensions dynamic that actually vary at runtime; unnecessary dynamism degrades performance.
|
||||
|
||||
## Step 4 — Run inference and verify
|
||||
|
||||
Raw-tensor comparison (`assert_close`) is fine for FP32 but **misleading for FP16** — a few outlier positions exceed the tolerance even when downstream behavior is identical. Use a **semantic check** appropriate to the model type:
|
||||
|
||||
```python
|
||||
with torch.inference_mode():
|
||||
torch_out = model(example)
|
||||
trt_out = trt_gm(example)
|
||||
```
|
||||
|
||||
**Image classification** — top-1 (or top-5) class match, plus mean-softmax-distance:
|
||||
|
||||
```python
|
||||
assert torch.equal(torch_out.argmax(-1), trt_out.argmax(-1)), "top-1 mismatch"
|
||||
mean_prob_diff = (torch.softmax(torch_out.float(), -1) - torch.softmax(trt_out.float(), -1)).abs().mean()
|
||||
assert mean_prob_diff < 1e-3
|
||||
```
|
||||
|
||||
**Sentence / token embeddings** — cosine similarity per token, mean across the batch:
|
||||
|
||||
```python
|
||||
cos = torch.nn.functional.cosine_similarity(torch_out.flatten(0, -2), trt_out.flatten(0, -2), dim=-1)
|
||||
assert cos.mean() > 0.999
|
||||
```
|
||||
|
||||
**Detection / regression heads** — bounded raw-tensor closeness on the meaningful sub-tensor (e.g., bbox coordinates), softmax check on the class logits.
|
||||
|
||||
**FP32 sanity** — for any model, an FP32 (no `.half()`) run should pass `torch.testing.assert_close(trt_out, torch_out, rtol=1e-4, atol=1e-4)`. If that fails, the issue is in the compilation, not numeric precision.
|
||||
|
||||
## Loading the serialized engine
|
||||
|
||||
**From Python**:
|
||||
```python
|
||||
import torch_tensorrt
|
||||
loaded = torch_tensorrt.load("resnet50_trt.ep").module()
|
||||
out = loaded(example)
|
||||
```
|
||||
|
||||
**From C++**: this skill does NOT cover C++ loading. The `.ep` file `torch_tensorrt.save` produces by default is a `torch.export` archive for the **Python** `torch_tensorrt.load(...)` path — it is *not* directly consumable by the plain TensorRT C++ runtime (`IRuntime` / `deserializeCudaEngine`), which expects a serialized TensorRT engine (`.plan`). Two supported C++ routes:
|
||||
|
||||
- Save with `output_format="torchscript"` and deploy the TorchScript module with libtorch + the Torch-TensorRT C++ runtime.
|
||||
- If you only need the raw TensorRT engine in C++, build it through the ONNX path (`trt-onnx-quickstart`) and load the resulting `.plan` with `trt-cpp-runtime-quickstart` (modern `IRuntime` + `enqueueV3` + `setTensorAddress`).
|
||||
|
||||
Do not inline a C++ snippet here — it will diverge from the canonical pattern.
|
||||
|
||||
## Common issues
|
||||
|
||||
**`RuntimeError: Trying to create tensor with negative dimension`** during export — the model has a shape-dependent control flow path that `torch.export` can't trace. Either rewrite with `torch.cond` / `torch.where`, or fall back to `ir="dynamo"` + `min_block_size=1` so the unsupported region runs in PyTorch.
|
||||
|
||||
**`Unsupported operator` warnings** — Torch-TensorRT partitions around them and runs those nodes in PyTorch. Set `require_full_compilation=True` to turn this into an error; use during development to find what's actually falling back.
|
||||
|
||||
## Diagnosing excessive PyTorch fallback
|
||||
|
||||
If the compile completes but most of the model fell back to PyTorch (only a few subgraphs run on TRT), three knobs in order:
|
||||
|
||||
1. **Identify what fell back.** Run with `TORCH_LOGS="graph_breaks"` and `require_full_compilation=True` to escalate fallbacks to errors that name the offending op. Without this you don't know what the autotuner rejected.
|
||||
```bash
|
||||
TORCH_LOGS="graph_breaks" python3 your_compile.py
|
||||
```
|
||||
2. **Reconsider `min_block_size`.** This is a real trade-off — not just a knob.
|
||||
- Default `min_block_size=5`: Torch-TensorRT only hands a subgraph to TRT if it has ≥5 nodes. Tiny TRT subgraphs cost more in kernel-launch overhead than they save.
|
||||
- `min_block_size=1`: aggressive — every supported op goes to TRT, even single-node subgraphs. Useful to see what's *theoretically* supported, but commonly slower at inference due to the per-subgraph launch tax.
|
||||
- Recommended: leave at default `5` for production; drop to `1` only during diagnosis to see maximum TRT coverage.
|
||||
3. **Look for shape-dependent control flow.** `torch.export` traces only one branch of `if x.shape[0] > 0:` style code; rewrite with `torch.where`, `torch.cond`, or lift the condition out of the model. The trace error names the offending op.
|
||||
|
||||
If fallback persists after all three, you have a genuine unsupported op — write a converter (see Torch-TensorRT upstream docs) or switch to the ONNX path (`trt-onnx-quickstart`).
|
||||
|
||||
**Engine builds but output is garbage** — almost always a dtype issue. Check:
|
||||
1. Model/input dtype matches your pipeline — under strong typing precision follows the model dtype (`.half()` for FP16); `enabled_precisions` applies only under weak typing (`use_explicit_typing=False`).
|
||||
2. `truncate_double=True` if the model has any FP64 constants (common in positional encodings).
|
||||
3. The model was `.eval()` before export.
|
||||
|
||||
**Compile takes >10 minutes** — turn on the builder log to see what's going on:
|
||||
```python
|
||||
import torch_tensorrt.logging as ttlog
|
||||
ttlog.set_reportable_log_level(ttlog.Level.Info)
|
||||
```
|
||||
Usually it's the autotuner exploring kernel variants for a heavy GEMM/conv. To cap exploration, set `workspace_size` smaller (less scratch → fewer candidates).
|
||||
|
||||
**`ImportError: cannot import name 'XYZ' from 'torch_tensorrt'`** — version mismatch. Confirm the pin matrix; the `torch_tensorrt` Python API moves between minor versions.
|
||||
|
||||
## Numerical debugging
|
||||
|
||||
If `assert_close` fails by more than ~5× tolerance:
|
||||
1. Re-run in FP32 — drop the model's `.half()` (or under weak typing, `enabled_precisions={torch.float32}`). If that matches PyTorch, the issue is FP16 accumulator drift in a specific op — usually a softmax or LayerNorm. The pragmatic fix is to wrap the offending submodule in a module excluded from the compile (`min_block_size` won't help; explicit exclusion is needed). See the [Torch-TensorRT lowering guide](https://pytorch.org/TensorRT/) for the version-specific exclusion API.
|
||||
2. Sanity-check at FP32 first, then re-introduce FP16 and bisect — almost always reveals one offending op.
|
||||
3. For FP8: check that the calibration distribution actually covers the runtime distribution. FP8 has no headroom for outliers.
|
||||
|
||||
See `trt-strong-typing-migration` for the broader weak-vs-strong typing discussion. By default Torch-TensorRT produces strongly-typed engines in TRT 11.0 — types inferred from the exported program — but you can override.
|
||||
|
||||
## What this skill is not
|
||||
|
||||
- Not a guide to writing custom Torch-TensorRT converters. Use the upstream Torch-TensorRT docs for that.
|
||||
- Not for TensorRT-LLM / LLM inference — use the TRT-LLM `examples/` flow instead.
|
||||
- Not for QAT (quantization-aware training) — Torch-TensorRT consumes the QAT model but the training loop is upstream.
|
||||
|
||||
## References
|
||||
|
||||
- Upstream Torch-TensorRT: https://pytorch.org/TensorRT/
|
||||
- Version pin matrix: [pytorch/TensorRT releases](https://github.com/pytorch/TensorRT/releases)
|
||||
- Related skills: `trt-onnx-quickstart`, `trt-strong-typing-migration`, `trt-cpp-runtime-quickstart`
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
---
|
||||
AccessModifierOffset: -4
|
||||
AlignAfterOpenBracket: DontAlign
|
||||
AlignConsecutiveAssignments: false
|
||||
AlignConsecutiveDeclarations: false
|
||||
AlignEscapedNewlinesLeft: false
|
||||
AlignOperands: false
|
||||
AlignTrailingComments: true
|
||||
AllowAllParametersOfDeclarationOnNextLine: true
|
||||
AllowShortBlocksOnASingleLine: false
|
||||
AllowShortCaseLabelsOnASingleLine: true
|
||||
AllowShortFunctionsOnASingleLine: Empty
|
||||
AllowShortIfStatementsOnASingleLine: false
|
||||
AllowShortLoopsOnASingleLine: false
|
||||
AlwaysBreakAfterDefinitionReturnType: None
|
||||
AlwaysBreakAfterReturnType: None
|
||||
AlwaysBreakBeforeMultilineStrings: true
|
||||
AlwaysBreakTemplateDeclarations: true
|
||||
BasedOnStyle: None
|
||||
BinPackArguments: true
|
||||
BinPackParameters: true
|
||||
# Almost the same as Allman style, but explicitly disabling BeforeLambdaBody
|
||||
# for backwards compatibility with clang-format-10 Allman style.
|
||||
# See also https://reviews.llvm.org/D44609
|
||||
BreakBeforeBraces: Custom
|
||||
BraceWrapping:
|
||||
AfterCaseLabel: true
|
||||
AfterClass: true
|
||||
AfterControlStatement: Always
|
||||
AfterEnum: true
|
||||
AfterFunction: true
|
||||
AfterNamespace: true
|
||||
AfterObjCDeclaration: true
|
||||
AfterStruct: true
|
||||
AfterUnion: true
|
||||
AfterExternBlock: true
|
||||
BeforeCatch: true
|
||||
BeforeElse: true
|
||||
BeforeLambdaBody: false
|
||||
BeforeWhile: false
|
||||
IndentBraces: false
|
||||
SplitEmptyFunction: true
|
||||
SplitEmptyRecord: true
|
||||
SplitEmptyNamespace: true
|
||||
BreakBeforeBinaryOperators: All
|
||||
BreakBeforeTernaryOperators: true
|
||||
BreakConstructorInitializersBeforeComma: true
|
||||
ColumnLimit: 120
|
||||
CommentPragmas: '^ IWYU pragma:'
|
||||
ConstructorInitializerAllOnOneLineOrOnePerLine: false
|
||||
ConstructorInitializerIndentWidth: 4
|
||||
ContinuationIndentWidth: 4
|
||||
Cpp11BracedListStyle: true
|
||||
DerivePointerAlignment: false
|
||||
DisableFormat: false
|
||||
ExperimentalAutoDetectBinPacking: false
|
||||
ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ]
|
||||
IncludeBlocks: Preserve
|
||||
IncludeCategories:
|
||||
- Regex: '^"(llvm|llvm-c|clang|clang-c)/'
|
||||
Priority: 2
|
||||
- Regex: '^(<|"(gtest|isl|json)/)'
|
||||
Priority: 3
|
||||
- Regex: '.*'
|
||||
Priority: 1
|
||||
IndentCaseLabels: false
|
||||
IndentWidth: 4
|
||||
IndentWrappedFunctionNames: false
|
||||
KeepEmptyLinesAtTheStartOfBlocks: true
|
||||
Language: Cpp
|
||||
MacroBlockBegin: ''
|
||||
MacroBlockEnd: ''
|
||||
MaxEmptyLinesToKeep: 1
|
||||
NamespaceIndentation: None
|
||||
ObjCBlockIndentWidth: 4
|
||||
ObjCSpaceAfterProperty: true
|
||||
ObjCSpaceBeforeProtocolList: true
|
||||
PenaltyBreakBeforeFirstCallParameter: 19
|
||||
PenaltyBreakComment: 300
|
||||
PenaltyBreakFirstLessLess: 120
|
||||
PenaltyBreakString: 1000
|
||||
PenaltyExcessCharacter: 1000000
|
||||
PenaltyReturnTypeOnItsOwnLine: 60
|
||||
PointerAlignment: Left
|
||||
PointerBindsToType: false
|
||||
QualifierAlignment: Right
|
||||
ReflowComments: true
|
||||
SortIncludes: true
|
||||
SpaceAfterCStyleCast: true
|
||||
SpaceBeforeAssignmentOperators: true
|
||||
SpaceBeforeParens: ControlStatements
|
||||
SpaceInEmptyParentheses: false
|
||||
SpacesBeforeTrailingComments: 1
|
||||
SpacesInAngles: false
|
||||
SpacesInCStyleCastParentheses: false
|
||||
SpacesInContainerLiterals: true
|
||||
SpacesInParentheses: false
|
||||
SpacesInSquareBrackets: false
|
||||
Standard: Cpp11
|
||||
StatementMacros: [API_ENTRY_TRY,TRT_TRY]
|
||||
TabWidth: 4
|
||||
UseTab: Never
|
||||
@@ -0,0 +1,3 @@
|
||||
/.git*
|
||||
build*
|
||||
/third_party
|
||||
@@ -0,0 +1,4 @@
|
||||
# This file defines code ownership rules for the repository.
|
||||
|
||||
# Default ownership
|
||||
* @NVIDIA/trt-devs
|
||||
@@ -0,0 +1,67 @@
|
||||
---
|
||||
name: Report a TensorRT issue
|
||||
about: The more information you share, the more feedback we can provide.
|
||||
title: 'XXX failure of TensorRT X.Y when running XXX on GPU XXX'
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## Description
|
||||
|
||||
<!--
|
||||
A clear and concise description of the issue.
|
||||
|
||||
For example: I tried to run model ABC on GPU, but it fails with the error below (share a 2-3 line error log).
|
||||
-->
|
||||
|
||||
|
||||
## Environment
|
||||
|
||||
<!-- Please share any setup information you know. This will help us to understand and address your case. -->
|
||||
|
||||
**TensorRT Version**:
|
||||
|
||||
**NVIDIA GPU**:
|
||||
|
||||
**NVIDIA Driver Version**:
|
||||
|
||||
**CUDA Version**:
|
||||
|
||||
**CUDNN Version**:
|
||||
|
||||
|
||||
Operating System:
|
||||
|
||||
Python Version (if applicable):
|
||||
|
||||
Tensorflow Version (if applicable):
|
||||
|
||||
PyTorch Version (if applicable):
|
||||
|
||||
Baremetal or Container (if so, version):
|
||||
|
||||
|
||||
## Relevant Files
|
||||
|
||||
<!-- Please include links to any models, data, files, or scripts necessary to reproduce your issue. (Github repo, Google Drive/Dropbox, etc.) -->
|
||||
|
||||
**Model link**:
|
||||
|
||||
|
||||
## Steps To Reproduce
|
||||
|
||||
<!--
|
||||
Craft a minimal bug report following this guide - https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports
|
||||
|
||||
Please include:
|
||||
* Exact steps/commands to build your repro
|
||||
* Exact steps/commands to run your repro
|
||||
* Full traceback of errors encountered
|
||||
-->
|
||||
|
||||
**Commands or scripts**:
|
||||
|
||||
**Have you tried [the latest release](https://developer.nvidia.com/tensorrt)?**:
|
||||
|
||||
**Can this model run on other frameworks?** For example run ONNX model with ONNXRuntime (`polygraphy run <model.onnx> --onnxrt`):
|
||||
@@ -0,0 +1,100 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# A workflow to trigger ci on hybrid infra (github + self hosted runner)
|
||||
name: Blossom-CI
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
platform:
|
||||
description: "runs-on argument"
|
||||
required: false
|
||||
args:
|
||||
description: "argument"
|
||||
required: false
|
||||
jobs:
|
||||
Authorization:
|
||||
name: Authorization
|
||||
runs-on: blossom
|
||||
outputs:
|
||||
args: ${{ env.args }}
|
||||
|
||||
# This job only runs for pull request comments
|
||||
if: |
|
||||
github.event.comment.body == '/blossom-ci' &&
|
||||
(
|
||||
github.actor == 'rajeevsrao' ||
|
||||
github.actor == 'kevinch-nv' ||
|
||||
github.actor == 'ttyio' ||
|
||||
github.actor == 'zerollzeng' ||
|
||||
github.actor == 'nvpohanh' ||
|
||||
github.actor == 'poweiw'
|
||||
)
|
||||
steps:
|
||||
- name: Check if comment is issued by authorized person
|
||||
run: blossom-ci
|
||||
env:
|
||||
OPERATION: "AUTH"
|
||||
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
|
||||
|
||||
Vulnerability-scan:
|
||||
name: Vulnerability scan
|
||||
needs: [Authorization]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
repository: ${{ fromJson(needs.Authorization.outputs.args).repo }}
|
||||
ref: ${{ fromJson(needs.Authorization.outputs.args).ref }}
|
||||
lfs: "true"
|
||||
- name: Run blossom action
|
||||
uses: NVIDIA/blossom-action@main
|
||||
env:
|
||||
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
|
||||
with:
|
||||
args1: ${{ fromJson(needs.Authorization.outputs.args).args1 }}
|
||||
args2: ${{ fromJson(needs.Authorization.outputs.args).args2 }}
|
||||
args3: ${{ fromJson(needs.Authorization.outputs.args).args3 }}
|
||||
|
||||
Job-trigger:
|
||||
name: Start ci job
|
||||
needs: [Vulnerability-scan]
|
||||
runs-on: blossom
|
||||
steps:
|
||||
- name: Start ci job
|
||||
run: blossom-ci
|
||||
env:
|
||||
OPERATION: "START-CI-JOB"
|
||||
CI_SERVER: ${{ secrets.CI_SERVER }}
|
||||
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
Upload-Log:
|
||||
name: Upload log
|
||||
runs-on: blossom
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Jenkins log for pull request ${{ fromJson(github.event.inputs.args).pr }} (click here)
|
||||
run: blossom-ci
|
||||
env:
|
||||
OPERATION: "POST-PROCESSING"
|
||||
CI_SERVER: ${{ secrets.CI_SERVER }}
|
||||
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,18 @@
|
||||
name: Docker Image CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
|
||||
jobs:
|
||||
|
||||
build-ubuntu2004:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Build TensorRT-OSS ubuntu20.04 container
|
||||
run: docker build . --file docker/ubuntu-20.04.Dockerfile --build-arg uid=1000 --build-arg gid=1000 --tag tensorrt-ubuntu20.04:$(date +%s)
|
||||
@@ -0,0 +1,16 @@
|
||||
name: Remove feedback label on comment
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
remove_label:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.issue.user.id == github.event.comment.user.id
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions-ecosystem/action-remove-labels@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
labels: "waiting for feedback"
|
||||
@@ -0,0 +1,46 @@
|
||||
name: Label New Issues
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
label-issue:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout private action repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: NVIDIA/goggles_action
|
||||
path: ./.github/actions/goggles_action # local path to store the action
|
||||
ref: v1.3.0
|
||||
|
||||
- name: AI Label Issue
|
||||
uses: ./.github/actions/goggles_action/actions/llm_label
|
||||
with:
|
||||
ACTION_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
LLM_MODEL_NAME: ${{ secrets.LLM_MODEL_NAME }}
|
||||
LLM_TOKEN_SERVER_URL: ${{ secrets.LLM_TOKEN_SERVER_URL }}
|
||||
LLM_TOKEN_CLIENT_ID: ${{ secrets.LLM_TOKEN_CLIENT_ID }}
|
||||
LLM_TOKEN_CLIENT_SECRET: ${{ secrets.LLM_TOKEN_CLIENT_SECRET }}
|
||||
LLM_GENERATE_URL: ${{ secrets.LLM_GENERATE_URL }}
|
||||
LLM_TOKEN_SCOPE: ${{ secrets.LLM_TOKEN_SCOPE }}
|
||||
REPO_OWNER: ${{ github.repository_owner }}
|
||||
REPO_NAME: ${{ github.event.repository.name }}
|
||||
ISSUE_NUMBER: ${{ github.event.issue.number }}
|
||||
ISSUE_TITLE: ${{ github.event.issue.title }}
|
||||
ISSUE_BODY: ${{ github.event.issue.body }}
|
||||
GITHUB_API_URL: ${{ github.api_url }}
|
||||
ACTIONS_STEP_VERBOSE: false
|
||||
EXCLUDED_LABELS: "Investigating,internal-bug-tracked,stale,triaged,wontfix"
|
||||
LLM_SYSTEM_PROMPT: |
|
||||
You are an expert GitHub issue labeler. Your task is to analyze the provided issue title, issue body, and a list of available labels with their descriptions.
|
||||
Based on this information, select the single most appropriate label from the list that best captures the primary issue or request.
|
||||
Prefer selecting only one label that represents the main topic or problem. Only suggest multiple labels if the issue genuinely spans multiple distinct areas that are equally important.
|
||||
Respond with ONLY the chosen label name (e.g., 'bug', 'feature-request') or comma-separated names if multiple are truly needed.
|
||||
If no labels seem appropriate, respond with 'NONE'.
|
||||
Do not add any other text, explanation, or markdown formatting.
|
||||
@@ -0,0 +1,28 @@
|
||||
name: Label and close inactive issues
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 * * * *"
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- uses: actions/stale@v9
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
stale-issue-message: 'Issue has not received an update in over 14 days. Adding stale label. Please note the issue will be closed in 14 days after being marked stale if there is no update.'
|
||||
stale-pr-message: 'PR has not received an update in over 14 days. Adding stale label. Please note the PR will be closed in 14 days after being marked stale if there is no update.'
|
||||
close-issue-message: 'This issue was closed because it has been 14 days without activity since it has been marked as stale.'
|
||||
close-pr-message: 'This PR was closed because it has been 14 days without activity since it has been marked as stale.'
|
||||
days-before-issue-stale: 14
|
||||
days-before-close: 14
|
||||
only-labels: 'waiting for feedback'
|
||||
labels-to-add-when-unstale: 'investigating'
|
||||
labels-to-remove-when-unstale: 'stale,waiting for feedback'
|
||||
stale-issue-label: 'stale'
|
||||
stale-pr-label: 'stale'
|
||||
@@ -0,0 +1,9 @@
|
||||
build/
|
||||
/demo/BERT/models
|
||||
/demo/BERT/engines
|
||||
/demo/BERT/squad/*.json
|
||||
/docker/jetpack_files/*
|
||||
*.sln
|
||||
*.vcxproj
|
||||
externals/
|
||||
**/.DS_Store
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
[submodule "third_party/protobuf"]
|
||||
path = third_party/protobuf
|
||||
url = https://github.com/protocolbuffers/protobuf.git
|
||||
branch = 3.20.x
|
||||
[submodule "third_party/cub"]
|
||||
path = third_party/cub
|
||||
url = https://github.com/NVlabs/cub.git
|
||||
branch = 1.8.0
|
||||
[submodule "parsers/onnx"]
|
||||
path = parsers/onnx
|
||||
url = https://github.com/onnx/onnx-tensorrt.git
|
||||
branch = release/10.7-GA
|
||||
+1292
File diff suppressed because it is too large
Load Diff
+444
@@ -0,0 +1,444 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
cmake_minimum_required(VERSION 3.31 FATAL_ERROR)
|
||||
include(cmake/modules/set_ifndef.cmake)
|
||||
include(cmake/modules/find_library_create_target.cmake)
|
||||
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules)
|
||||
include(InstallUtils)
|
||||
|
||||
set_ifndef(TRT_LIB_DIR ${CMAKE_BINARY_DIR})
|
||||
set_ifndef(TRT_OUT_DIR ${CMAKE_BINARY_DIR})
|
||||
|
||||
# Converts Windows paths
|
||||
cmake_path(SET TRT_LIB_DIR ${TRT_LIB_DIR})
|
||||
cmake_path(SET TRT_OUT_DIR ${TRT_OUT_DIR})
|
||||
|
||||
# Resolve TRT_INCLUDE_DIR: prefer installed package headers over in-repo headers
|
||||
# to avoid ABI mismatches when headers lag behind the installed package.
|
||||
if(NOT DEFINED TRT_INCLUDE_DIR)
|
||||
if(EXISTS "${TRT_LIB_DIR}/../include/NvInferVersion.h")
|
||||
# Tar-style package: include/ is a sibling of the lib directory.
|
||||
cmake_path(SET TRT_INCLUDE_DIR "${TRT_LIB_DIR}/../include")
|
||||
elseif(EXISTS "/usr/include/NvInferVersion.h")
|
||||
# Deb package: headers at standard system path.
|
||||
set(TRT_INCLUDE_DIR "/usr/include")
|
||||
else()
|
||||
# No package headers found; fall back to in-repo headers.
|
||||
cmake_path(SET TRT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include")
|
||||
set(_TRT_USING_REPO_HEADERS TRUE)
|
||||
endif()
|
||||
else()
|
||||
cmake_path(SET TRT_INCLUDE_DIR "${TRT_INCLUDE_DIR}")
|
||||
endif()
|
||||
|
||||
if(NOT EXISTS "${TRT_INCLUDE_DIR}/NvInferVersion.h")
|
||||
message(FATAL_ERROR "NvInferVersion.h not found in ${TRT_INCLUDE_DIR}. "
|
||||
"Set -DTRT_INCLUDE_DIR to the directory containing TensorRT headers.")
|
||||
endif()
|
||||
message(STATUS "Using TensorRT headers from: ${TRT_INCLUDE_DIR}")
|
||||
|
||||
# Required to export symbols to build *.libs
|
||||
if(WIN32)
|
||||
add_compile_definitions(TENSORRT_BUILD_LIB 1)
|
||||
endif()
|
||||
|
||||
# Route all targets to TRT_OUT_DIR by default. Per-target overrides still win.
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${TRT_OUT_DIR} CACHE PATH "")
|
||||
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${TRT_OUT_DIR} CACHE PATH "")
|
||||
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${TRT_OUT_DIR} CACHE PATH "")
|
||||
|
||||
if(WIN32)
|
||||
set(STATIC_LIB_EXT "lib")
|
||||
else()
|
||||
set(STATIC_LIB_EXT "a")
|
||||
endif()
|
||||
|
||||
file(STRINGS "${TRT_INCLUDE_DIR}/NvInferVersion.h" VERSION_STRINGS REGEX "#define TRT_.*_ENTERPRISE")
|
||||
|
||||
foreach(TYPE MAJOR MINOR PATCH BUILD)
|
||||
string(REGEX MATCH "TRT_${TYPE}_ENTERPRISE [0-9]+" TRT_TYPE_STRING ${VERSION_STRINGS})
|
||||
string(REGEX MATCH "[0-9]+" TRT_${TYPE} ${TRT_TYPE_STRING})
|
||||
endforeach(TYPE)
|
||||
|
||||
set(TRT_VERSION "${TRT_MAJOR}.${TRT_MINOR}.${TRT_PATCH}" CACHE STRING "TensorRT project version")
|
||||
set(ONNX2TRT_VERSION "${TRT_MAJOR}.${TRT_MINOR}.${TRT_PATCH}" CACHE STRING "ONNX2TRT project version")
|
||||
set(TRT_SOVERSION "${TRT_MAJOR}" CACHE STRING "TensorRT library so version")
|
||||
message("Building for TensorRT version: ${TRT_VERSION}, library version: ${TRT_SOVERSION}")
|
||||
|
||||
# Warn if in-repo headers don't match the installed library version.
|
||||
if(_TRT_USING_REPO_HEADERS AND NOT "${TRT_LIB_DIR}" STREQUAL "${CMAKE_BINARY_DIR}" AND NOT WIN32)
|
||||
file(GLOB _trt_libs "${TRT_LIB_DIR}/libnvinfer.so.*")
|
||||
if(_trt_libs AND NOT EXISTS "${TRT_LIB_DIR}/libnvinfer.so.${TRT_MAJOR}")
|
||||
message(WARNING
|
||||
"In-repo headers report TensorRT major version ${TRT_MAJOR}, but "
|
||||
"no matching libnvinfer.so.${TRT_MAJOR} was found in ${TRT_LIB_DIR}. "
|
||||
"The headers may not match the installed library. Set -DTRT_INCLUDE_DIR "
|
||||
"to the headers matching your TensorRT installation.")
|
||||
endif()
|
||||
unset(_trt_libs)
|
||||
endif()
|
||||
unset(_TRT_USING_REPO_HEADERS)
|
||||
|
||||
if(NOT DEFINED CMAKE_TOOLCHAIN_FILE)
|
||||
find_program(CMAKE_CXX_COMPILER NAMES $ENV{CXX} g++)
|
||||
endif()
|
||||
|
||||
set(CMAKE_SKIP_BUILD_RPATH True)
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
|
||||
# Set CMAKE_CUDA_ARCHITECTURES before project() so CMake's CMP0104 policy does not
|
||||
# auto-detect and overwrite it with the host GPU's SM. CUDAToolkit_VERSION is not
|
||||
# available yet; SM 110 is conditionally appended after find_package(CUDAToolkit) below.
|
||||
if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
|
||||
set(CMAKE_CUDA_ARCHITECTURES 75 80 86 89 90 100 120)
|
||||
set(_TRT_CUDA_ARCHS_DEFAULTED TRUE)
|
||||
endif()
|
||||
|
||||
project(TensorRT
|
||||
LANGUAGES CXX CUDA
|
||||
VERSION ${TRT_VERSION}
|
||||
DESCRIPTION "TensorRT is a C++ library that facilitates high-performance inference on NVIDIA GPUs and deep learning accelerators."
|
||||
HOMEPAGE_URL "https://github.com/NVIDIA/TensorRT")
|
||||
|
||||
if (WIN32)
|
||||
enable_language(C)
|
||||
endif()
|
||||
|
||||
# Speed up rebuilds with ccache when available; respect a user-set launcher.
|
||||
if(CMAKE_GENERATOR MATCHES "Makefiles|Ninja")
|
||||
find_program(CCACHE_PROGRAM ccache)
|
||||
mark_as_advanced(CCACHE_PROGRAM)
|
||||
if(CCACHE_PROGRAM)
|
||||
foreach(lang IN ITEMS C CXX CUDA)
|
||||
set(CMAKE_${lang}_COMPILER_LAUNCHER "${CCACHE_PROGRAM}" CACHE STRING "Path to ccache")
|
||||
endforeach()
|
||||
message(STATUS "ccache enabled: ${CCACHE_PROGRAM}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
|
||||
set(CMAKE_INSTALL_PREFIX ${TRT_LIB_DIR}/../ CACHE PATH "TensorRT installation" FORCE)
|
||||
endif(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
|
||||
|
||||
option(BUILD_PLUGINS "Build TensorRT plugin" ON)
|
||||
option(BUILD_PARSERS "Build TensorRT parsers" ON)
|
||||
option(BUILD_SAMPLES "Build TensorRT samples" ON)
|
||||
option(BUILD_SAFE_SAMPLES "Build TensorRT safety samples" OFF)
|
||||
option(TRT_SAFETY_INFERENCE_ONLY "Build only the safety inference components (no safety builders)" OFF)
|
||||
option(TRT_BUILD_TESTING "Build gtests for TensorRT components" OFF)
|
||||
|
||||
# Must be set before add_subdirectory(plugin); the option() below is gated on BUILD_SAMPLES.
|
||||
set(TRT_BUILD_WINML OFF)
|
||||
|
||||
############################################################################################
|
||||
# Early dependency discovery
|
||||
# These must be found before they are used in target definitions
|
||||
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
# QNX has built-in threading support and doesn't need FindThreads
|
||||
if(NOT CMAKE_SYSTEM_NAME STREQUAL "QNX")
|
||||
find_package(Threads REQUIRED)
|
||||
else()
|
||||
# For QNX, create a dummy Threads::Threads target if it doesn't exist
|
||||
if(NOT TARGET Threads::Threads)
|
||||
add_library(Threads::Threads INTERFACE IMPORTED GLOBAL)
|
||||
# QNX threading is built into libc, no explicit linking needed
|
||||
endif()
|
||||
endif()
|
||||
|
||||
find_package(CUDAToolkit REQUIRED)
|
||||
message(STATUS "CUDA version: ${CUDAToolkit_VERSION}")
|
||||
|
||||
# SM 110 (Thor) requires CUDA 13.0+.
|
||||
# CUDAToolkit_VERSION is only available once the toolkit is found.
|
||||
if (_TRT_CUDA_ARCHS_DEFAULTED AND CUDAToolkit_VERSION VERSION_GREATER_EQUAL "13.0")
|
||||
list(APPEND CMAKE_CUDA_ARCHITECTURES 110)
|
||||
endif()
|
||||
message(STATUS "Generating CUDA code for SMs: ${CMAKE_CUDA_ARCHITECTURES}")
|
||||
|
||||
############################################################################################
|
||||
# OSS bridging: top-level setup for plugin and samples code.
|
||||
|
||||
# OSS vendors third-party headers under third_party/.
|
||||
set(TRT_EXTERNALS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party)
|
||||
|
||||
if(NOT TARGET TRT::cudart)
|
||||
add_library(TRT::cudart INTERFACE IMPORTED)
|
||||
target_link_libraries(TRT::cudart INTERFACE CUDA::cudart_static)
|
||||
endif()
|
||||
|
||||
include(BundleLibraries)
|
||||
include(SymbolExports)
|
||||
include(Stubify)
|
||||
include(WindowsLibSuffixes)
|
||||
include(StaticLibSmokeTest)
|
||||
|
||||
############################################################################################
|
||||
# Safety runtime libraries (libnvinfer_safe) used by safety samples and
|
||||
# inference-only builds.
|
||||
if(BUILD_SAFE_SAMPLES OR TRT_SAFETY_INFERENCE_ONLY)
|
||||
set(TRT_NVINFER_SAFE_NAME "nvinfer_safe")
|
||||
|
||||
# Shared safety runtime.
|
||||
find_library(nvinfer_safe_path
|
||||
${TRT_NVINFER_SAFE_NAME}
|
||||
PATHS ${TRT_LIB_DIR}
|
||||
NO_CMAKE_FIND_ROOT_PATH
|
||||
)
|
||||
if(NOT nvinfer_safe_path)
|
||||
message(FATAL_ERROR "nvinfer_safe library not found. Please ensure safety runtime libraries are available in TRT_LIB_DIR ('${TRT_LIB_DIR}').")
|
||||
endif()
|
||||
add_library(TRTSAFE::nvinfer_safe_shared SHARED IMPORTED)
|
||||
set_target_properties(TRTSAFE::nvinfer_safe_shared PROPERTIES IMPORTED_LOCATION ${nvinfer_safe_path})
|
||||
|
||||
target_link_libraries(TRTSAFE::nvinfer_safe_shared INTERFACE cuda) # nvinfer_safe needs the cuda driver.
|
||||
|
||||
# Debug runtime library (provides debugging features like tensor dumping).
|
||||
set(nvinfer_safe_debug_lib_name "${TRT_NVINFER_SAFE_NAME}_debug")
|
||||
find_library(nvinfer_safe_debug_path
|
||||
${nvinfer_safe_debug_lib_name}
|
||||
PATHS ${TRT_LIB_DIR}
|
||||
NO_CMAKE_FIND_ROOT_PATH
|
||||
)
|
||||
if(NOT nvinfer_safe_debug_path)
|
||||
message(FATAL_ERROR "nvinfer_safe_debug library not found. Please ensure debug runtime library is available in TRT_LIB_DIR.")
|
||||
endif()
|
||||
add_library(TRTSAFE::nvinfer_safe_debug SHARED IMPORTED)
|
||||
set_target_properties(TRTSAFE::nvinfer_safe_debug PROPERTIES IMPORTED_LOCATION ${nvinfer_safe_debug_path})
|
||||
|
||||
# Headers for the safety runtime.
|
||||
# Try to find include directory relative to lib dir first, then fall back to standard locations
|
||||
if(EXISTS "${TRT_LIB_DIR}/../include/NvInfer.h")
|
||||
target_include_directories(TRTSAFE::nvinfer_safe_shared INTERFACE ${TRT_LIB_DIR}/../include)
|
||||
target_include_directories(TRTSAFE::nvinfer_safe_debug INTERFACE ${TRT_LIB_DIR}/../include)
|
||||
elseif(EXISTS "/usr/include/NvInfer.h")
|
||||
target_include_directories(TRTSAFE::nvinfer_safe_shared INTERFACE /usr/include)
|
||||
target_include_directories(TRTSAFE::nvinfer_safe_debug INTERFACE /usr/include)
|
||||
elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/include/NvInfer.h")
|
||||
target_include_directories(TRTSAFE::nvinfer_safe_shared INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include)
|
||||
target_include_directories(TRTSAFE::nvinfer_safe_debug INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include)
|
||||
else()
|
||||
message(WARNING "Could not find TensorRT headers. Please ensure they are installed.")
|
||||
endif()
|
||||
|
||||
# On QNX, TRT depends on DLA symbols stored in the DriveOS PDK.
|
||||
# Since trying to find these shared libs at link time will be difficult, we ignore unresolved symbols in shared libs.
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "QNX")
|
||||
target_link_options(TRTSAFE::nvinfer_safe_shared INTERFACE LINKER:--unresolved-symbols=ignore-in-shared-libs)
|
||||
target_link_options(TRTSAFE::nvinfer_safe_debug INTERFACE LINKER:--unresolved-symbols=ignore-in-shared-libs)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# OSS safety inference-only mode: require safety samples and disable enterprise
|
||||
# components.
|
||||
if(TRT_SAFETY_INFERENCE_ONLY)
|
||||
if(NOT BUILD_SAFE_SAMPLES)
|
||||
set(BUILD_SAFE_SAMPLES ON CACHE BOOL "Build TensorRT safety samples" FORCE)
|
||||
endif()
|
||||
|
||||
set(TRT_SAFETY_INFERENCE_ONLY ON CACHE BOOL "" FORCE)
|
||||
|
||||
# Disable enterprise OSS components for this configuration.
|
||||
set(BUILD_PLUGINS OFF CACHE BOOL "" FORCE)
|
||||
set(BUILD_PARSERS OFF CACHE BOOL "" FORCE)
|
||||
set(BUILD_SAMPLES OFF CACHE BOOL "" FORCE)
|
||||
|
||||
# Add CUDA library directory early so all samples can find it
|
||||
if(CUDAToolkit_LIBRARY_DIR)
|
||||
link_directories(${CUDAToolkit_LIBRARY_DIR})
|
||||
endif()
|
||||
|
||||
# Interface target for safety samples in inference-only mode.
|
||||
add_library(trt_global_definitions INTERFACE)
|
||||
|
||||
target_link_libraries(trt_global_definitions INTERFACE
|
||||
TRTSAFE::nvinfer_safe_shared
|
||||
cudart
|
||||
Threads::Threads
|
||||
)
|
||||
|
||||
if(NOT WIN32 AND NOT CMAKE_SYSTEM_NAME STREQUAL "QNX")
|
||||
target_link_libraries(trt_global_definitions INTERFACE dl rt)
|
||||
endif()
|
||||
target_include_directories(trt_global_definitions INTERFACE
|
||||
${TRT_INCLUDE_DIR}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/samples/common
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/shared
|
||||
${CUDAToolkit_INCLUDE_DIRS}
|
||||
)
|
||||
target_compile_options(trt_global_definitions INTERFACE
|
||||
$<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
|
||||
)
|
||||
endif()
|
||||
|
||||
# C++17
|
||||
set(CMAKE_CXX_STANDARD 20)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
|
||||
if(NOT MSVC)
|
||||
set(CMAKE_CXX_FLAGS "-Wno-deprecated-declarations ${CMAKE_CXX_FLAGS} -DBUILD_SYSTEM=cmake_oss")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBUILD_SYSTEM=cmake_oss")
|
||||
endif()
|
||||
|
||||
############################################################################################
|
||||
# Cross-compilation settings
|
||||
|
||||
set_ifndef(TRT_PLATFORM_ID "x86_64")
|
||||
message(STATUS "Targeting TRT Platform: ${TRT_PLATFORM_ID}")
|
||||
|
||||
############################################################################################
|
||||
# Debug settings
|
||||
|
||||
set(TRT_DEBUG_POSTFIX _debug CACHE STRING "suffix for debug builds")
|
||||
|
||||
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
message("Building in debug mode ${DEBUG_POSTFIX}")
|
||||
endif()
|
||||
|
||||
############################################################################################
|
||||
# Dependencies
|
||||
set(DEFAULT_CUDNN_VERSION 8.9)
|
||||
set(DEFAULT_PROTOBUF_VERSION 3.20.3)
|
||||
|
||||
# Dependency Version Resolution
|
||||
set_ifndef(CUDNN_VERSION ${DEFAULT_CUDNN_VERSION})
|
||||
message(STATUS "cuDNN version set to ${CUDNN_VERSION}")
|
||||
set_ifndef(PROTOBUF_VERSION ${DEFAULT_PROTOBUF_VERSION})
|
||||
message(STATUS "Protobuf version set to ${PROTOBUF_VERSION}")
|
||||
|
||||
if (BUILD_PLUGINS OR BUILD_PARSERS)
|
||||
include(third_party/protobuf.cmake)
|
||||
endif()
|
||||
if(BUILD_PARSERS)
|
||||
configure_protobuf(${PROTOBUF_VERSION})
|
||||
endif()
|
||||
|
||||
# Define library names
|
||||
set(TRT_NVINFER_NAME "nvinfer")
|
||||
set(TRT_ONNXPARSER_NAME "nvonnxparser")
|
||||
|
||||
# Windows library names have major version appended.
|
||||
if (MSVC)
|
||||
set(nvinfer_lib_name "${TRT_NVINFER_NAME}_${TRT_SOVERSION}${TRT_LIB_SUFFIX}")
|
||||
set(nvinfer_plugin_lib_name "nvinfer_plugin_${TRT_SOVERSION}")
|
||||
set(nvinfer_vc_plugin_lib_name "nvinfer_vc_plugin_${TRT_SOVERSION}")
|
||||
set(nvonnxparser_lib_name "${TRT_ONNXPARSER_NAME}_${TRT_SOVERSION}${TRT_LIB_SUFFIX}")
|
||||
|
||||
else()
|
||||
set(nvinfer_lib_name ${TRT_NVINFER_NAME})
|
||||
set(nvinfer_plugin_lib_name "nvinfer_plugin")
|
||||
set(nvinfer_vc_plugin_lib_name "nvinfer_vc_plugin")
|
||||
set(nvonnxparser_lib_name ${TRT_ONNXPARSER_NAME})
|
||||
endif()
|
||||
|
||||
find_library_create_target(nvinfer ${nvinfer_lib_name} SHARED "${TRT_LIB_DIR}")
|
||||
set_property(TARGET nvinfer PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${TRT_INCLUDE_DIR})
|
||||
|
||||
# tensorrt is aliased downstream; CMake forbids aliasing an alias.
|
||||
add_library(tensorrt INTERFACE IMPORTED)
|
||||
target_link_libraries(tensorrt INTERFACE nvinfer)
|
||||
|
||||
set(CMAKE_CUDA_RUNTIME_LIBRARY "static" CACHE STRING "")
|
||||
|
||||
if (NOT MSVC)
|
||||
find_library(RT_LIB rt)
|
||||
endif()
|
||||
|
||||
############################################################################################
|
||||
|
||||
if(NOT MSVC)
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr -Xcompiler -Wno-deprecated-declarations")
|
||||
else()
|
||||
set(CMAKE_CUDA_SEPARABLE_COMPILATION ON)
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr -Xcompiler")
|
||||
endif()
|
||||
|
||||
############################################################################################
|
||||
# TensorRT
|
||||
|
||||
set(HINT_PATHS "${TRT_OUT_DIR}" "${TRT_LIB_DIR}")
|
||||
|
||||
if(NOT TARGET trt_global_definitions)
|
||||
add_library(trt_global_definitions INTERFACE)
|
||||
target_include_directories(trt_global_definitions INTERFACE ${CUDAToolkit_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
if(BUILD_PLUGINS)
|
||||
option(TRT_BUILD_ENABLE_DLA "Build TensorRT with DLA features enabled." OFF)
|
||||
set(TRT_BUILD_ENABLE_STATIC_LIBS OFF CACHE INTERNAL "Static libs are no longer supported")
|
||||
option(TRT_BUILD_STUB_LIBS "Build linker stub libraries." OFF)
|
||||
|
||||
add_subdirectory(plugin)
|
||||
else()
|
||||
find_library_create_target(${nvinfer_plugin_lib_name} ${nvinfer_plugin_lib_name} SHARED "${HINT_PATHS}")
|
||||
endif()
|
||||
|
||||
if(BUILD_PARSERS)
|
||||
add_subdirectory(parsers)
|
||||
else()
|
||||
find_library_create_target(${nvonnxparser_lib_name} ${nvonnxparser_lib_name} SHARED "${HINT_PATHS}")
|
||||
endif()
|
||||
|
||||
add_library(tensorrt_headers INTERFACE)
|
||||
target_include_directories(tensorrt_headers INTERFACE ${TRT_INCLUDE_DIR})
|
||||
|
||||
# Samples
|
||||
if(BUILD_SAMPLES OR BUILD_SAFE_SAMPLES)
|
||||
set(TRT_BUILD_ENABLE_NEW_SAMPLES_FLOW ON)
|
||||
|
||||
# Map OSS option names to the internal names used by samples/CMakeLists.txt.
|
||||
set(TRT_BUILD_SAMPLES ${BUILD_SAMPLES})
|
||||
set(TRT_BUILD_TRTEXEC ${BUILD_SAMPLES})
|
||||
set(TRT_BUILD_ONNX_PARSER ${BUILD_PARSERS})
|
||||
set(TRT_BUILD_PLUGINS ${BUILD_PLUGINS})
|
||||
|
||||
# Set defaults for specific feature enablement for samples.
|
||||
option(TRT_BUILD_ENABLE_DLA "Build TensorRT with DLA features enabled." OFF)
|
||||
option(TRT_BUILD_ENABLE_UNIFIED_BUILDER "Build TensorRT with unified builder (safety) features enabled." ${BUILD_SAFE_SAMPLES})
|
||||
option(TRT_BUILD_WINML "Build TensorRT with WinML support." OFF)
|
||||
option(TRT_BUILD_ENABLE_MULTIDEVICE "Build TensorRT with multi-device support." OFF)
|
||||
set(TRT_BUILD_SAMPLES_LINK_STATIC_TRT OFF CACHE INTERNAL "")
|
||||
|
||||
target_include_directories(${nvonnxparser_lib_name} INTERFACE ${TRT_INCLUDE_DIR})
|
||||
|
||||
add_subdirectory(shared)
|
||||
|
||||
include(InstallUtils)
|
||||
|
||||
if (TRT_BUILD_TESTING)
|
||||
find_package(GTest QUIET)
|
||||
if (GTest_FOUND)
|
||||
if (NOT TARGET gtest_main)
|
||||
add_library(gtest_main ALIAS GTest::gtest_main)
|
||||
endif()
|
||||
else()
|
||||
include(FetchContent)
|
||||
FetchContent_Declare(
|
||||
googletest
|
||||
GIT_REPOSITORY https://github.com/google/googletest.git
|
||||
GIT_TAG v1.14.0
|
||||
)
|
||||
FetchContent_MakeAvailable(googletest)
|
||||
endif()
|
||||
set(TRT_GTEST_DISCOVERY_MODE PRE_TEST CACHE STRING "gtest discovery mode.")
|
||||
endif()
|
||||
|
||||
add_subdirectory(samples)
|
||||
endif()
|
||||
@@ -0,0 +1,436 @@
|
||||
|
||||
## TensorRT C++ Coding Guidelines
|
||||
|
||||
The TensorRT C++ Coding Guidelines are derived from several sources, primarily:
|
||||
|
||||
- [AUTOSAR C++ 2014](https://www.autosar.org/fileadmin/user_upload/standards/adaptive/17-03/AUTOSAR_RS_CPP14Guidelines.pdf)
|
||||
- [MISRA C++ 2008](https://www.misra.org.uk/Activities/MISRAC/tabid/171/Default.aspx)
|
||||
- [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html)
|
||||
|
||||
------
|
||||
|
||||
#### Namespaces
|
||||
|
||||
1. *MISRA C++: 2008 Rule 7-3-1*
|
||||
Global namespace shall only contain main, namespace declarations and extern "C" declarations. Use explicit or anon namespaces for everything else.
|
||||
2. Closing braces of namespaces should have a comment saying the namespace it closes:
|
||||
```cpp
|
||||
namespace foo
|
||||
{
|
||||
...
|
||||
} // namespace foo
|
||||
```
|
||||
|
||||
#### Constants
|
||||
1. Prefer `const` or `constexpr` variables over `#defines` whenever possible, as the latter are not visible to the compiler.
|
||||
2. *MISRA C++: 2008 Rule 7-1-1 and 7-1-2*
|
||||
A variable that is not modified after its initialization should be declared as `const`.
|
||||
3. For naming of constants, see the Naming section of this document.
|
||||
|
||||
|
||||
|
||||
#### Literals
|
||||
1. Except `0` (only used in comparison for checking signness/existence/emptiness) and `nullptr`, `true`, `false`, all other literals should only be used for variable initialization.
|
||||
Example:
|
||||
```cpp
|
||||
if (nbInputs == 2U){/*...*/}
|
||||
```
|
||||
Should be changed to:
|
||||
```cpp
|
||||
constexpr size_t kNbInputsWBias = 2U;
|
||||
if (nbInputs == kNbInputsWBias) {/*...*/}
|
||||
```
|
||||
|
||||
|
||||
#### Brace Notation
|
||||
1. Use the [Allman indentation](https://en.wikipedia.org/wiki/Indent_style#Allman_style) style.
|
||||
2. Put the semicolon for an empty `for` or `while` loop in a new line.
|
||||
3. *AUTOSAR C++14 Rule 6.6.3*, *MISRA C++: 2008 6-3-1*
|
||||
The statement forming the body of a `switch`, `while`, `do .. while` or `for` statement shall be a compound statement. (use brace-delimited statements)
|
||||
4. *AUTOSAR C++14 Rule 6.6.4*, *MISRA C++: 2008 Rule 6-4-1*
|
||||
`If` and `else` should always be followed by brace-delimited statements, even if empty or a single statement.
|
||||
|
||||
|
||||
#### Naming
|
||||
1. Filenames
|
||||
* Camel case with first letter lowercase: `thisIsASubDir` and `thisIsAFilename.cpp`
|
||||
* *NOTE*: All files involved in the compilation of a compilation target (.exe/.so) must have filenames that are case-insensitive unique.
|
||||
|
||||
2. Types
|
||||
* All types (including, but not limited to, class names) are [camel case](https://en.wikipedia.org/wiki/Camel_case) with uppercase first letter. Example: `FooBarClass`
|
||||
|
||||
3. Local variables, methods and namespaces
|
||||
* Camel case with first letter lowercase. Example: `localFooBar`
|
||||
|
||||
4. Non-magic-number global variables that are non-static and not defined in anonymous namespace
|
||||
* Camel case prefixed by a lower case 'g'. Example: `gDontUseGlobalFoos`
|
||||
|
||||
5. Non-magic-number global variables that are static or defined in an anonymous namespace
|
||||
* Camel case prefixed by a lower case 's'. Example: `sMutableStaticGlobal`
|
||||
|
||||
6. Locally visible static variable
|
||||
* Camel case with lowercase prefix ''s" as the first letter of the name. Example: `static std::once_flag sCaskInitOnce;`
|
||||
|
||||
7. Public, private and protected class member variables
|
||||
* Camelcase prefixed with an 'm': `mNbFooValues`.
|
||||
* Public member variables do not require the 'm' prefix but it is highly encouraged to use the prefix when needed to improve code clarity, especially in cases where the class is a base class in an inheritance chain.
|
||||
|
||||
8. Constants
|
||||
* Enumerations, global constants, static constants at class-scope and function-scope magic-number/literal constants are uppercase snakecase with prefix 'k':
|
||||
```cpp
|
||||
const int kDIGIT_NUM = 10;
|
||||
```
|
||||
> *NOTE*: Function-scope constants that are not magic numbers or literals are named like non-constant variables:
|
||||
```cpp
|
||||
const bool pass = a && b;
|
||||
```
|
||||
|
||||
9. Macros
|
||||
* See [Constants](CODING-GUIDELINES.md#constants), which are preferred over `#define`.
|
||||
* If you must use macros, however, follow uppercase snakecase: `FOO_VERSION`
|
||||
|
||||
Notes:
|
||||
* In general we don't use [hungarian notation](https://en.wikipedia.org/wiki/Hungarian_notation), except for 'apps hungarian' in some cases such as 'nb' in a variable name to indicate count: `mNbTensorDescriptors`
|
||||
* If a constructor's parameter name `foo` conflicts with a public member name `foo`, add a trailing underscore to the parameter name: `foo_`.
|
||||
* *MISRA C++: 2008 Rule 2-13-4*
|
||||
Literal suffixes should be upper case. For example, use `1234L` instead of `1234l`.
|
||||
|
||||
|
||||
#### Tabs vs Spaces
|
||||
1. Use only spaces. Do not use tabs.
|
||||
2. Indent 4 spaces at a time. This is enforced automatically if you format your code using our clang-format config.
|
||||
|
||||
|
||||
#### Formatting
|
||||
1. Use the [LLVM clang-format](https://clang.llvm.org/docs/ClangFormat.html) tool for formatting your changes prior to submitting the PR.
|
||||
2. Use a maximum of 120 characters per line. The auto formatting tool will wrap longer lines.
|
||||
3. Exceptions to formatting violations must be justified on a per-case basis. Bypassing the formatting rules is discouraged, but can be achieved for exceptions as follows:
|
||||
```cpp
|
||||
// clang-format off
|
||||
// .. Unformatted code ..
|
||||
// clang-format on
|
||||
```
|
||||
|
||||
|
||||
#### Pointers and Memory Allocation
|
||||
1. *AUTOSAR C++ 2014: 18-5-2/3*
|
||||
Use smart pointers for allocating objects on the heap.
|
||||
2. When picking a smart pointer, prefer `unique_ptr` for single resource ownership and `shared_ptr` for shared resource ownership. Use `weak_ptr` only in exceptional cases.
|
||||
3. Do not use smart pointers that have been deprecated in C++11.
|
||||
|
||||
|
||||
#### Comments
|
||||
1. C++ comments are required. C comments are not allowed except for special cases (inline).
|
||||
2. C++ style for single-line comments. `// This is a single line comment`
|
||||
3. In function calls where parameters are not obvious from inspection, it can be helpful to use an inline C comment to document the parameter for readers:
|
||||
```cpp
|
||||
doSomeOperation(/* checkForErrors = */ false);
|
||||
```
|
||||
4. If the comment is a full sentence, it should be capitalized i.e. start with capital letter and punctuated properly.
|
||||
5. Follow [Doxygen rules](http://www.doxygen.nl/manual/docblocks.html) for documenting new class interfaces and function prototypes.
|
||||
* For C++-style single-line comments use `//!`.
|
||||
* For class members, use `//!<`.
|
||||
```cpp
|
||||
//! This is a Doxygen comment
|
||||
//! in C++ style
|
||||
|
||||
struct Foo
|
||||
{
|
||||
int x; //!< This is a Doxygen comment for members
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
#### Disabling Code
|
||||
1. Use `#if` / `#endif` to disable code, preferably with a mnemonic condition like this:
|
||||
```cpp
|
||||
#if DEBUG_CONVOLUTION_INSTRUMENTATION
|
||||
// ...code to be disabled...
|
||||
#endif
|
||||
```
|
||||
|
||||
```cpp
|
||||
// Alternative: use a macro which evaluates to a noop in release code.
|
||||
#if DEBUG_CONVOLUTION_INSTRUMENTATION
|
||||
# define DEBUG_CONV_CODE(x) x
|
||||
#else
|
||||
# define DEBUG_CONV_CODE(x)
|
||||
#endif
|
||||
```
|
||||
|
||||
2. *MISRA C++: 2008 Rule 0-1-9*, *AutoSAR C++ 2014: 6-0-1*
|
||||
Dead code is forbidden in safety-critical software - you may not use compile-time expressions and DCE to disable code. However, this technique can be useful elsewhere (e.g. tools, tests) to help prevent bitrot.
|
||||
|
||||
```cpp
|
||||
// Not allowed in safety-critical code.
|
||||
const bool gDisabledFeature = false;
|
||||
|
||||
void foo()
|
||||
{
|
||||
if (gDisabledFeature)
|
||||
{
|
||||
doSomething();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. *MISRA C++: 2008 Rule 2-7-2 and 2-7-3*
|
||||
Do NOT use comments to disable code. Use comments to explain code, not hide it.
|
||||
|
||||
|
||||
#### Exceptions
|
||||
1. Exceptions must not be thrown across library boundaries.
|
||||
|
||||
|
||||
#### Casts
|
||||
1. Use the least forceful cast necessary, or no cast if possible, to help the compiler diagnose unintended consequences.
|
||||
2. Casting a pointer to a `void*` should be implicit (except if removing `const`).
|
||||
3. *MISRA C++: 2008 Rule 5-2-5*
|
||||
Casting should not remove any `const` or `volatile` qualification from the type of a pointer or reference.
|
||||
4. *MISRA C++: 2008 Rule 5-2-4*
|
||||
Do not use C-style casts (other than void casts) and functional notation casts (other than explicit constructor calls).
|
||||
6. Casting from a `void*` to a `T*` should be done with `static_cast`, not `reinterpret_cast`, since the latter is more forceful.
|
||||
7. Use `reinterpret_cast` as a last resort, where `const_cast` and `static_cast` won't work.
|
||||
8. Avoid `dynamic_cast`.
|
||||
|
||||
|
||||
#### Expressions
|
||||
1. *MISRA C++: 2008 Rule 6-2-1*
|
||||
Do not use assignment operator in subexpressions.
|
||||
```cpp
|
||||
// Not compliant
|
||||
x = y = z;
|
||||
|
||||
// Not compliant
|
||||
if (x = y)
|
||||
{
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
#### Ternary operator
|
||||
1. *AUTOSAR C++ 2014: 7-1-1*
|
||||
Ternary operator should not be used as a sub-expression. Ternary operator expressions should be encapsulated with braces. Example:
|
||||
```cpp
|
||||
const auto var = (condition0 ? a : (condition1 ? b : c));
|
||||
```
|
||||
should be changed to:
|
||||
```cpp
|
||||
const auto d = (condition1 ? b : c);
|
||||
const auto var = (condition0 ? a : d);
|
||||
```
|
||||
|
||||
|
||||
#### Statements
|
||||
1. When practical, a `switch` statement controlled by an `enum` should have a case for each enum value and not have a default clause so that we get a compile-time error if a new enum value is added.
|
||||
2. *MISRA C++:2008 Rules 6-4-3, 6-4-4, and 6-4-5*
|
||||
Switch statements should be well structured. An informal guideline is to treat switch statements as structured multi-way branches and not "glorified gotos" such as:
|
||||
```cpp
|
||||
// Not compliant
|
||||
switch (x) case 4: if (y) case 5: return 0; else default: return 1;
|
||||
```
|
||||
3. The "well structured" requirement prohibits fall-though except from one case label to another. Each case clause must be terminated in a break or throw. If a case clause has multiple statements, the braces are optional. The following example illustrates these requirements:
|
||||
```cpp
|
||||
switch (x)
|
||||
{
|
||||
case 0: // Fall-through allowed from case 0: to case 1: since case 0 is empty.
|
||||
case 1:
|
||||
a();
|
||||
b();
|
||||
break;
|
||||
case 2:
|
||||
case 4:
|
||||
{ // With optional braces
|
||||
c();
|
||||
d();
|
||||
break;
|
||||
}
|
||||
case 5:
|
||||
c();
|
||||
throw 42; // Terminating with throw is okay
|
||||
default:
|
||||
throw 42;
|
||||
}
|
||||
```
|
||||
|
||||
4. *MISRA C++:2008 Rule 6-4-3*
|
||||
Ending a case clause with return is not allowed.
|
||||
5. If a switch clause is a compound statement, put the break inside the braces.
|
||||
```cpp
|
||||
switch (x)
|
||||
{
|
||||
case 0:
|
||||
case 1:
|
||||
{
|
||||
y();
|
||||
z();
|
||||
break;
|
||||
}
|
||||
...other cases...
|
||||
}
|
||||
```
|
||||
|
||||
#### Functions
|
||||
1. Avoid declaring large functions as `inline`, absent a quantifiable benefit. Remember that functions defined in class declarations are implicitly inline.
|
||||
2. Rather than using the `static` keyword to mark a function as having internal linkage, prefer to use anonymous namespaces instead.
|
||||
3. *MISRA C++:2008 Rule 0-1-10*
|
||||
Every defined function must be called at least once. That is, do not have unused methods.
|
||||
4. *MISRA C++:2008 Rule 8-4-2*
|
||||
Parameter names should be consistent across function definition and corresponding function declarations.
|
||||
|
||||
|
||||
#### Forward declarations and extern variables
|
||||
|
||||
1. *MISRA C++: 2008 Rule 3-2-3*
|
||||
For safety critical code, a type, object or function that is used in multiple translation units shall be declared in one and only one file.
|
||||
* This means we cannot forward declare incomplete types in files where they are needed. Instead, we should put forward declarations in header files, and include these header files as needed.
|
||||
|
||||
|
||||
#### Structures and Classes
|
||||
1. *MISRA C++: 2008 Rule 14-7-1*
|
||||
All class templates, function templates, class template member functions and class template static members shall be instantiated at least once. This prevents use of uninitialized variables.
|
||||
2. *MISRA C++: 2008 Rule 11-01*
|
||||
If class is not a *Plain Old Data Structure*, then its data members should be private.
|
||||
|
||||
|
||||
#### Preprocessor Directives
|
||||
1. *MISRA C++: 2008 Rule 16-0-2*
|
||||
`#define` and `#undef` of macros should be done only at global namespace.
|
||||
2. Avoid the use of `#ifdef` and `#ifndef` directives (except in the case of header include guards). Prefer to use `#if defined(...)` or `#if !defined(...)` instead. The latter syntax is more consistent with C syntax, and allows you to use more complicated preprocessor conditionals, e.g.:
|
||||
```cpp
|
||||
#if defined(FOO) || defined(BAR)
|
||||
void foo();
|
||||
#endif // defined(FOO) || defined(BAR)
|
||||
```
|
||||
|
||||
3. When nesting preprocessor directives, use indentation after the hash mark (#). For example:
|
||||
```cpp
|
||||
#if defined(FOO)
|
||||
# if FOO == 0
|
||||
# define BAR 0
|
||||
# elif FOO == 1
|
||||
# define BAR 5
|
||||
# else
|
||||
# error "invalid FOO value"
|
||||
# endif
|
||||
#endif
|
||||
```
|
||||
|
||||
4. Do not use `#pragma` once as include guard.
|
||||
5. Use a preprocessor guard. It's standard-conforming and modern compilers are smart enough to open the file only once.
|
||||
* The guard name must have prefix `TRT_` followed by the filename, all in caps. For a header file named `FooBarHello.h`, name the symbol as `TRT_FOO_BAR_HELLO_H`.
|
||||
* Only use the file name to create the symbol. Unlike the Google C++ guideline, we do not include the directory names in the symbol. This is because we ensure all filenames are unique in the compilation unit.
|
||||
* Do not use prefix with underscore. Such symbols are reserved in C++ standard for compilers or implementation.
|
||||
* Do not use trailing underscore for the symbol. We differ in this from Google C++ guideline, which uses trailing underscore: `TRT_FOO_BAR_HELLO_H_`
|
||||
```cpp
|
||||
#ifndef TRT_FOO_BAR_HELLO_H
|
||||
#define TRT_FOO_BAR_HELLO_H
|
||||
// ...
|
||||
#endif // TRT_FOO_BAR_HELLO_H
|
||||
```
|
||||
|
||||
6. *AUTOSAR C++ 2014: 7-1-6*
|
||||
Use `using` instead of `typedef`.
|
||||
|
||||
|
||||
#### Signed vs Unsigned Integers
|
||||
1. Use signed integers instead of unsigned, except for the cases below.
|
||||
* The integer is a bitmap - use an unsigned type, since sign extension could lead to surprises.
|
||||
* The integer is being used with an external library that expects an unsigned integer. A common example is a loop that compares against `std::vector::size()`, such as:
|
||||
```cpp
|
||||
for (size_t i = 0; i < mTensors.size(); ++i) // preferred style
|
||||
```
|
||||
* Using only signed integers for the above would lead to prolixity and perhaps unsafe narrowing:
|
||||
```cpp
|
||||
for (int i = 0; i < static_cast<int>(mTensors.size()); ++i)
|
||||
```
|
||||
|
||||
|
||||
#### Special Considerations for API
|
||||
1. The API consists, with very few exceptions, of methodless structs and pure virtual interface classes.
|
||||
2. API class methods should be either virtual or inline.
|
||||
3. The API does not use integral types with platform-dependent sizes, other than `int`, `unsigned`, and `bool`. `size_t` should be used only for sizes of memory buffers.
|
||||
4. The API does not use any aggregate types (e.g. `std::string`) which may be compiled differently with different compilers and libraries.
|
||||
5. The API minimizes dependencies on system headers - currently only `<cstddef>` and `<cstdint>`.
|
||||
6. Memory ownership may not be transferred across API boundaries - any memory allocated inside a library must be freed inside the library.
|
||||
7. The API should be C++03.
|
||||
8. New methods should be added at the end of interfaces so as to preserve v-table compatibility (compilers don't guarantee this, but de facto it works.)
|
||||
9. Avoid optional arguments to functions, since they can make it difficult to extend interfaces.
|
||||
10. Do not throw exceptions across library boundaries.
|
||||
11. Document all APIs with doxygen.
|
||||
|
||||
|
||||
#### Common Pitfalls
|
||||
|
||||
1. C headers should not be used directly.
|
||||
- Example: Use `<cstdint>` instead of `<stdint.h>`
|
||||
2. Do not use C library functions, whenever possible.
|
||||
* Use brace initialization or `std::fill_n()` instead of `memset()`. This is especially important when dealing with non-[POD types](http://en.cppreference.com/w/cpp/concept/PODType). In the example below, using `memset()` will corrupt the vtable of `Foo:`
|
||||
```cpp
|
||||
struct Foo {
|
||||
virtual int getX() { return x; }
|
||||
int x;
|
||||
};
|
||||
...
|
||||
|
||||
// Bad: use memset() to initialize Foo
|
||||
{
|
||||
Foo foo;
|
||||
memset(&foo, 0, sizeof(foo)); // Destroys hiddien virtual-function-table pointer!
|
||||
}
|
||||
// Good: use brace initialization to initialize Foo
|
||||
{
|
||||
Foo foo = {};
|
||||
}
|
||||
```
|
||||
|
||||
2. When specifying pointers to `const` data, the pointer itself may be `const`, in some usecases.
|
||||
```cpp
|
||||
char const * const errStr = getErrorStr(status);
|
||||
```
|
||||
|
||||
----
|
||||
|
||||
## Appendix
|
||||
|
||||
#### Abbreviation Words and Compound Words as Part of Names
|
||||
|
||||
* Abbreviation words, which are usually fully-capitalized in literature, are treated as normal words without special capitalization, e.g. `gpuAllocator`, where GPU is converted to `gpu` before constructing the camel case name.
|
||||
* Compound words, which are usually used in full in literature, e.g. `runtime`, can be abbreviated into fully capitalized letters, e.g. `RT` in NvInferRT.h.
|
||||
|
||||
#### Terminology
|
||||
|
||||
* *CUDA code* is code that must be compiled with a CUDA compiler. Typically, it includes:
|
||||
* Declaration or definition of global or static variables with one of the following CUDA keywords: `__device__`, `__managed__` and `__constant__`.
|
||||
* Declaration or definition of device functions decorated with `__device__`.
|
||||
* Declaration or definition of kernels decorated with `__global__`.
|
||||
* Kernel launching with <<<...>>> syntax.
|
||||
|
||||
> NOTE:
|
||||
* Definition of kernel function pointer type aliases is not device code, e.g. `typedef __global__ void(*KernelFunc)(void* /*arg*/);`.
|
||||
* Definition of pointers to kernel functions is not device code, either, e.g. `__global__ void(*KernelFunc)(void* /*arg*/) = getKernelFunc(parameters);` .
|
||||
* Kernel launching with the CUDA runtime/driver API's, e.g. `cuLaunch` and `cudaLaunch`, is not CUDA code.
|
||||
|
||||
----
|
||||
|
||||
## NVIDIA Copyright
|
||||
|
||||
1. All TensorRT Open Source Software code should contain an NVIDIA copyright header that includes the current year. The following block of text should be prepended to the top of all OSS files. This includes .cpp, .h, .cu, .py, and any other source files which are compiled or interpreted.
|
||||
```cpp
|
||||
/*
|
||||
* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
```
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
|
||||
## TensorRT OSS Contribution Rules
|
||||
|
||||
#### Issue Tracking
|
||||
|
||||
* All enhancement, bugfix, or change requests must begin with the creation of a [TensorRT Issue Request](https://github.com/nvidia/TensorRT/issues).
|
||||
* The issue request must be reviewed by TensorRT engineers and approved prior to code review.
|
||||
|
||||
|
||||
#### Coding Guidelines
|
||||
|
||||
- All source code contributions must strictly adhere to the [TensorRT Coding Guidelines](CODING-GUIDELINES.md).
|
||||
|
||||
- In addition, please follow the existing conventions in the relevant file, submodule, module, and project when you add new code or when you extend/fix existing functionality.
|
||||
|
||||
- To maintain consistency in code formatting and style, you should also run `clang-format` on the modified sources with the provided configuration file. This applies TensorRT code formatting rules to:
|
||||
- class, function/method, and variable/field naming
|
||||
- comment style
|
||||
- indentation
|
||||
- line length
|
||||
|
||||
- Format git changes:
|
||||
```bash
|
||||
# Commit ID is optional - if unspecified, run format on staged changes.
|
||||
git-clang-format --style file [commit ID/reference]
|
||||
```
|
||||
|
||||
- Format individual source files:
|
||||
```bash
|
||||
# -style=file : Obtain the formatting rules from .clang-format
|
||||
# -i : In-place modification of the processed file
|
||||
clang-format -style=file -i -fallback-style=none <file(s) to process>
|
||||
```
|
||||
|
||||
- Format entire codebase (for project maintainers only):
|
||||
```bash
|
||||
find samples plugin -iname *.h -o -iname *.c -o -iname *.cpp -o -iname *.hpp \
|
||||
| xargs clang-format -style=file -i -fallback-style=none
|
||||
```
|
||||
|
||||
- Avoid introducing unnecessary complexity into existing code so that maintainability and readability are preserved.
|
||||
|
||||
- Try to keep pull requests (PRs) as concise as possible:
|
||||
- Avoid committing commented-out code.
|
||||
- Wherever possible, each PR should address a single concern. If there are several otherwise-unrelated things that should be fixed to reach a desired endpoint, our recommendation is to open several PRs and indicate the dependencies in the description. The more complex the changes are in a single PR, the more time it will take to review those changes.
|
||||
|
||||
- Write commit titles using imperative mood and [these rules](https://chris.beams.io/posts/git-commit/), and reference the Issue number corresponding to the PR. Following is the recommended format for commit texts:
|
||||
```
|
||||
#<Issue Number> - <Commit Title>
|
||||
|
||||
<Commit Body>
|
||||
```
|
||||
|
||||
- Ensure that the build log is clean, meaning no warnings or errors should be present.
|
||||
|
||||
- Ensure that all `sample_*` tests pass prior to submitting your code.
|
||||
|
||||
- All OSS components must contain accompanying documentation (READMEs) describing the functionality, dependencies, and known issues.
|
||||
|
||||
- See `README.md` for existing samples and plugins for reference.
|
||||
|
||||
- All OSS components must have an accompanying test.
|
||||
|
||||
- If introducing a new component, such as a plugin, provide a test sample to verify the functionality.
|
||||
|
||||
- To add or disable functionality:
|
||||
- Add a CMake option with a default value that matches the existing behavior.
|
||||
- Where entire files can be included/excluded based on the value of this option, selectively include/exclude the relevant files from compilation by modifying `CMakeLists.txt` rather than using `#if` guards around the entire body of each file.
|
||||
- Where the functionality involves minor changes to existing files, use `#if` guards.
|
||||
|
||||
- Make sure that you can contribute your work to open source (no license and/or patent conflict is introduced by your code). You will need to [`sign`](#signing-your-work) your commit.
|
||||
|
||||
- Thanks in advance for your patience as we review your contributions; we do appreciate them!
|
||||
|
||||
|
||||
#### Pull Requests
|
||||
Developer workflow for code contributions is as follows:
|
||||
|
||||
1. Developers must first [fork](https://help.github.com/en/articles/fork-a-repo) the [upstream](https://github.com/nvidia/TensorRT) TensorRT OSS repository.
|
||||
|
||||
2. Git clone the forked repository and push changes to the personal fork.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/YOUR_USERNAME/YOUR_FORK.git TensorRT
|
||||
# Checkout the targeted branch and commit changes
|
||||
# Push the commits to a branch on the fork (remote).
|
||||
git push -u origin <local-branch>:<remote-branch>
|
||||
```
|
||||
|
||||
3. Once the code changes are staged on the fork and ready for review, a [Pull Request](https://help.github.com/en/articles/about-pull-requests) (PR) can be [requested](https://help.github.com/en/articles/creating-a-pull-request) to merge the changes from a branch of the fork into a selected branch of upstream.
|
||||
* Exercise caution when selecting the source and target branches for the PR.
|
||||
Note that versioned releases of TensorRT OSS are posted to `release/` branches of the upstream repo.
|
||||
* Creation of a PR creation kicks off the code review process.
|
||||
* Atleast one TensorRT engineer will be assigned for the review.
|
||||
* While under review, mark your PRs as work-in-progress by prefixing the PR title with [WIP].
|
||||
|
||||
4. Since there is no CI/CD process in place yet, the PR will be accepted and the corresponding issue closed only after adequate testing has been completed, manually, by the developer and/or TensorRT engineer reviewing the code.
|
||||
|
||||
|
||||
#### Signing Your Work
|
||||
|
||||
* We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license.
|
||||
|
||||
* Any contribution which contains commits that are not Signed-Off will not be accepted.
|
||||
|
||||
* To sign off on a commit you simply use the `--signoff` (or `-s`) option when committing your changes:
|
||||
```bash
|
||||
$ git commit -s -m "Add cool feature."
|
||||
```
|
||||
This will append the following to your commit message:
|
||||
```
|
||||
Signed-off-by: Your Name <your@email.com>
|
||||
```
|
||||
|
||||
* Full text of the DCO:
|
||||
|
||||
```
|
||||
Developer Certificate of Origin
|
||||
Version 1.1
|
||||
|
||||
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
|
||||
1 Letterman Drive
|
||||
Suite D4700
|
||||
San Francisco, CA, 94129
|
||||
|
||||
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
|
||||
```
|
||||
|
||||
```
|
||||
Developer's Certificate of Origin 1.1
|
||||
|
||||
By making a contribution to this project, I certify that:
|
||||
|
||||
(a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or
|
||||
|
||||
(b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or
|
||||
|
||||
(c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it.
|
||||
|
||||
(d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved.
|
||||
```
|
||||
@@ -0,0 +1,479 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2021 NVIDIA Corporation
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
|
||||
PORTIONS LICENSED AS FOLLOWS
|
||||
|
||||
> tools/pytorch-quantization/examples/torchvision/models/classification/resnet.py
|
||||
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) Soumith Chintala 2016,
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
> samples/common/windows/getopt.c
|
||||
|
||||
Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
Sponsored in part by the Defense Advanced Research Projects
|
||||
Agency (DARPA) and Air Force Research Laboratory, Air Force
|
||||
Materiel Command, USAF, under agreement number F39502-99-1-0512.
|
||||
|
||||
|
||||
Copyright (c) 2000 The NetBSD Foundation, Inc.
|
||||
All rights reserved.
|
||||
|
||||
This code is derived from software contributed to The NetBSD Foundation
|
||||
by Dieter Baron and Thomas Klausner.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
|
||||
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
|
||||
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
- Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
|
||||
- Copyright (c) 2000 The NetBSD Foundation, Inc.
|
||||
|
||||
|
||||
> parsers/common/ieee_half.h
|
||||
> samples/common/half.h
|
||||
> third_party/ieee/half.h
|
||||
|
||||
The MIT License
|
||||
|
||||
Copyright (c) 2012-2017 Christian Rau <rauy@users.sourceforge.net>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of this software and associated documentation files (the "Software"),
|
||||
to deal in the Software without restriction, including without limitation
|
||||
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
and/or sell copies of the Software, and to permit persons to whom the
|
||||
Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included
|
||||
in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
> plugin/multiscaleDeformableAttnPlugin/multiscaleDeformableAttn.cu
|
||||
> plugin/multiscaleDeformableAttnPlugin/multiscaleDeformableAttn.h
|
||||
> plugin/multiscaleDeformableAttnPlugin/multiscaleDeformableIm2ColCuda.cuh
|
||||
|
||||
Copyright 2020 SenseTime
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
DETR
|
||||
|
||||
Copyright 2020 - present, Facebook, Inc
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
> demo/Diffusion/utilities.py
|
||||
> demo/Diffusion/stable_video_diffusion_pipeline.py
|
||||
|
||||
HuggingFace diffusers library.
|
||||
|
||||
Copyright 2024 The HuggingFace Team.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
> demo/Diffusion/utils_sd3/sd3_impls.py
|
||||
> demo/Diffusion/utils_sd3/other_impls.py
|
||||
> demo/Diffusion/utils_sd3/mmdit.py
|
||||
> demo/Diffusion/stable_diffusion_3_pipeline.py
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Stability AI
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
> demo/Diffusion/utilities.py
|
||||
|
||||
ModelScope library.
|
||||
|
||||
Copyright (c) Alibaba, Inc. and its affiliates.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
> plugin/scatterElementsPlugin/atomics.cuh
|
||||
> plugin/scatterElementsPlugin/reducer.cuh
|
||||
> plugin/scatterElementsPlugin/scatterElementsPluginKernel.cu
|
||||
> plugin/scatterElementsPlugin/scatterElementsPluginKernel.h
|
||||
> plugin/scatterElementsPlugin/TensorInfo.cuh
|
||||
|
||||
Copyright (c) 2020 Matthias Fey <matthias.fey@tu-dortmund.de>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
> plugin/roiAlignPlugin/roiAlignKernel.cu
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
> samples/common/sampleTuning.cpp
|
||||
> samples/common/sampleUtils.cpp
|
||||
> samples/trtexec/trtexec.cpp
|
||||
|
||||
nlohmann/json library.
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2013-2022 Niels Lohmann
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,8 @@
|
||||
TensorRT Open Source Software
|
||||
Copyright (c) 2021, NVIDIA CORPORATION.
|
||||
|
||||
This product includes software developed at
|
||||
NVIDIA CORPORATION (https://www.nvidia.com/).
|
||||
|
||||
This software contains code derived by Soumith Chintala.
|
||||
BSD 3-Clause License (https://github.com/pytorch/vision/blob/master/LICENSE)
|
||||
@@ -0,0 +1,351 @@
|
||||
[](https://opensource.org/licenses/Apache-2.0) [](https://docs.nvidia.com/deeplearning/sdk/tensorrt-developer-guide/index.html) [](documents/tensorrt_roadmap_2026q3.pdf)
|
||||
|
||||
# :mega::mega: Announcement :mega::mega:
|
||||
|
||||
TensorRT 11.X is now released with powerful new capabilities designed to accelerate your AI inference workflows. With this major version bump, TensorRT's API has been streamlined and a few legacy features from 10.X have been removed.
|
||||
|
||||
Below provides migration guides for the following features:
|
||||
- Weakly-typed networks and related APIs have been removed, replaced by [Strongly Typed Networks](https://docs.nvidia.com/deeplearning/tensorrt/latest/inference-library/advanced.html#strongly-typed-networks).
|
||||
- Implicit quantization and related APIs have been removed, replaced by [Explicit Quantization](https://docs.nvidia.com/deeplearning/tensorrt/latest/inference-library/work-quantized-types.html#explicit-quantization)
|
||||
- IPluginV2 and related APIs have been removed, replaced by [IPluginV3](https://docs.nvidia.com/deeplearning/tensorrt/latest/inference-library/extending-custom-layers.html#migrating-v2-plugins-to-ipluginv3)
|
||||
- TREX tool has been removed, replaced by [Nsight Deep Learning Designer](https://docs.nvidia.com/nsight-dl-designer/UserGuide/index.html#visualizing-a-tensorrt-engine)
|
||||
- Python bindings for Python 3.9 and older versions have been removed. RPM packages for RHEL/Rocky Linux 8 and RHEL/Rocky Linux 9 now depend on Python 3.12.
|
||||
|
||||
# TensorRT Open Source Software
|
||||
|
||||
This repository contains the Open Source Software (OSS) components of NVIDIA TensorRT. It includes the sources for TensorRT plugins and ONNX parser, as well as sample applications demonstrating usage and capabilities of the TensorRT platform. These open source software components are a subset of the TensorRT General Availability (GA) release with some extensions and bug-fixes.
|
||||
|
||||
- For step-by-step walkthroughs of the TensorRT import paths (ONNX, Torch-TensorRT, HuggingFace/Optimum, Network Definition API) with examples and tooling tips, see the [Import Workflows Guide](documents/import_workflows.md).
|
||||
- For the per-model support matrix across import paths (LLM, encoder-NLP, vision, audio, diffusion, multimodal), see [Supported Models](documents/supported_models.md).
|
||||
- For code contributions to TensorRT-OSS, please see our [Contribution Guide](CONTRIBUTING.md) and [Coding Guidelines](CODING-GUIDELINES.md).
|
||||
- For a summary of new additions and updates shipped with TensorRT-OSS releases, please refer to the [Changelog](CHANGELOG.md).
|
||||
- For business inquiries, please contact [researchinquiries@nvidia.com](mailto:researchinquiries@nvidia.com)
|
||||
- For press and other inquiries, please contact Hector Marinez at [hmarinez@nvidia.com](mailto:hmarinez@nvidia.com)
|
||||
|
||||
Need enterprise support? NVIDIA global support is available for TensorRT with the [NVIDIA AI Enterprise software suite](https://www.nvidia.com/en-us/data-center/products/ai-enterprise/). Check out [NVIDIA LaunchPad](https://www.nvidia.com/en-us/launchpad/ai/ai-enterprise/) for free access to a set of hands-on labs with TensorRT hosted on NVIDIA infrastructure.
|
||||
|
||||
Join the [TensorRT and Triton community](https://www.nvidia.com/en-us/deep-learning-ai/triton-tensorrt-newsletter/) and stay current on the latest product updates, bug fixes, content, best practices, and more.
|
||||
|
||||
# Agentic Coding Skills
|
||||
Various skills related to TensorRT usage and benchmarking are available [here](.agents/skills). For installation, refer to the instructions of your preferred coding agent.
|
||||
|
||||
# Prebuilt TensorRT Python Package
|
||||
|
||||
We provide the TensorRT Python package for an easy installation. \
|
||||
To install:
|
||||
|
||||
```bash
|
||||
pip install tensorrt
|
||||
```
|
||||
|
||||
You can skip the **Build** section to enjoy TensorRT with Python.
|
||||
|
||||
# Build
|
||||
|
||||
## Prerequisites
|
||||
|
||||
To build the TensorRT-OSS components, you will first need the following software packages.
|
||||
|
||||
**TensorRT GA build**
|
||||
|
||||
- TensorRT v11.1.0.106
|
||||
- Available from direct download links listed below
|
||||
|
||||
**System Packages**
|
||||
|
||||
- [CUDA](https://developer.nvidia.com/cuda-toolkit)
|
||||
- Recommended versions:
|
||||
- cuda-13.3.0
|
||||
- cuda-12.9.0
|
||||
- [CUDNN (optional)](https://developer.nvidia.com/cudnn)
|
||||
- cuDNN 8.9
|
||||
- [GNU make](https://ftp.gnu.org/gnu/make/) >= v4.1
|
||||
- [cmake](https://github.com/Kitware/CMake/releases) >= v3.31
|
||||
- [python](https://www.python.org/downloads/) >= v3.10, <= v3.14.x
|
||||
- [pip](https://pypi.org/project/pip/#history) >= v19.0
|
||||
- Essential utilities
|
||||
- [git](https://git-scm.com/downloads), [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/), [wget](https://www.gnu.org/software/wget/faq.html#download)
|
||||
|
||||
**Optional Packages**
|
||||
|
||||
- [NCCL](https://developer.nvidia.com/nccl/nccl-download) >= v2.19, < v3.0 — only when building with multi-device support (`-DTRT_BUILD_ENABLE_MULTIDEVICE=ON`) for the `sampleDistCollective` sample.
|
||||
- Containerized build
|
||||
- [Docker](https://docs.docker.com/install/) >= 19.03
|
||||
- [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-docker)
|
||||
- PyPI packages (for demo applications/tests)
|
||||
- [onnx](https://pypi.org/project/onnx/)
|
||||
- [onnxruntime](https://pypi.org/project/onnxruntime/)
|
||||
- [tensorflow-gpu](https://pypi.org/project/tensorflow/) >= 2.5.1
|
||||
- [Pillow](https://pypi.org/project/Pillow/) >= 9.0.1
|
||||
- [pycuda](https://pypi.org/project/pycuda/) < 2021.1
|
||||
- [numpy](https://pypi.org/project/numpy/)
|
||||
- [pytest](https://pypi.org/project/pytest/)
|
||||
- Code formatting tools (for contributors)
|
||||
|
||||
- [Clang-format](https://clang.llvm.org/docs/ClangFormat.html)
|
||||
- [Git-clang-format](https://github.com/llvm-mirror/clang/blob/master/tools/clang-format/git-clang-format)
|
||||
|
||||
> NOTE: [onnx-tensorrt](https://github.com/onnx/onnx-tensorrt), [cub](http://nvlabs.github.io/cub/), and [protobuf](https://github.com/protocolbuffers/protobuf.git) packages are downloaded along with TensorRT OSS, and not required to be installed.
|
||||
|
||||
## Downloading TensorRT Build
|
||||
|
||||
1. #### Download TensorRT OSS
|
||||
|
||||
```bash
|
||||
git clone -b main https://github.com/nvidia/TensorRT TensorRT
|
||||
cd TensorRT
|
||||
git submodule update --init --recursive
|
||||
```
|
||||
|
||||
2. #### (Optional - if not using TensorRT container) Specify the TensorRT GA release build path
|
||||
|
||||
If using the TensorRT OSS build container, TensorRT libraries are preinstalled under `/usr/lib/x86_64-linux-gnu` and you may skip this step.
|
||||
|
||||
Else download and extract the TensorRT GA build from [NVIDIA Developer Zone](https://developer.nvidia.com) with the direct links below:
|
||||
|
||||
- [TensorRT 11.1.0.106 for CUDA 13.3, Linux x86_64](https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/11.1.0/tars/TensorRT-Enterprise-11.1.0.106-Linux-x86_64-cuda-13.3-Release-external.tar.zst)
|
||||
- [TensorRT 11.1.0.106 for CUDA 12.9, Linux x86_64](https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/11.1.0/tars/TensorRT-Enterprise-11.1.0.106-Linux-x86_64-cuda-12.9-Release-external.tar.zst)
|
||||
- [TensorRT 11.1.0.106 for CUDA 13.3, Windows x86_64](https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/11.1.0/zip/TensorRT-Enterprise-11.1.0.106-Windows-amd64-cuda-13.3-Release-external.zip)
|
||||
- [TensorRT 11.1.0.106 for CUDA 12.9, Windows x86_64](https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/11.1.0/zip/TensorRT-Enterprise-11.1.0.106-Windows-amd64-cuda-12.9-Release-external.zip)
|
||||
|
||||
**Example: Ubuntu 22.04 on x86-64 with cuda-13.3**
|
||||
|
||||
```bash
|
||||
cd ~/Downloads
|
||||
tar --zstd -xvf TensorRT-Enterprise-11.1.0.106-Linux-x86_64-cuda-13.3-Release-external.tar.zst
|
||||
export TRT_LIBPATH=`pwd`/TensorRT-11.1.0.106/lib
|
||||
```
|
||||
|
||||
**Example: Windows on x86-64 with cuda-12.9**
|
||||
|
||||
```powershell
|
||||
Expand-Archive -Path TensorRT-Enterprise-11.1.0.106-Windows-amd64-cuda-12.9-Release-external.zip
|
||||
$env:TRT_LIBPATH="$pwd\TensorRT-11.1.0.106\lib"
|
||||
```
|
||||
|
||||
## Setting Up The Build Environment
|
||||
|
||||
For Linux platforms, we recommend that you generate a docker container for building TensorRT OSS as described below. For native builds, please install the [prerequisite](#prerequisites) _System Packages_.
|
||||
|
||||
1. #### Generate the TensorRT-OSS build container.
|
||||
|
||||
**Example: Ubuntu 24.04 on x86-64 with cuda-13.3 (default)**
|
||||
|
||||
```bash
|
||||
./docker/build.sh --file docker/ubuntu-24.04.Dockerfile --tag tensorrt-ubuntu24.04-cuda13.3
|
||||
```
|
||||
|
||||
**Example: Rockylinux8 on x86-64 with cuda-13.3**
|
||||
|
||||
```bash
|
||||
./docker/build.sh --file docker/rockylinux8.Dockerfile --tag tensorrt-rockylinux8-cuda13.3
|
||||
```
|
||||
|
||||
**Example: Ubuntu 24.04 cross-compile for Jetson (aarch64) with cuda-13.3 (JetPack SDK)**
|
||||
|
||||
```bash
|
||||
./docker/build.sh --file docker/ubuntu-cross-aarch64.Dockerfile --tag tensorrt-jetpack-cuda13.3
|
||||
```
|
||||
|
||||
**Example: Ubuntu 24.04 on aarch64 with cuda-13.3**
|
||||
|
||||
```bash
|
||||
./docker/build.sh --file docker/ubuntu-24.04-aarch64.Dockerfile --tag tensorrt-aarch64-ubuntu24.04-cuda13.3
|
||||
```
|
||||
|
||||
2. #### Launch the TensorRT-OSS build container.
|
||||
**Example: Ubuntu 24.04 build container**
|
||||
```bash
|
||||
./docker/launch.sh --tag tensorrt-ubuntu24.04-cuda13.3 --gpus all
|
||||
```
|
||||
> NOTE:
|
||||
> <br> 1. Use the `--tag` corresponding to build container generated in Step 1.
|
||||
> <br> 2. [NVIDIA Container Toolkit](#prerequisites) is required for GPU access (running TensorRT applications) inside the build container.
|
||||
> <br> 3. `sudo` password for Ubuntu build containers is 'nvidia'.
|
||||
> <br> 4. Specify port number using `--jupyter <port>` for launching Jupyter notebooks.
|
||||
> <br> 5. Write permission to this folder is required as this folder will be mounted inside the docker container for uid:gid of 1000:1000.
|
||||
|
||||
## Building TensorRT-OSS
|
||||
|
||||
- Generate Makefiles and build
|
||||
|
||||
**Example: Linux (x86-64) build with default cuda-13.3**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
cmake .. -DTRT_LIB_DIR=$TRT_LIBPATH -DTRT_OUT_DIR=`pwd`/out
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
**Example: Linux (aarch64) build with default cuda-13.3**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
cmake .. -DTRT_LIB_DIR=$TRT_LIBPATH -DTRT_OUT_DIR=`pwd`/out -DCMAKE_TOOLCHAIN_FILE=$TRT_OSSPATH/cmake/toolchains/cmake_aarch64-native.toolchain
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
**Example: Native build on Jetson Thor (aarch64) with cuda-13.3**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
cmake .. -DTRT_LIB_DIR=$TRT_LIBPATH -DTRT_OUT_DIR=`pwd`/out -DTRT_PLATFORM_ID=aarch64
|
||||
CC=/usr/bin/gcc make -j$(nproc)
|
||||
```
|
||||
|
||||
> NOTE: C compiler must be explicitly specified via CC= for native aarch64 builds of protobuf.
|
||||
|
||||
**Example: Ubuntu 24.04 Cross-Compile for Jetson Thor (aarch64) with cuda-13.3 (JetPack)**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
cmake .. -DTRT_LIB_DIR=$TRT_LIBPATH -DCMAKE_TOOLCHAIN_FILE=$TRT_OSSPATH/cmake/toolchains/cmake_aarch64_cross.toolchain
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
**Example: Ubuntu 24.04 Cross-Compile for DriveOS (aarch64) with cuda-13.3**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
cmake .. -DTRT_LIB_DIR=$TRT_LIBPATH -DCMAKE_TOOLCHAIN_FILE=$TRT_OSSPATH/cmake/toolchains/cmake_aarch64_dos_cross.toolchain
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
**Example: Native builds on Windows (x86) with cuda-13.3**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
New-Item -ItemType Directory -Path build
|
||||
cd build
|
||||
cmake .. -DTRT_LIB_DIR="$env:TRT_LIBPATH" -DTRT_OUT_DIR="$pwd\\out"
|
||||
msbuild TensorRT.sln /property:Configuration=Release -m:$env:NUMBER_OF_PROCESSORS
|
||||
```
|
||||
|
||||
> NOTE: The default CUDA version used by CMake is 13.3. To override this, for example to 12.9, append `-DCUDA_VERSION=12.9` to the cmake command.
|
||||
|
||||
- Required CMake build arguments are:
|
||||
- `TRT_LIB_DIR`: Path to the TensorRT installation directory containing libraries.
|
||||
- `TRT_OUT_DIR`: Output directory where generated build artifacts will be copied.
|
||||
- Optional CMake build arguments:
|
||||
- `CMAKE_BUILD_TYPE`: Specify if binaries generated are for release or debug (contain debug symbols). Values consists of [`Release`] | `Debug`
|
||||
- `CUDA_VERSION`: The version of CUDA to target, for example [`12.9.9`].
|
||||
- `CUDNN_VERSION`: The version of cuDNN to target, for example [`8.9`].
|
||||
- `PROTOBUF_VERSION`: The version of Protobuf to use, for example [`3.20.1`]. Note: Changing this will not configure CMake to use a system version of Protobuf, it will configure CMake to download and try building that version.
|
||||
- `CMAKE_TOOLCHAIN_FILE`: The path to a toolchain file for cross compilation.
|
||||
- `BUILD_PARSERS`: Specify if the parsers should be built, for example [`ON`] | `OFF`. If turned OFF, CMake will try to find precompiled versions of the parser libraries to use in compiling samples. First in `${TRT_LIB_DIR}`, then on the system. If the build type is Debug, then it will prefer debug builds of the libraries before release versions if available.
|
||||
- `BUILD_PLUGINS`: Specify if the plugins should be built, for example [`ON`] | `OFF`. If turned OFF, CMake will try to find a precompiled version of the plugin library to use in compiling samples. First in `${TRT_LIB_DIR}`, then on the system. If the build type is Debug, then it will prefer debug builds of the libraries before release versions if available.
|
||||
- `BUILD_SAMPLES`: Specify if the samples should be built, for example [`ON`] | `OFF`.
|
||||
- `BUILD_SAFE_SAMPLES`: Specify if safety samples should be built, for example [`ON`] | `OFF`.
|
||||
- `TRT_SAFETY_INFERENCE_ONLY`: Specify if only build the safety inference components, for example [`ON`] | `OFF`. If turned ON, all other components will be turned OFF except `BUILD_SAFE_SAMPLES`.
|
||||
- `TRT_PLATFORM_ID`: Bare-metal build (unlike containerized cross-compilation). Currently supported options: `x86_64` (default).
|
||||
- `TRT_BUILD_ENABLE_MULTIDEVICE`: Enable the multi-device sample (`sampleDistCollective`). Use `-DTRT_BUILD_ENABLE_MULTIDEVICE=ON` to build it; requires [NCCL](https://developer.nvidia.com/nccl/nccl-download) >= v2.19, < v3.0.
|
||||
- `TRT_BUILD_TESTING` : Build gTests for samples. Requires [gtest](https://github.com/google/googletest) if available; otherwise fetches googletest at configure time.
|
||||
|
||||
## Building TensorRT DriveOS Samples
|
||||
|
||||
- Generate Makefiles and build
|
||||
|
||||
**Example: Cross-Compile for DOS7 Linux (aarch64)**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
cmake .. -DBUILD_SAMPLES=ON -DBUILD_PLUGINS=OFF -DBUILD_PARSERS=OFF -DTRT_OUT_DIR=`pwd`/bin_dynamic_cross -DTRT_LIB_DIR=$TRT_LIBPATH -DCMAKE_TOOLCHAIN_FILE=$TRT_OSSPATH/cmake/toolchains/cmake_aarch64_dos_cross.toolchain
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
**Example: Cross-Compile for DOS6.5 Linux (aarch64)**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
cmake .. -DBUILD_SAMPLES=ON -DBUILD_PLUGINS=OFF -DBUILD_PARSERS=OFF -DTRT_OUT_DIR=`pwd`/bin_dynamic_cross -DTRT_LIB_DIR=$TRT_LIBPATH -DCMAKE_TOOLCHAIN_FILE=$TRT_OSSPATH/cmake/toolchains/cmake_aarch64_dos_cross.toolchain -DCUDA_VERSION=11.4 -DCMAKE_CUDA_ARCHITECTURES=87
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
**Example: Native build for DOS6.5 and DOS7 Linux (aarch64)**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
cmake .. -DTRT_LIB_DIR=$TRT_LIBPATH -DTRT_OUT_DIR=`pwd`/out -DCMAKE_TOOLCHAIN_FILE=$TRT_OSSPATH/cmake/toolchains/cmake_aarch64-native.toolchain -DBUILD_SAMPLES=ON -DBUILD_PLUGINS=OFF -DBUILD_PARSERS=OFF
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
**Example: Cross-Compile for DOS6.5 QNX (aarch64)**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
export CUDA_VERSION=11.4
|
||||
export CUDA=cuda-$CUDA_VERSION
|
||||
export CUDA_ROOT=/usr/local/cuda-safe-$CUDA_VERSION
|
||||
export QNX_BASE=/drive/toolchains/qnx_toolchain # Set to your QNX toolchain installation path
|
||||
export QNX_HOST=$QNX_BASE/host/linux/x86_64/
|
||||
export QNX_TARGET=$QNX_BASE/target/qnx7/
|
||||
export PATH=$PATH:$QNX_HOST/usr/bin
|
||||
cmake .. -DBUILD_SAMPLES=ON -DBUILD_PLUGINS=OFF -DBUILD_PARSERS=OFF -DBUILD_SAFE_SAMPLES=OFF -DCMAKE_CUDA_COMPILER=$CUDA_ROOT/bin/nvcc -DTRT_OUT_DIR=`pwd`/bin_dynamic_cross -DTRT_LIB_DIR=$TRT_LIBPATH -DCMAKE_TOOLCHAIN_FILE=$TRT_OSSPATH/cmake/toolchains/cmake_qnx.toolchain -DCUDA_VERSION=$CUDA_VERSION -DCMAKE_CUDA_ARCHITECTURES=87
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
> NOTE: Set `QNX_BASE` to your QNX toolchain installation path.
|
||||
> If your CUDA version is not the same as in the example, set `CUDA_VERSION` (for examples that use it in multiple places) or add `-DCUDA_VERSION=<version>` to the cmake command.
|
||||
|
||||
**Example: Cross-Compile for DOS6.5 QNX Safety (aarch64)**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
export CUDA_VERSION=11.4
|
||||
export QNX_BASE=/drive/toolchains/qnx_toolchain # Set to your QNX toolchain installation path
|
||||
export QNX_HOST=$QNX_BASE/host/linux/x86_64/
|
||||
export QNX_TARGET=$QNX_BASE/target/qnx7/
|
||||
export PATH=$PATH:$QNX_HOST/usr/bin
|
||||
export CUDA=cuda-$CUDA_VERSION
|
||||
export CUDA_ROOT=/usr/local/cuda-safe-$CUDA_VERSION
|
||||
cmake .. -DBUILD_SAMPLES=OFF -DBUILD_SAFE_SAMPLES=ON -DBUILD_PLUGINS=OFF -DBUILD_PARSERS=OFF -DTRT_SAFETY_INFERENCE_ONLY=ON -DTRT_OUT_DIR=`pwd`/bin_dynamic_cross -DTRT_LIB_DIR=$TRT_LIBPATH -DCMAKE_TOOLCHAIN_FILE=$TRT_OSSPATH/cmake/toolchains/cmake_qnx_safe.toolchain -DCUDA_VERSION=$CUDA_VERSION -DCMAKE_CUDA_COMPILER=$CUDA_ROOT/bin/nvcc -DCMAKE_CUDA_ARCHITECTURES=87
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
> NOTE: Set `QNX_BASE` to your QNX toolchain installation path.
|
||||
> If your CUDA version is not the same as in the example, set `CUDA_VERSION` (for examples that use it in multiple places) or add `-DCUDA_VERSION=<version>` to the cmake command.
|
||||
|
||||
**Example: Cross-Compile for DOS7 QNX (aarch64)**
|
||||
|
||||
```bash
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
export CUDA_VERSION=13.3
|
||||
export CUDA=cuda-$CUDA_VERSION
|
||||
export CUDA_ROOT=/usr/local/cuda-safe-$CUDA_VERSION
|
||||
export QNX_BASE=/drive/toolchains/qnx_toolchain # Set to your QNX toolchain installation path
|
||||
export QNX_HOST=$QNX_BASE/host/linux/x86_64/
|
||||
export QNX_TARGET=$QNX_BASE/target/qnx/
|
||||
export PATH=$PATH:$QNX_HOST/usr/bin
|
||||
cmake .. -DBUILD_SAMPLES=ON -DBUILD_PLUGINS=OFF -DBUILD_PARSERS=OFF -DBUILD_SAFE_SAMPLES=OFF -DCMAKE_CUDA_COMPILER=$CUDA_ROOT/bin/nvcc -DTRT_OUT_DIR=`pwd`/bin_dynamic_cross -DTRT_LIB_DIR=$TRT_LIBPATH -DCMAKE_TOOLCHAIN_FILE=$TRT_OSSPATH/cmake/toolchains/cmake_qnx.toolchain -DCUDA_VERSION=$CUDA_VERSION -DCMAKE_CUDA_ARCHITECTURES=110
|
||||
make -j$(nproc)
|
||||
```
|
||||
|
||||
> NOTE: Set `QNX_BASE` to your QNX toolchain installation path.
|
||||
> If your CUDA version is not the same as in the example, set `CUDA_VERSION` (for examples that use it in multiple places) or add `-DCUDA_VERSION=<version>` to the cmake command.
|
||||
|
||||
# References
|
||||
|
||||
## TensorRT Resources
|
||||
|
||||
- [TensorRT Developer Home](https://developer.nvidia.com/tensorrt)
|
||||
- [TensorRT QuickStart Guide](https://docs.nvidia.com/deeplearning/tensorrt/quick-start-guide/index.html)
|
||||
- [TensorRT Developer Guide](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html)
|
||||
- [TensorRT Sample Support Guide](https://docs.nvidia.com/deeplearning/tensorrt/sample-support-guide/index.html)
|
||||
- [TensorRT ONNX Tools](https://docs.nvidia.com/deeplearning/tensorrt/index.html#tools)
|
||||
- [TensorRT Discussion Forums](https://devtalk.nvidia.com/default/board/304/tensorrt/)
|
||||
- [TensorRT Release Notes](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html)
|
||||
|
||||
## Known Issues
|
||||
|
||||
- Please refer to [TensorRT Release Notes](https://docs.nvidia.com/deeplearning/tensorrt/release-notes)
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`NVIDIA/TensorRT`
|
||||
- 原始仓库：https://github.com/NVIDIA/TensorRT
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
@@ -0,0 +1,349 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# This file handles logic relating to building "fat" static libraries, that is, ones which contain other static libraries.
|
||||
# Idiomatically, CMake would prefer we ship all static libraries as part of our release and ship a file which specifies the linkages.
|
||||
# However, for various reasons, this is both inadvisable and annoying. Instead, we would prefer to ship one static library containing all dependencies.
|
||||
#
|
||||
# To do that, we need rules to bundle static libraries into other static libraries. Hence, this class.
|
||||
|
||||
define_property(TARGET
|
||||
PROPERTY BUNDLED_LIBRARY_TEMPLATE_PATH
|
||||
BRIEF_DOCS "File path to the template script which will be written to when calling target_bundle_libraries."
|
||||
)
|
||||
|
||||
define_property(TARGET
|
||||
PROPERTY BUNDLE_LIBRARIES
|
||||
BRIEF_DOCS "The list of libraries that have been bundled into this target by target_bundle_libraries."
|
||||
)
|
||||
|
||||
define_property(TARGET
|
||||
PROPERTY BUNDLE_LIBRARY_KNOWN_TYPE
|
||||
BRIEF_DOCS "Fallback type used when a target's TYPE is UNKNOWN_LIBRARY."
|
||||
)
|
||||
|
||||
define_property(TARGET
|
||||
PROPERTY BUNDLE_VISITED_INTERFACES
|
||||
BRIEF_DOCS "Interface libraries already traversed by __bundleRecursiveDeps for this target."
|
||||
)
|
||||
|
||||
# Internal helper to prefix all messages with "[target_bundle_libraries]: ".
|
||||
#
|
||||
# \param mode The message mode to be passed to message(...)
|
||||
# \param argn The message contents.
|
||||
macro(__bundleMessage mode)
|
||||
message(${mode} "[target_bundle_libraries]: " ${ARGN})
|
||||
endmacro()
|
||||
|
||||
# Illegal genex magic™
|
||||
# Given a string containing a generator expression (var), edits the variable in-place to escape any generator expression literals.
|
||||
# The escaped generator expression is then suitable for use within a LIST:TRANSFORM replacement block.
|
||||
# This more or less allows for mapping lists to generator expressions, which can be recursively evaluated.
|
||||
function(escape_generator_expression var)
|
||||
string(REPLACE ">" "__ANGLE_R__" ${var} "${${var}}")
|
||||
string(REPLACE "$" "$<1:$>" ${var} "${${var}}")
|
||||
string(REPLACE "," "$<COMMA>" ${var} "${${var}}")
|
||||
string(REPLACE "__ANGLE_R__" "$<ANGLE-R>" ${var} "${${var}}")
|
||||
return(PROPAGATE ${var})
|
||||
endfunction()
|
||||
|
||||
# Recursively unwraps alias targets until finding the real target. Non-targets are returned verbatim.
|
||||
# We need to ensure that the generated .mri script contains a deduplicated list of bundled targets
|
||||
# so we need to resolve aliases, otherwise we may end up with multiple entries for the same target.
|
||||
#
|
||||
# \param target_name The name of the target to unwrap.
|
||||
# \param result_var The variable to store the unwrapped target name in.
|
||||
function(unwrapAlias target_name result_var)
|
||||
# Check cache first (keyed on the raw input string).
|
||||
get_property(_cache_set GLOBAL PROPERTY _UNWRAP_ALIAS_CACHE_${target_name} SET)
|
||||
if(_cache_set)
|
||||
get_property(_cached GLOBAL PROPERTY _UNWRAP_ALIAS_CACHE_${target_name})
|
||||
set(${result_var} ${_cached} PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
|
||||
# First, try to unwrap common generator expressions that may wrap the target name.
|
||||
string(REGEX MATCH "\\$<LINK_LIBRARY:WHOLE_ARCHIVE,([a-zA-Z0-9_.:]+)>" _ ${target_name})
|
||||
if(TARGET ${CMAKE_MATCH_1})
|
||||
set(target_name ${CMAKE_MATCH_1})
|
||||
endif()
|
||||
|
||||
string(REGEX MATCH "\\$<LINK_ONLY:([a-zA-Z0-9_.:]+)>" _ ${target_name})
|
||||
if(TARGET ${CMAKE_MATCH_1})
|
||||
set(target_name ${CMAKE_MATCH_1})
|
||||
endif()
|
||||
|
||||
if(TARGET ${target_name})
|
||||
get_target_property(aliased_target ${target_name} ALIASED_TARGET)
|
||||
if(aliased_target)
|
||||
# Recursively unwrap in case there are multiple levels
|
||||
unwrapAlias(${aliased_target} unwrapped)
|
||||
set(_result ${unwrapped})
|
||||
else()
|
||||
# Not an alias, return the original name
|
||||
set(_result ${target_name})
|
||||
endif()
|
||||
else()
|
||||
# Not a target at all, return the original name
|
||||
set(_result ${target_name})
|
||||
endif()
|
||||
|
||||
set_property(GLOBAL PROPERTY _UNWRAP_ALIAS_CACHE_${target_name} ${_result})
|
||||
set(${result_var} ${_result} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# Internal function to retrieve the type of library for a given target.
|
||||
# This function will fallback to the value of BUNDLE_LIBRARY_KNOWN_TYPE if it encounters an UNKNOWN_LIBRARY.
|
||||
#
|
||||
# \param lib A target to evaluate the type for.
|
||||
# \param outVar The output variable to store the type name in.
|
||||
function(__get_lib_type lib outVar)
|
||||
get_target_property(libType ${lib} TYPE)
|
||||
if (${libType} STREQUAL UNKNOWN_LIBRARY)
|
||||
get_target_property(knownType ${lib} BUNDLE_LIBRARY_KNOWN_TYPE)
|
||||
if (NOT ${knownType} STREQUAL "knownType-NOTFOUND")
|
||||
set(libType ${knownType})
|
||||
endif()
|
||||
__bundleMessage(DEBUG "Using known type of unknown library ${lib}: ${knownType}")
|
||||
endif()
|
||||
set(${outVar} ${libType} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# This is an internal-only function called the first time that target_bundle_libraries is called.
|
||||
# It "registers" a target for bundling by creating the base template file and populating the target property BUNDLED_LIBRARY_TEMPLATE_PATH.
|
||||
# Additionally, it registers the file generation logic for making the "final" script, as well as the custom command for running it after the build.
|
||||
#
|
||||
# \param lib The mainLib from target_bundle_libraries.
|
||||
# \param templatePath The file path to the template file that will be created.
|
||||
function(__registerTargetForBundling lib templatePath)
|
||||
if(MSVC)
|
||||
set(scriptPath $<TARGET_FILE_DIR:${lib}>/archive-${lib}.bat)
|
||||
|
||||
set(template "/OUT:\"$<TARGET_FILE:${lib}>\" \"$<TARGET_FILE:${lib}>\"\n")
|
||||
|
||||
# Windows-syntax version of the same logic from the linux build below.
|
||||
# Main differences is that windows does not have `addlib`, and uses `\n` instead of `\n`. Additionally, the values need to be quoted to account for spaces in file paths.
|
||||
set(replaceExpr "\"$<IF:$<TARGET_EXISTS:\\1>,$<TARGET_FILE:\\1>,\\1>\"")
|
||||
escape_generator_expression(replaceExpr)
|
||||
string(APPEND template "$<TARGET_GENEX_EVAL:${lib},$<JOIN:$<LIST:TRANSFORM,$<TARGET_PROPERTY:${lib},BUNDLE_LIBRARIES>,REPLACE,(.+),${replaceExpr}>,\n>>\n")
|
||||
|
||||
file(WRITE ${templatePath} ${template})
|
||||
file(GENERATE
|
||||
OUTPUT ${scriptPath}
|
||||
INPUT ${templatePath}
|
||||
)
|
||||
add_custom_command(TARGET ${lib} POST_BUILD
|
||||
COMMAND ${CMAKE_AR} /NOLOGO @\"${scriptPath}\"
|
||||
COMMAND ${CMAKE_COMMAND} -E echo "Bundled $<LIST:LENGTH,$<TARGET_PROPERTY:${lib},BUNDLE_LIBRARIES>> static libraries into target ${lib}. Script: ${scriptPath}"
|
||||
WORKING_DIRECTORY $<TARGET_FILE_DIR:${lib}>
|
||||
)
|
||||
else()
|
||||
set(scriptPath $<TARGET_FILE_DIR:${lib}>/archive-${lib}.mri)
|
||||
|
||||
set(template "create $<TARGET_FILE:${lib}>\n")
|
||||
string(APPEND template "addlib $<TARGET_FILE:${lib}>\n")
|
||||
# Expand BUNDLE_LIBRARIES into the appropriate chain of addlib commands needed.
|
||||
# BUNDLE_LIBRARIES will contain either (a) target names or (b) absolute file paths to libraries to include.
|
||||
# This first part will disambiguate between (a) and (b) by evaluating (a) to `addlib $<TARGET_FILE:lib>` and (b) to `addlib [[filepath]]`
|
||||
set(replaceExpr "addlib $<IF:$<TARGET_EXISTS:\\1>,$<TARGET_FILE:\\1>,\\1>")
|
||||
escape_generator_expression(replaceExpr)
|
||||
|
||||
# The second part maps every element in BUNDLE_LIBRARIES to `replaceExpr` and evaluates the resulting replacement, which produces the final .mri file.
|
||||
string(APPEND template "$<TARGET_GENEX_EVAL:${lib},$<JOIN:$<LIST:TRANSFORM,$<TARGET_PROPERTY:${lib},BUNDLE_LIBRARIES>,REPLACE,(.+),${replaceExpr}>,\n>>\n")
|
||||
string(APPEND template "save\n")
|
||||
string(APPEND template "end\n")
|
||||
|
||||
file(WRITE ${templatePath} ${template})
|
||||
file(GENERATE
|
||||
OUTPUT ${scriptPath}
|
||||
INPUT ${templatePath}
|
||||
)
|
||||
|
||||
add_custom_command(TARGET ${lib} POST_BUILD
|
||||
COMMAND ${CMAKE_AR} -M < ${scriptPath}
|
||||
COMMAND ${CMAKE_RANLIB} $<TARGET_FILE:${lib}>
|
||||
COMMAND ${CMAKE_COMMAND} -E echo "Bundled $<LIST:LENGTH,$<TARGET_PROPERTY:${lib},BUNDLE_LIBRARIES>> static libraries into target ${lib}. Script: ${scriptPath}"
|
||||
WORKING_DIRECTORY $<TARGET_FILE_DIR:${lib}>
|
||||
)
|
||||
endif()
|
||||
|
||||
set_target_properties(${lib}
|
||||
PROPERTIES BUNDLED_LIBRARY_TEMPLATE_PATH ${templatePath}
|
||||
)
|
||||
endfunction()
|
||||
|
||||
# Subcomponent of target_bundle_libraries which is responsible for walking the provided depLibs and recursively calling target_bundle_libraries.
|
||||
# This macro must only be used within target_bundle_libraries.
|
||||
#
|
||||
# \param mainLib The current main library from target_bundle_libraries
|
||||
# \param linkVis The link visibility from target_bundle_libraries
|
||||
# \param argn The dependencies to walk. Usually the INTERFACE_LINK_LIBRARIES of a target currently being bundled.
|
||||
function(__bundleRecursiveDeps mainLib linkVis)
|
||||
if(ARGN)
|
||||
get_target_property(bundledLibs ${mainLib} BUNDLE_LIBRARIES)
|
||||
|
||||
foreach(dep IN LISTS ARGN)
|
||||
unwrapAlias(${dep} dep)
|
||||
if(${dep} IN_LIST bundledLibs)
|
||||
continue() # Skip bundling of already-bundled libs to avoid many invocations of the same warnings.
|
||||
endif()
|
||||
|
||||
if(TARGET ${dep})
|
||||
__get_lib_type(${dep} depType)
|
||||
if (${depType} STREQUAL STATIC_LIBRARY)
|
||||
target_bundle_libraries(${mainLib} ${linkVis} ${dep})
|
||||
elseif(${depType} STREQUAL INTERFACE_LIBRARY)
|
||||
# For interface libraries, we want to add all of the static libraries they may be pointing to, without the library itself (since it is not a static).
|
||||
# Skip if we've already traversed this interface lib's deps for mainLib to avoid re-walking shared transitive subtrees.
|
||||
get_target_property(_visited ${mainLib} BUNDLE_VISITED_INTERFACES)
|
||||
if(NOT _visited)
|
||||
set(_visited "")
|
||||
endif()
|
||||
if(${dep} IN_LIST _visited)
|
||||
continue()
|
||||
endif()
|
||||
set_property(TARGET ${mainLib} APPEND PROPERTY BUNDLE_VISITED_INTERFACES ${dep})
|
||||
get_target_property(interfaceLibs ${dep} INTERFACE_LINK_LIBRARIES)
|
||||
__bundleRecursiveDeps(${mainLib} ${linkVis} ${interfaceLibs})
|
||||
elseif(${depType} STREQUAL SHARED_LIBRARY)
|
||||
# Skip SO's, since static libraries at the SO-boundary should not be bundled into the target mainLib.
|
||||
else()
|
||||
__bundleMessage(DEBUG "Skipping unhandled dependency ${dep} (a dependency of ${bundledLib}) with type ${depType} in ${mainLib}")
|
||||
endif()
|
||||
else()
|
||||
__bundleMessage(DEBUG "Failed to recursively bundle dependency ${dep} (a dependency of ${bundledLib}) in ${mainLib}")
|
||||
endif()
|
||||
endforEach()
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
# This function acts as a replacement target_link_libraries, instead linking
|
||||
# one or more "bundled" static libraries into a "main" static library.
|
||||
# The bundled libs are embedded inside the main lib during the post-build step.
|
||||
# CMake interface properties for definitions, etc. are propagated using the specified visibility.
|
||||
#
|
||||
# This function is recursive. Any static dependencies of any bundled library will also be bundled into the mainLib.
|
||||
#
|
||||
# \param mainLib The main library that other libraries are to be bundled into.
|
||||
# \param linkVis The link visiblity for interface properties. One of PRIVATE or PUBLIC.
|
||||
# Using PRIVATE hides all interface properties of bundled libraries from consumers of this library.
|
||||
# Using PUBLIC will share interface properties with dependents.
|
||||
# \param argn Variadic arguments - The list of targets to be linked into the mainLib.
|
||||
function(target_bundle_libraries mainLib linkVis)
|
||||
if (NOT ${linkVis} STREQUAL PUBLIC AND NOT ${linkVis} STREQUAL PRIVATE)
|
||||
__bundleMessage(FATAL_ERROR "Error: Called target_bundle_libraries with unknown visibility ${linkVis}. Must be either \"PUBLIC\" or \"PRIVATE\".")
|
||||
endif()
|
||||
|
||||
# TODO: Allow direct insertion of absolute file paths when CMAKE_LINK_LIBRARIES_ONLY_TARGETS is off, instead of always forcing targets.
|
||||
if (NOT TARGET ${mainLib})
|
||||
__bundleMessage(FATAL_ERROR "Error: Called target_bundle_libraries for target ${mainLib}, but no target with that name is known.")
|
||||
endif()
|
||||
|
||||
__get_lib_type(${mainLib} mainLibType)
|
||||
if (NOT mainLibType STREQUAL STATIC_LIBRARY)
|
||||
__bundleMessage(FATAL_ERROR "Error: Called target_bundle_libraries for target ${mainLib}, but it is not a static library.")
|
||||
endif()
|
||||
|
||||
get_target_property(isMainLibImported ${mainLib} IMPORTED)
|
||||
if(isMainLibImported)
|
||||
__bundleMessage(FATAL_ERROR "Error: Called target_bundle_libraries for target ${mainLib}, but it is an imported target.")
|
||||
endif()
|
||||
|
||||
if(NOT ARGN)
|
||||
__bundleMessage(WARNING "Called target_bundle_libraries with no libraries for target ${mainLib}")
|
||||
endif()
|
||||
|
||||
__bundleMessage(DEBUG "Bundling ${ARGN} into ${mainLib}")
|
||||
|
||||
get_target_property(templatePath ${mainLib} BUNDLED_LIBRARY_TEMPLATE_PATH)
|
||||
if(NOT EXISTS ${templatePath})
|
||||
if(MSVC)
|
||||
set(templatePath ${PROJECT_BINARY_DIR}/archive-${mainLib}.bat.template)
|
||||
else()
|
||||
set(templatePath ${PROJECT_BINARY_DIR}/archive-${mainLib}.mri.template)
|
||||
endif()
|
||||
__bundleMessage(STATUS "Registering default script path ${templatePath} for target ${mainLib}")
|
||||
__registerTargetForBundling(${mainLib} ${templatePath})
|
||||
endif()
|
||||
|
||||
get_target_property(bundledLibs ${mainLib} BUNDLE_LIBRARIES)
|
||||
if (NOT bundledLibs)
|
||||
set(bundledLibs "")
|
||||
endif()
|
||||
|
||||
foreach(bundledLib IN LISTS ARGN)
|
||||
unwrapAlias(${bundledLib} bundledLib)
|
||||
__get_lib_type(${bundledLib} bundledLibType)
|
||||
|
||||
if(${bundledLibType} STREQUAL INTERFACE_LIBRARY)
|
||||
# Interface libraries are not bundled, since they do not contain any static libraries.
|
||||
# Their dependencies will be bundled recursively in the next step.
|
||||
continue()
|
||||
endif()
|
||||
|
||||
if (NOT ${bundledLibType} STREQUAL STATIC_LIBRARY AND NOT ${bundledLibType} STREQUAL OBJECT_LIBRARY)
|
||||
__bundleMessage(FATAL_ERROR "Attempted to bundle ${bundledLibType} library ${bundledLib} into target ${mainLib} (only static and object libraries may be bundled)")
|
||||
endif()
|
||||
|
||||
if (${bundledLibType} STREQUAL STATIC_LIBRARY)
|
||||
list(APPEND bundledLibs ${bundledLib})
|
||||
else()
|
||||
# Exclude object libs from the BUNDLE_LIBRARIES property as they get added into the static lib using normal means.
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
list(REMOVE_DUPLICATES bundledLibs)
|
||||
set_target_properties(${mainLib}
|
||||
PROPERTIES BUNDLE_LIBRARIES "${bundledLibs}"
|
||||
)
|
||||
|
||||
foreach(bundledLib IN LISTS ARGN)
|
||||
unwrapAlias(${bundledLib} bundledLib)
|
||||
__get_lib_type(${bundledLib} bundledLibType)
|
||||
|
||||
# Recursively bundle all static dependencies of each lib to be bundled.
|
||||
get_target_property(depLibs ${bundledLib} LINK_LIBRARIES)
|
||||
__bundleRecursiveDeps(${mainLib} ${linkVis} ${depLibs})
|
||||
|
||||
# Since we want the main library to be linkable standalone, we need both the LINK_LIBRARIES and INTERFACE_LINK_LIBRARIES bundled in.
|
||||
# Otherwise, private static dependencies may be lost.
|
||||
get_target_property(depLibs ${bundledLib} INTERFACE_LINK_LIBRARIES)
|
||||
__bundleRecursiveDeps(${mainLib} ${linkVis} ${depLibs})
|
||||
|
||||
# Use `target_link_libraries` to propagate INTERFACE definitions from bundled libs
|
||||
# BUILD_LOCAL_INTERFACE prevents clients from seeing this internal link relationship
|
||||
# COMPILE_ONLY prevents the bundled lib from appearing in the link command redundantly
|
||||
if (${bundledLibType} STREQUAL STATIC_LIBRARY OR ${bundledLibType} STREQUAL INTERFACE_LIBRARY)
|
||||
target_link_libraries(${mainLib} ${linkVis}
|
||||
$<BUILD_LOCAL_INTERFACE:$<COMPILE_ONLY:${bundledLib}>>
|
||||
)
|
||||
else()
|
||||
# Include Object Libraries as full libraries, since they do not get added by the bundling stage.
|
||||
# To do this without breaking the link dependency logic, we need to steal the target objects and link the lib as local + compile only.
|
||||
target_link_libraries(${mainLib} ${linkVis}
|
||||
$<BUILD_LOCAL_INTERFACE:$<COMPILE_ONLY:${bundledLib}>>
|
||||
)
|
||||
target_sources(${mainLib} PRIVATE $<TARGET_OBJECTS:${bundledLib}>)
|
||||
endif()
|
||||
|
||||
# require that bundled libs are built before we try to bundle them
|
||||
add_dependencies(${mainLib} ${bundledLib})
|
||||
endforeach()
|
||||
|
||||
if(NOT EXISTS ${templatePath})
|
||||
__bundleMessage(FATAL_ERROR "Template file ${templatePath} for target ${mainLib} does not exist.")
|
||||
endif()
|
||||
endfunction()
|
||||
@@ -0,0 +1,38 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
include_guard()
|
||||
|
||||
set(_cccl_default_repo "https://github.com/NVIDIA/cccl.git")
|
||||
|
||||
|
||||
set(CCCL_REPO ${_cccl_default_repo} CACHE STRING "The base project URL to FetchContent_Declare for CCCL" )
|
||||
set(CCCL_TAG "v3.4.0-rc0" CACHE STRING "The commit hash to FetchContent_Declare for CCCL")
|
||||
|
||||
# We use this directory to ensure we only fetch a single copy of dependencies, even between builds.
|
||||
# $HOME/storage is expected to be mounted from the host for developers.
|
||||
set(TRT_THIRD_PARTY_DL_DIR "$ENV{HOME}/storage" CACHE PATH "Directory to download third party dependencies to")
|
||||
|
||||
FetchContent_Declare(
|
||||
cccl
|
||||
PREFIX "${CMAKE_BINARY_DIR}/third_party/cccl"
|
||||
GIT_REPOSITORY ${CCCL_REPO}
|
||||
GIT_TAG ${CCCL_TAG}
|
||||
GIT_SHALLOW TRUE
|
||||
SOURCE_DIR "${TRT_THIRD_PARTY_DL_DIR}/cccl/${CCCL_TAG}"
|
||||
EXCLUDE_FROM_ALL
|
||||
)
|
||||
|
||||
FetchContent_MakeAvailable(cccl)
|
||||
@@ -0,0 +1,49 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
include_guard()
|
||||
|
||||
set(NLOHMANN_JSON_REPO "https://github.com/nlohmann/json.git" CACHE STRING "The base project URL to FetchContent_Declare for nlohmann/json")
|
||||
set(NLOHMANN_JSON_TAG "v3.11.3" CACHE STRING "The git tag to FetchContent_Declare for nlohmann/json")
|
||||
|
||||
# We use this directory to ensure we only fetch a single copy of dependencies, even between builds.
|
||||
# $HOME/storage is expected to be mounted from the host for developers.
|
||||
set(TRT_THIRD_PARTY_DL_DIR "$ENV{HOME}/storage" CACHE PATH "Directory to download third party dependencies to")
|
||||
|
||||
# nlohmann/json's own CMake builds tests and install rules by default. Disable both
|
||||
# so our configure stays fast and our install tree doesn't pick up unrelated targets.
|
||||
set(JSON_BuildTests OFF CACHE INTERNAL "")
|
||||
set(JSON_Install OFF CACHE INTERNAL "")
|
||||
|
||||
include(FetchContent)
|
||||
|
||||
FetchContent_Declare(
|
||||
nlohmann_json
|
||||
PREFIX "${CMAKE_BINARY_DIR}/third_party/nlohmann_json"
|
||||
GIT_REPOSITORY ${NLOHMANN_JSON_REPO}
|
||||
GIT_TAG ${NLOHMANN_JSON_TAG}
|
||||
GIT_SHALLOW TRUE
|
||||
SOURCE_DIR "${TRT_THIRD_PARTY_DL_DIR}/nlohmann_json/${NLOHMANN_JSON_TAG}"
|
||||
EXCLUDE_FROM_ALL
|
||||
UPDATE_DISCONNECTED ${TRT_FETCH_CONTENT_UPDATES_DISCONNECTED}
|
||||
OVERRIDE_FIND_PACKAGE # downstream find_package(nlohmann_json) is satisfied from here.
|
||||
)
|
||||
FetchContent_MakeAvailable(nlohmann_json)
|
||||
|
||||
# Populate NLOHMANN_JSON_INCLUDE_DIRS for samples/common.
|
||||
if(nlohmann_json_SOURCE_DIR)
|
||||
set(NLOHMANN_JSON_INCLUDE_DIRS "${nlohmann_json_SOURCE_DIR}/include"
|
||||
CACHE PATH "nlohmann/json include directory (set by FetchNlohmannJson.cmake)")
|
||||
endif()
|
||||
@@ -0,0 +1,141 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# FindNCCL.cmake
|
||||
#
|
||||
# This module finds and imports the NCCL (NVIDIA Collective Communication Library) from system paths.
|
||||
#
|
||||
# Installation:
|
||||
# -------------
|
||||
# On Ubuntu/Debian:
|
||||
# apt-get install libnccl2 libnccl-dev
|
||||
# On RHEL/CentOS:
|
||||
# yum install libnccl libnccl-devel
|
||||
# From source:
|
||||
# See https://github.com/NVIDIA/nccl
|
||||
#
|
||||
# Input Variables (optional):
|
||||
# ---------------------------
|
||||
# NCCL_ROOT - Root directory where NCCL is installed (e.g., /usr, /usr/local)
|
||||
# NCCL_INCLUDE_DIR - Directory containing nccl.h
|
||||
# NCCL_LIBRARY - Full path to libnccl.so
|
||||
#
|
||||
# Provided Targets:
|
||||
# -----------------
|
||||
# NCCL::nccl - Imported NCCL library target
|
||||
#
|
||||
# Provided Variables:
|
||||
# -------------------
|
||||
# NCCL_FOUND - True if NCCL was found
|
||||
# NCCL_INCLUDE_DIRS - Include directories for NCCL
|
||||
# NCCL_LIBRARIES - NCCL libraries to link against
|
||||
# NCCL_VERSION - Version of NCCL found
|
||||
#
|
||||
# Usage:
|
||||
# ------
|
||||
# find_package(NCCL REQUIRED)
|
||||
# target_link_libraries(your_target PRIVATE NCCL::nccl)
|
||||
#
|
||||
# For custom installations:
|
||||
# cmake -DNCCL_ROOT=/path/to/nccl ..
|
||||
# or
|
||||
# set(NCCL_ROOT "/path/to/nccl")
|
||||
# find_package(NCCL REQUIRED)
|
||||
|
||||
# Search for NCCL header
|
||||
find_path(NCCL_INCLUDE_DIR
|
||||
NAMES nccl.h
|
||||
HINTS
|
||||
${NCCL_ROOT}
|
||||
${NCCL_ROOT}/include
|
||||
$ENV{NCCL_ROOT}
|
||||
$ENV{NCCL_ROOT}/include
|
||||
PATHS
|
||||
/usr/include
|
||||
/usr/local/include
|
||||
/usr/include/x86_64-linux-gnu
|
||||
/usr/local/cuda/include
|
||||
DOC "Path to NCCL include directory"
|
||||
)
|
||||
|
||||
# Search for NCCL library
|
||||
find_library(NCCL_LIBRARY
|
||||
NAMES nccl libnccl
|
||||
HINTS
|
||||
${NCCL_ROOT}
|
||||
${NCCL_ROOT}/lib
|
||||
${NCCL_ROOT}/lib64
|
||||
$ENV{NCCL_ROOT}
|
||||
$ENV{NCCL_ROOT}/lib
|
||||
$ENV{NCCL_ROOT}/lib64
|
||||
PATHS
|
||||
/usr/lib
|
||||
/usr/lib64
|
||||
/usr/local/lib
|
||||
/usr/local/lib64
|
||||
/usr/lib/x86_64-linux-gnu
|
||||
/usr/local/cuda/lib64
|
||||
DOC "Path to NCCL library"
|
||||
)
|
||||
|
||||
# Extract version from header if found
|
||||
if(NCCL_INCLUDE_DIR AND EXISTS "${NCCL_INCLUDE_DIR}/nccl.h")
|
||||
file(READ "${NCCL_INCLUDE_DIR}/nccl.h" NCCL_HEADER_CONTENTS)
|
||||
|
||||
string(REGEX MATCH "#define NCCL_MAJOR[ \t]+([0-9]+)" _ "${NCCL_HEADER_CONTENTS}")
|
||||
set(NCCL_VERSION_MAJOR "${CMAKE_MATCH_1}")
|
||||
|
||||
string(REGEX MATCH "#define NCCL_MINOR[ \t]+([0-9]+)" _ "${NCCL_HEADER_CONTENTS}")
|
||||
set(NCCL_VERSION_MINOR "${CMAKE_MATCH_1}")
|
||||
|
||||
string(REGEX MATCH "#define NCCL_PATCH[ \t]+([0-9]+)" _ "${NCCL_HEADER_CONTENTS}")
|
||||
set(NCCL_VERSION_PATCH "${CMAKE_MATCH_1}")
|
||||
|
||||
if(NCCL_VERSION_MAJOR AND NCCL_VERSION_MINOR AND NCCL_VERSION_PATCH)
|
||||
set(NCCL_VERSION "${NCCL_VERSION_MAJOR}.${NCCL_VERSION_MINOR}.${NCCL_VERSION_PATCH}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Standard FindPackage handling
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(NCCL
|
||||
REQUIRED_VARS NCCL_LIBRARY NCCL_INCLUDE_DIR
|
||||
VERSION_VAR NCCL_VERSION
|
||||
)
|
||||
|
||||
if(NCCL_FOUND)
|
||||
set(NCCL_INCLUDE_DIRS ${NCCL_INCLUDE_DIR})
|
||||
set(NCCL_LIBRARIES ${NCCL_LIBRARY})
|
||||
|
||||
# Create imported target if it doesn't exist
|
||||
if(NOT TARGET NCCL::nccl)
|
||||
add_library(NCCL::nccl SHARED IMPORTED)
|
||||
set_target_properties(NCCL::nccl PROPERTIES
|
||||
IMPORTED_LOCATION "${NCCL_LIBRARY}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${NCCL_INCLUDE_DIR}"
|
||||
)
|
||||
|
||||
# Get library directory for installation purposes
|
||||
get_filename_component(NCCL_LIB_DIR "${NCCL_LIBRARY}" DIRECTORY)
|
||||
message(STATUS "Found NCCL ${NCCL_VERSION} at ${NCCL_LIB_DIR}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
mark_as_advanced(
|
||||
NCCL_INCLUDE_DIR
|
||||
NCCL_LIBRARY
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
include_guard()
|
||||
|
||||
if(NOT TARGET dl)
|
||||
# libdl is included in the system library on Windows and QNX.
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
find_library(DL_LIB_PATH
|
||||
NAMES ${CMAKE_DL_LIBS}
|
||||
REQUIRED
|
||||
)
|
||||
|
||||
message(STATUS "Creating imported target 'dl' for ${DL_LIB_PATH}")
|
||||
add_library(dl SHARED IMPORTED)
|
||||
set_target_properties(dl PROPERTIES IMPORTED_LOCATION "${DL_LIB_PATH}")
|
||||
else()
|
||||
message(STATUS "Creating no-op target 'dl' since libdl is not available on this platform.")
|
||||
add_library(dl INTERFACE) # Add a fake dl target so we can still call target_link_libraries without error, even though it's a no-op.
|
||||
endif()
|
||||
endif()
|
||||
@@ -0,0 +1,144 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This module provides functionality to install imported shared libraries
|
||||
# while properly resolving and installing complete symlink chains.
|
||||
#
|
||||
# This is particularly useful for system libraries that use versioned symlinks
|
||||
# (e.g., libfoo.so -> libfoo.so.1 -> libfoo.so.1.2.3).
|
||||
|
||||
# Copies imported shared library files (including versioned symlinks) into the
|
||||
# build library output directory so tests using LD_LIBRARY_PATH find them without
|
||||
# a cmake --install step. Runs at configure time via file(COPY).
|
||||
#
|
||||
# \param targets One or more CMake imported shared-library targets to copy.
|
||||
# \param destination Destination directory (default: resolved CMAKE_LIBRARY_OUTPUT_DIRECTORY)
|
||||
function(copyImportedLibrariesToBuildTree)
|
||||
set(oneValueArgs DESTINATION)
|
||||
set(multiValueArgs TARGETS)
|
||||
cmake_parse_arguments(ARG "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
|
||||
|
||||
if(NOT ARG_TARGETS)
|
||||
message(FATAL_ERROR "copyImportedLibrariesToBuildTree requires at least one target.")
|
||||
endif()
|
||||
|
||||
# Resolve any $<CONFIG> generator expression in CMAKE_LIBRARY_OUTPUT_DIRECTORY.
|
||||
# TRT uses single-config generators (Ninja/Makefiles), so CMAKE_BUILD_TYPE is always set.
|
||||
if(ARG_DESTINATION)
|
||||
set(_dest "${ARG_DESTINATION}")
|
||||
elseif(CMAKE_BUILD_TYPE)
|
||||
string(REPLACE "$<CONFIG>" "${CMAKE_BUILD_TYPE}" _dest "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
|
||||
else()
|
||||
string(REPLACE "$<CONFIG>/" "" _dest "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
|
||||
endif()
|
||||
|
||||
foreach(target_name IN LISTS ARG_TARGETS)
|
||||
if(NOT TARGET ${target_name})
|
||||
message(FATAL_ERROR "Target ${target_name} does not exist.")
|
||||
endif()
|
||||
|
||||
get_target_property(target_type ${target_name} TYPE)
|
||||
if(NOT target_type MATCHES "SHARED_LIBRARY|UNKNOWN_LIBRARY")
|
||||
message(FATAL_ERROR "Target ${target_name} is not a shared library (type: ${target_type})")
|
||||
endif()
|
||||
|
||||
# IMPORTED_LOCATION may be unset for config-specific imported targets; fall back to
|
||||
# IMPORTED_LOCATION_<CONFIG>.
|
||||
get_target_property(target_loc ${target_name} IMPORTED_LOCATION)
|
||||
if(NOT target_loc)
|
||||
string(TOUPPER "${CMAKE_BUILD_TYPE}" _config_upper)
|
||||
get_target_property(target_loc ${target_name} "IMPORTED_LOCATION_${_config_upper}")
|
||||
endif()
|
||||
if(NOT target_loc)
|
||||
message(FATAL_ERROR "Target ${target_name} has no IMPORTED_LOCATION or IMPORTED_LOCATION_<CONFIG>.")
|
||||
endif()
|
||||
|
||||
get_filename_component(target_dir "${target_loc}" DIRECTORY)
|
||||
get_filename_component(target_base "${target_loc}" NAME_WE)
|
||||
|
||||
file(GLOB target_libs "${target_dir}/${target_base}${CMAKE_SHARED_LIBRARY_SUFFIX}*")
|
||||
if(target_libs)
|
||||
file(MAKE_DIRECTORY "${_dest}")
|
||||
file(COPY ${target_libs} DESTINATION "${_dest}")
|
||||
message(STATUS "Copied ${target_name} to build tree: ${_dest}")
|
||||
else()
|
||||
message(WARNING "copyImportedLibrariesToBuildTree: no libraries found for ${target_name} in ${target_dir}")
|
||||
endif()
|
||||
endforeach()
|
||||
endfunction()
|
||||
|
||||
# Installs an imported library target with all its symlinks.
|
||||
#
|
||||
# \param targets One or more CMake targets to install. Targets must be a shared library (or an unknown library pointing to a shared library).
|
||||
# \param destination Destination directory relative to CMAKE_INSTALL_PREFIX (default: lib)
|
||||
# \param component Installation component name (optional)
|
||||
#
|
||||
# This function attempts to resolve symlinks by globbing for libname.so* in the directory where the target is located.
|
||||
function(installImportedLibraries)
|
||||
set(options)
|
||||
set(oneValueArgs DESTINATION COMPONENT)
|
||||
set(multiValueArgs TARGETS)
|
||||
cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
|
||||
|
||||
# Set defaults
|
||||
if(NOT ARG_DESTINATION)
|
||||
# On Windows, shared libraries (dlls) go to the bin dir.
|
||||
# Since this function only handles shared libraries, we can simply default to bin/ here.
|
||||
if(WIN32)
|
||||
set(ARG_DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||
else()
|
||||
set(ARG_DESTINATION ${CMAKE_INSTALL_LIBDIR})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ARG_COMPONENT)
|
||||
set(INSTALL_COMPONENT_INPUT COMPONENT ${ARG_COMPONENT}) # Don't quote this, or the component will fail to eval in the install() command.
|
||||
else()
|
||||
set(INSTALL_COMPONENT_INPUT "")
|
||||
endif()
|
||||
|
||||
if(NOT ARG_TARGETS)
|
||||
message(FATAL_ERROR "installImportedLibraries requires at least one target to install.")
|
||||
endif()
|
||||
|
||||
foreach(target_name IN LISTS ARG_TARGETS)
|
||||
if(NOT TARGET ${target_name})
|
||||
message(FATAL_ERROR "Target ${target_name} does not exist")
|
||||
endif()
|
||||
|
||||
get_target_property(target_type ${target_name} TYPE)
|
||||
if(NOT target_type MATCHES "MODULE_LIBRARY|SHARED_LIBRARY|UNKNOWN_LIBRARY")
|
||||
message(FATAL_ERROR "Target ${target_name} is not a shared library (type: ${target_type})")
|
||||
endif()
|
||||
|
||||
# Install all library files and symlinks directly
|
||||
install(CODE "
|
||||
# Resolve where the target is located at install time so we can grab any adjacent symlinks.
|
||||
# Trying to do so earlier will require knowing the build mode at configure time (which we don't).
|
||||
get_filename_component(TARGET_DIR \$<TARGET_FILE:${target_name}> DIRECTORY)
|
||||
get_filename_component(TARGET_BASE_NAME \$<TARGET_FILE:${target_name}> NAME_WE)
|
||||
|
||||
set(TARGET_LIBS_EXPR \"\${TARGET_DIR}/\${TARGET_BASE_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}*\")
|
||||
|
||||
file(GLOB TARGET_LIBS \${TARGET_LIBS_EXPR})
|
||||
file(INSTALL \${TARGET_LIBS}
|
||||
DESTINATION \"\${CMAKE_INSTALL_PREFIX}/${ARG_DESTINATION}\"
|
||||
FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
|
||||
)
|
||||
"
|
||||
${INSTALL_COMPONENT_INPUT}
|
||||
)
|
||||
endforeach()
|
||||
endfunction()
|
||||
@@ -0,0 +1,97 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
include_guard()
|
||||
include(GNUInstallDirs)
|
||||
|
||||
# Install one or more targets including PDB files on Windows/MSVC
|
||||
# Usage:
|
||||
# installLibraries(
|
||||
# TARGETS target1 [target2 ...]
|
||||
# [COMPONENT component] # Optional component name for packaging
|
||||
# [EXPORT export] # Optional export set name.
|
||||
# [CONFIGURATIONS config1 [config2 ...]] # Optional configurations to install
|
||||
# )
|
||||
function(installLibraries)
|
||||
cmake_parse_arguments(
|
||||
ARG # Prefix for parsed args
|
||||
"OPTIONAL;RUNTIME_ONLY" # Options (flags)
|
||||
"COMPONENT;EXPORT" # Single value args
|
||||
"TARGETS;CONFIGURATIONS" # Multi-value args
|
||||
${ARGN}
|
||||
)
|
||||
|
||||
# Validate required arguments
|
||||
if(NOT ARG_TARGETS)
|
||||
message(FATAL_ERROR "installLibrary() requires TARGETS argument")
|
||||
endif()
|
||||
|
||||
# Prepare optional arguments for regular install command
|
||||
if(ARG_COMPONENT)
|
||||
set(component_arg COMPONENT ${ARG_COMPONENT})
|
||||
endif()
|
||||
|
||||
if(ARG_CONFIGURATIONS)
|
||||
set(config_arg CONFIGURATIONS ${ARG_CONFIGURATIONS})
|
||||
endif()
|
||||
|
||||
if(ARG_OPTIONAL)
|
||||
set(optional_arg OPTIONAL)
|
||||
endif()
|
||||
|
||||
if(ARG_EXPORT)
|
||||
set(export_arg EXPORT ${ARG_EXPORT})
|
||||
endif()
|
||||
|
||||
# When RUNTIME_ONLY is passed, we only want to install .dll files.
|
||||
# Instead of also installing the import library (.lib) files.
|
||||
# This is only relevant on Windows since Linux doesn't have this distinction.
|
||||
if(ARG_RUNTIME_ONLY AND WIN32)
|
||||
set(runtime_only_arg
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
)
|
||||
endif()
|
||||
|
||||
# Install the libraries
|
||||
install(
|
||||
TARGETS ${ARG_TARGETS}
|
||||
${optional_arg}
|
||||
${component_arg}
|
||||
${config_arg}
|
||||
${export_arg}
|
||||
${runtime_only_arg}
|
||||
)
|
||||
|
||||
# Install PDB files for MSVC builds
|
||||
if(MSVC)
|
||||
foreach(target ${ARG_TARGETS})
|
||||
# Get target type (SHARED_LIBRARY, STATIC_LIBRARY, EXECUTABLE)
|
||||
get_target_property(target_type ${target} TYPE)
|
||||
|
||||
# For shared libraries and executables, PDBs are placed alongside the binaries
|
||||
if(target_type STREQUAL "SHARED_LIBRARY" OR target_type STREQUAL "EXECUTABLE")
|
||||
# Use generator expression to get the PDB file path
|
||||
install(
|
||||
FILES "$<TARGET_PDB_FILE:${target}>"
|
||||
DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
${component_arg}
|
||||
CONFIGURATIONS Debug RelWithDebInfo
|
||||
OPTIONAL
|
||||
)
|
||||
endif()
|
||||
endforeach()
|
||||
endif()
|
||||
endfunction()
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# Contains constants for the various platform names TRT supports.
|
||||
|
||||
set(TRT_PLATFORM_X86
|
||||
"x86_64"
|
||||
CACHE INTERNAL "Linux")
|
||||
set(TRT_PLATFORM_AARCH64
|
||||
"aarch64"
|
||||
CACHE INTERNAL "ARM Linux")
|
||||
set(TRT_PLATFORM_QNX
|
||||
"qnx"
|
||||
CACHE INTERNAL "QNX")
|
||||
set(TRT_PLATFORM_QNX_SAFE
|
||||
"qnx-safe"
|
||||
CACHE INTERNAL "QNX Safe")
|
||||
set(TRT_PLATFORM_WIN10
|
||||
"win10"
|
||||
CACHE INTERNAL "Windows 10")
|
||||
|
||||
|
||||
# Checks if the current build platform matches any of the passed (ARGN) platforms.
|
||||
#
|
||||
# \param outVar The output variable name.
|
||||
# \param argn The list of platforms to check against.
|
||||
# \returns TRUE if TRT_BUILD_PLATFORM matches any of the platforms, FALSE otherwise.
|
||||
function(checkPlatform outVar)
|
||||
if(NOT DEFINED TRT_BUILD_PLATFORM)
|
||||
message(FATAL_ERROR "checkPlatform was called before TRT_BUILD_PLATFORM was defined!")
|
||||
endif()
|
||||
|
||||
set(isPlatform FALSE)
|
||||
foreach(platform IN LISTS ARGN)
|
||||
if(${platform} STREQUAL ${TRT_BUILD_PLATFORM})
|
||||
set(isPlatform TRUE)
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
set(${outVar} ${isPlatform} PARENT_SCOPE)
|
||||
endfunction()
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# \brief Converts a SM string (i.e. 86+abc) into the numeric SM version (i.e. 86).
|
||||
# \returns the sm in the name specified by OUT_VAR.
|
||||
function(get_numeric_sm SM OUT_VAR)
|
||||
# Convert the SM string to a numeric value
|
||||
if(${SM} MATCHES "^([0-9]+).*$")
|
||||
set(${OUT_VAR} ${CMAKE_MATCH_1} PARENT_SCOPE)
|
||||
else()
|
||||
message(FATAL_ERROR "Invalid SM version: ${SM}")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
# \brief Converts the CMAKE_CUDA_ARCHITECTURES list into a list of numeric SM values.
|
||||
# \returns the list in the name specified by OUT_VAR.
|
||||
function(get_all_numeric_sms OUT_VAR)
|
||||
set(ALL_NUMERIC_SMS "")
|
||||
foreach(SM IN LISTS CMAKE_CUDA_ARCHITECTURES)
|
||||
get_numeric_sm(${SM} "SM")
|
||||
list(APPEND ALL_NUMERIC_SMS ${SM})
|
||||
endforeach()
|
||||
set(${OUT_VAR} ${ALL_NUMERIC_SMS} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# \brief Converts the list returned by get_all_numeric_sms into a list of arch values.
|
||||
# \returns the list in the name specified by OUT_VAR for native platform and OUT_VAR_CROSS for cross OS support. e.g. ptx, sm75, sm80, sm86, sm89, sm100, sm120.
|
||||
function(get_all_fatbin_archs OUT_VAR OUT_VAR_CROSS)
|
||||
# Use get_all_numeric_sms to get SM values and convert them to sm-prefixed format
|
||||
set(ARCH_LIST "")
|
||||
set(ARCH_LIST_CROSS "")
|
||||
get_all_numeric_sms(NUMERIC_SMS)
|
||||
foreach(SM IN LISTS NUMERIC_SMS)
|
||||
list(APPEND ARCH_LIST "sm${SM}")
|
||||
endforeach()
|
||||
|
||||
# Note: sm89 it is missing in NUMERIC_SMS since TRT treats sm89 as sm86.
|
||||
# We should add sm89 to the list to generate the builder resource for sm89.
|
||||
# If only sm86 is in the list, it means this build only supports sm86,
|
||||
# so no need to add sm89.
|
||||
list(FIND ARCH_LIST "sm86" SM86_INDEX)
|
||||
list(FIND ARCH_LIST "sm89" SM89_INDEX)
|
||||
list(LENGTH ARCH_LIST ARCH_LIST_COUNT)
|
||||
if(${SM86_INDEX} GREATER_EQUAL 0 AND ${SM89_INDEX} EQUAL -1 AND ${ARCH_LIST_COUNT} GREATER 1)
|
||||
list(APPEND ARCH_LIST "sm89")
|
||||
endif()
|
||||
|
||||
# On SBSA exclude sm87 explicitly since it is proxied to sm86.
|
||||
if(${TRT_IS_ARM_SERVER})
|
||||
list(FILTER ARCH_LIST EXCLUDE REGEX "sm87")
|
||||
endif()
|
||||
|
||||
# Exclude sm103 explicitly since it is proxied to sm100.
|
||||
list(FILTER ARCH_LIST EXCLUDE REGEX "sm103")
|
||||
# Exclude sm121 explicitly since it is proxied to sm120.
|
||||
list(FILTER ARCH_LIST EXCLUDE REGEX "sm121")
|
||||
|
||||
# There is also a klib which only contains PTX code.
|
||||
list(APPEND ARCH_LIST "ptx")
|
||||
|
||||
|
||||
set(ARCH_LIST_CROSS ${ARCH_LIST})
|
||||
set(${OUT_VAR} ${ARCH_LIST} PARENT_SCOPE)
|
||||
set(${OUT_VAR_CROSS} ${ARCH_LIST_CROSS} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# Certain cubins are binary compatible between different SM versions, so they are reused.
|
||||
# This function checks if a SM-named file should be compiled based on current SM enablement.
|
||||
# Specifically, the SM80 files are compiled if either 80, 86, or 89 are enabled.
|
||||
function(should_compile_kernel SM OUT_VAR)
|
||||
get_all_numeric_sms(__TRT_NUMERIC_CUDA_ARCHS)
|
||||
# If the target SM is any of 80/86/89, we need to check if any of those are enabled in __TRT_NUMERIC_CUDA_ARCHS.
|
||||
if((${SM} EQUAL 80) OR (${SM} EQUAL 86) OR (${SM} EQUAL 89))
|
||||
list(FIND __TRT_NUMERIC_CUDA_ARCHS 80 SM80_INDEX)
|
||||
list(FIND __TRT_NUMERIC_CUDA_ARCHS 86 SM86_INDEX)
|
||||
list(FIND __TRT_NUMERIC_CUDA_ARCHS 89 SM89_INDEX)
|
||||
if((NOT ${SM80_INDEX} EQUAL -1) OR
|
||||
(NOT ${SM86_INDEX} EQUAL -1) OR
|
||||
(NOT ${SM89_INDEX} EQUAL -1)
|
||||
)
|
||||
set(${OUT_VAR} TRUE PARENT_SCOPE)
|
||||
else()
|
||||
set(${OUT_VAR} FALSE PARENT_SCOPE)
|
||||
endif()
|
||||
else()
|
||||
list(FIND __TRT_NUMERIC_CUDA_ARCHS ${SM} SM_INDEX)
|
||||
if (NOT ${SM_INDEX} EQUAL -1)
|
||||
set(${OUT_VAR} TRUE PARENT_SCOPE)
|
||||
else()
|
||||
set(${OUT_VAR} FALSE PARENT_SCOPE)
|
||||
endif()
|
||||
endif()
|
||||
endfunction()
|
||||
@@ -0,0 +1,58 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Utility module for smoke testing a static library.
|
||||
# This allows us to verify that the static library can be linked without any unexpected dependencies.
|
||||
|
||||
include_guard()
|
||||
|
||||
define_property(TARGET
|
||||
PROPERTY STATIC_LIBRARY_SMOKE_TEST_SOURCE_CODE
|
||||
BRIEF_DOCS "Source code to use when executing the static library smoke test. Defaults to 'int main() { return 0; }'"
|
||||
)
|
||||
|
||||
# Creates a new target that links the given static library and builds a simple executable.
|
||||
# The executable will be added to the ALL target if the static library target is also in the ALL target.
|
||||
function(smoke_test_static_lib static_lib_target)
|
||||
get_target_property(static_lib_type ${static_lib_target} TYPE)
|
||||
if(NOT static_lib_type STREQUAL "STATIC_LIBRARY")
|
||||
message(FATAL_ERROR "Target ${static_lib_target} is not a static library.")
|
||||
endif()
|
||||
|
||||
set(smoke_test_target "${static_lib_target}_smoke_test")
|
||||
get_target_property(smoke_test_src ${static_lib_target} STATIC_LIBRARY_SMOKE_TEST_SOURCE_CODE)
|
||||
# Handle NOTFOUND sentinel and empty string; otherwise treat property as source content.
|
||||
if(NOT smoke_test_src)
|
||||
set(smoke_test_src "int main() { return 0; }\n")
|
||||
endif()
|
||||
|
||||
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${smoke_test_target}.cpp" "${smoke_test_src}")
|
||||
|
||||
add_executable(${smoke_test_target} "${CMAKE_CURRENT_BINARY_DIR}/${smoke_test_target}.cpp")
|
||||
|
||||
target_link_libraries(${smoke_test_target} PRIVATE $<LINK_LIBRARY:WHOLE_ARCHIVE,${static_lib_target}>)
|
||||
# cuDLA is not available as a static library, so we link it dynamically
|
||||
if(${TRT_BUILD_ENABLE_DLA} AND TARGET CUDA::cudla)
|
||||
target_link_libraries(${smoke_test_target} PRIVATE CUDA::cudla)
|
||||
endif()
|
||||
|
||||
# If the static library target is in the ALL target, add the smoke test target to the ALL target as well.
|
||||
get_target_property(static_lib_excluded_from_all ${static_lib_target} EXCLUDE_FROM_ALL)
|
||||
if(static_lib_excluded_from_all)
|
||||
set_target_properties(${smoke_test_target} PROPERTIES
|
||||
EXCLUDE_FROM_ALL ON
|
||||
)
|
||||
endif()
|
||||
endfunction()
|
||||
@@ -0,0 +1,97 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This file provides functionality to create stub libraries from existing CMake targets
|
||||
# by calling the existing stubify.sh script.
|
||||
#
|
||||
# Stub libraries contain the same exported symbols as the original shared library but with empty function bodies,
|
||||
# removing all dependencies while maintaining the same API surface for linking purposes.
|
||||
|
||||
# Creates a stub library from an existing CMake target by calling stubify.sh.
|
||||
#
|
||||
# \param target_name Name of an existing CMake shared library target
|
||||
#
|
||||
# Creates a new target called ${target_name}_stub that contains empty implementations
|
||||
# of all exported symbols from the original target. The stub library is also installed
|
||||
# to lib/stubs during the install phase.
|
||||
function(create_stub_lib target_name)
|
||||
if(MSVC)
|
||||
message(FATAL_ERROR "Creating stub libs is not supported on Windows.")
|
||||
endif()
|
||||
|
||||
if(NOT TARGET ${target_name})
|
||||
message(FATAL_ERROR "Target ${target_name} does not exist")
|
||||
endif()
|
||||
|
||||
get_target_property(target_type ${target_name} TYPE)
|
||||
if(NOT target_type STREQUAL "SHARED_LIBRARY")
|
||||
message(FATAL_ERROR "Target ${target_name} is not a shared library")
|
||||
endif()
|
||||
|
||||
set(stub_target "${target_name}_stub")
|
||||
|
||||
# The stub output should be the same base name of the library with a _stub suffix.
|
||||
# We need to read the OUTPUT_NAME of the target, and fallback to the target name otherwise.
|
||||
get_target_property(output_name ${target_name} OUTPUT_NAME)
|
||||
if(NOT output_name)
|
||||
set(output_name ${target_name})
|
||||
endif()
|
||||
set(stub_output "${CMAKE_CURRENT_BINARY_DIR}/stubs/lib${output_name}.so")
|
||||
|
||||
# Find the stubify.sh script
|
||||
find_file(STUBIFY_SCRIPT stubify.sh
|
||||
PATHS ${CMAKE_SOURCE_DIR}/scripts
|
||||
NO_DEFAULT_PATH
|
||||
REQUIRED
|
||||
)
|
||||
|
||||
# Create a custom target that calls stubify.sh
|
||||
add_custom_command(
|
||||
OUTPUT ${stub_output}
|
||||
COMMAND ${CMAKE_COMMAND} -E env
|
||||
CC=${CMAKE_C_COMPILER}
|
||||
CC_ARGS=${STUBIFY_CC_ARGS}
|
||||
--
|
||||
${STUBIFY_SCRIPT} $<TARGET_FILE:${target_name}> ${stub_output}
|
||||
DEPENDS $<TARGET_FILE:${target_name}> ${STUBIFY_SCRIPT}
|
||||
COMMENT "Creating stub library ${stub_output} for ${target_name}"
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
# Create an imported library target for the stub
|
||||
add_library(${stub_target} SHARED IMPORTED)
|
||||
set_target_properties(${stub_target} PROPERTIES
|
||||
IMPORTED_LOCATION ${stub_output}
|
||||
)
|
||||
|
||||
# Create a custom target to ensure the stub gets built
|
||||
add_custom_target(${stub_target}_build
|
||||
ALL
|
||||
DEPENDS ${stub_output}
|
||||
)
|
||||
|
||||
# Make sure the stub builds after the original
|
||||
add_dependencies(${stub_target}_build ${target_name})
|
||||
|
||||
# Make the imported target depend on the build target
|
||||
add_dependencies(${stub_target} ${stub_target}_build)
|
||||
|
||||
# Install the stub library to lib/stubs
|
||||
install(FILES ${stub_output}
|
||||
DESTINATION lib/stubs
|
||||
COMPONENT external
|
||||
OPTIONAL
|
||||
)
|
||||
endfunction()
|
||||
@@ -0,0 +1,93 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## Helper file for management of symbol exports in shared libraries via linker scripts.
|
||||
## This allows controlling symbol visibility across different platforms without needing platform-specific code.
|
||||
|
||||
# Applies an export map to one or more targets (specified by TARGETS).
|
||||
# The export map file should be provided without extension; the appropriate extension will be added based on the platform (Windows: .def, Unix: .map).
|
||||
# If CONFIGURE_FIRST is set, the export map will be configured via CMake configure_file (@ONLY). The source file must have the additional extension .in.
|
||||
#
|
||||
# \param CONFIGURE_FIRST Optional flag to indicate that the export map file should be configured first.
|
||||
# \param EXPORT_MAP_FILE The base name of the export map file (without extension). If not provided, the function will look for a file matching the target's output name in the 'exports' directory.
|
||||
# \param BINARY_DIR Optional path to the binary directory where the configured file will be placed. If not set, the current binary directory is used.
|
||||
# \param TARGETS The targets to which the export map should be applied. Must not be empty.
|
||||
function(apply_export_map)
|
||||
cmake_parse_arguments(ARG "CONFIGURE_FIRST" "EXPORT_MAP_FILE;BINARY_DIR" "TARGETS" ${ARGN})
|
||||
|
||||
if(NOT ARG_TARGETS)
|
||||
message(FATAL_ERROR "apply_export_map: TARGETS argument must not be empty.")
|
||||
endif()
|
||||
|
||||
if(NOT ARG_BINARY_DIR)
|
||||
set(ARG_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
|
||||
endif()
|
||||
|
||||
if(MSVC)
|
||||
set(export_extension ".def")
|
||||
else()
|
||||
set(export_extension ".map")
|
||||
endif()
|
||||
|
||||
foreach(target IN LISTS ARG_TARGETS)
|
||||
if(ARG_EXPORT_MAP_FILE)
|
||||
cmake_path(IS_RELATIVE ARG_EXPORT_MAP_FILE is_relative)
|
||||
if(is_relative)
|
||||
set(export_map_file "${CMAKE_CURRENT_SOURCE_DIR}/${ARG_EXPORT_MAP_FILE}")
|
||||
else()
|
||||
set(export_map_file "${ARG_EXPORT_MAP_FILE}")
|
||||
endif()
|
||||
else()
|
||||
# Otherwise, default to searching for a file that matches the target's output name.
|
||||
get_target_property(output_name ${target} OUTPUT_NAME)
|
||||
if(NOT output_name)
|
||||
set(output_name ${target})
|
||||
endif()
|
||||
set(export_map_file "${CMAKE_CURRENT_SOURCE_DIR}/exports/${output_name}")
|
||||
endif()
|
||||
|
||||
if(ARG_CONFIGURE_FIRST)
|
||||
if(NOT EXISTS "${export_map_file}${export_extension}.in")
|
||||
message(FATAL_ERROR "apply_export_map: Expected to find export map template file '${export_map_file}${export_extension}.in' for configuration.")
|
||||
endif()
|
||||
get_filename_component(export_file_name ${export_map_file} NAME_WE)
|
||||
configure_file(
|
||||
"${export_map_file}${export_extension}.in"
|
||||
"${ARG_BINARY_DIR}/${export_file_name}${export_extension}"
|
||||
@ONLY
|
||||
)
|
||||
set(export_map_file "${ARG_BINARY_DIR}/${export_file_name}")
|
||||
endif()
|
||||
|
||||
if(NOT EXISTS "${export_map_file}${export_extension}")
|
||||
message(FATAL_ERROR "apply_export_map: Expected to find export map file '${export_map_file}${export_extension}'.")
|
||||
endif()
|
||||
|
||||
if(MSVC)
|
||||
# On Windows, use a .def file to control exported symbols
|
||||
target_sources(${target} PRIVATE "${export_map_file}${export_extension}")
|
||||
else()
|
||||
# On Unix-like systems, use a version script for symbol visibility
|
||||
target_link_options(${target} PRIVATE "LINKER:--version-script=${export_map_file}${export_extension}")
|
||||
# We also need to update the link dependencies. We don't need to do this on Windows since the .def file is a source file.
|
||||
get_target_property(link_deps ${target} LINK_DEPENDS)
|
||||
if(NOT link_deps)
|
||||
set(link_deps "")
|
||||
endif()
|
||||
list(APPEND link_deps "${export_map_file}${export_extension}")
|
||||
set_target_properties(${target} PROPERTIES LINK_DEPENDS "${link_deps}")
|
||||
endif()
|
||||
endforeach()
|
||||
endfunction()
|
||||
@@ -0,0 +1,43 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
include_guard()
|
||||
|
||||
# \brief Handles setting output names for libraries on Windows to include version information.
|
||||
# We have two conventions:
|
||||
# 1. For enterprise, shared libraries have a suffix _${major_version}
|
||||
# 2. For TRT-RTX, shared libraries have a suffix _${major_version}_${minor_version}
|
||||
# \note This is a no-op for non-Windows platforms.
|
||||
#
|
||||
# \param target_name The name of the target to update.
|
||||
# \param major_version The major version of the library.
|
||||
# \param minor_version The minor version of the library.
|
||||
function(update_windows_output_name target_name major_version minor_version)
|
||||
if(MSVC)
|
||||
get_target_property(tgt_output_name ${target_name} OUTPUT_NAME)
|
||||
if(NOT tgt_output_name)
|
||||
set(tgt_output_name ${target_name})
|
||||
endif()
|
||||
|
||||
if(${TRT_BUILD_WINML})
|
||||
set(tgt_output_name "${tgt_output_name}_${major_version}_${minor_version}")
|
||||
else()
|
||||
set(tgt_output_name "${tgt_output_name}_${major_version}")
|
||||
endif()
|
||||
|
||||
set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${tgt_output_name})
|
||||
message(STATUS "Updated output name for target '${target_name}' to '${tgt_output_name}'")
|
||||
endif()
|
||||
endfunction()
|
||||
@@ -0,0 +1,40 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
macro(find_library_create_target target_name lib libtype hints)
|
||||
message(STATUS "========================= Importing and creating target ${target_name} ==========================")
|
||||
message(STATUS "Looking for library ${lib}")
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
find_library(
|
||||
${lib}_LIB_PATH ${lib}${TRT_DEBUG_POSTFIX}
|
||||
HINTS ${hints}
|
||||
NO_DEFAULT_PATH)
|
||||
endif()
|
||||
find_library(
|
||||
${lib}_LIB_PATH ${lib}
|
||||
HINTS ${hints}
|
||||
NO_DEFAULT_PATH)
|
||||
find_library(${lib}_LIB_PATH ${lib})
|
||||
message(STATUS "Library that was found ${${lib}_LIB_PATH}")
|
||||
add_library(${target_name} ${libtype} IMPORTED)
|
||||
if(MSVC)
|
||||
set_property(TARGET ${target_name} PROPERTY IMPORTED_IMPLIB ${${lib}_LIB_PATH})
|
||||
else()
|
||||
set_property(TARGET ${target_name} PROPERTY IMPORTED_LOCATION ${${lib}_LIB_PATH})
|
||||
endif()
|
||||
message(STATUS "==========================================================================================")
|
||||
endmacro()
|
||||
@@ -0,0 +1,23 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
function(set_ifndef variable value)
|
||||
if(NOT DEFINED ${variable})
|
||||
set(${variable}
|
||||
${value}
|
||||
PARENT_SCOPE)
|
||||
endif()
|
||||
endfunction()
|
||||
@@ -0,0 +1,37 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set(CMAKE_SYSTEM_NAME Linux)
|
||||
set(CMAKE_SYSTEM_PROCESSOR aarch64)
|
||||
|
||||
set(TRT_PLATFORM_ID "aarch64")
|
||||
|
||||
set(CMAKE_C_COMPILER /usr/bin/aarch64-linux-gnu-gcc)
|
||||
set(CMAKE_CXX_COMPILER /usr/bin/aarch64-linux-gnu-g++)
|
||||
|
||||
set(CMAKE_C_FLAGS "" CACHE STRING "" FORCE)
|
||||
set(CMAKE_CXX_FLAGS "" CACHE STRING "" FORCE)
|
||||
|
||||
set(CMAKE_C_COMPILER_TARGET aarch64-linux-gnu)
|
||||
set(CMAKE_CXX_COMPILER_TARGET aarch64-linux-gnu)
|
||||
|
||||
set(CMAKE_C_COMPILER_FORCED TRUE)
|
||||
set(CMAKE_CXX_COMPILER_FORCED TRUE)
|
||||
|
||||
if(NOT DEFINED CUDAToolkit_ROOT)
|
||||
set(CUDAToolkit_ROOT /usr/local/cuda CACHE STRING "CUDA ROOT dir")
|
||||
endif()
|
||||
@@ -0,0 +1,37 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set(CMAKE_SYSTEM_NAME Linux)
|
||||
set(CMAKE_SYSTEM_PROCESSOR aarch64)
|
||||
|
||||
set(TRT_PLATFORM_ID "aarch64")
|
||||
|
||||
set(CMAKE_C_COMPILER /usr/bin/aarch64-linux-gnu-gcc)
|
||||
set(CMAKE_CXX_COMPILER /usr/bin/aarch64-linux-gnu-g++)
|
||||
|
||||
set(CMAKE_C_FLAGS "" CACHE STRING "" FORCE)
|
||||
set(CMAKE_CXX_FLAGS "" CACHE STRING "" FORCE)
|
||||
|
||||
set(CMAKE_C_COMPILER_TARGET aarch64-linux-gnu)
|
||||
set(CMAKE_CXX_COMPILER_TARGET aarch64-linux-gnu)
|
||||
|
||||
set(CMAKE_C_COMPILER_FORCED TRUE)
|
||||
set(CMAKE_CXX_COMPILER_FORCED TRUE)
|
||||
|
||||
# Use host nvcc
|
||||
set(CMAKE_CUDA_COMPILER /usr/local/cuda/bin/nvcc)
|
||||
set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "" FORCE)
|
||||
@@ -0,0 +1,35 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set(CMAKE_SYSTEM_NAME Linux)
|
||||
set(CMAKE_SYSTEM_PROCESSOR aarch64)
|
||||
|
||||
set(TRT_PLATFORM_ID "aarch64")
|
||||
|
||||
set(CMAKE_C_COMPILER /usr/bin/aarch64-linux-gnu-gcc)
|
||||
set(CMAKE_CXX_COMPILER /usr/bin/aarch64-linux-gnu-g++)
|
||||
|
||||
set(CMAKE_C_COMPILER_TARGET aarch64-linux-gnu)
|
||||
set(CMAKE_CXX_COMPILER_TARGET aarch64-linux-gnu)
|
||||
|
||||
set(CMAKE_C_COMPILER_FORCED TRUE)
|
||||
set(CMAKE_CXX_COMPILER_FORCED TRUE)
|
||||
|
||||
set(RT_LIB /usr/aarch64-linux-gnu/lib/librt.so)
|
||||
|
||||
set(CMAKE_CUDA_COMPILER /usr/local/cuda/bin/nvcc)
|
||||
set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "" FORCE)
|
||||
@@ -0,0 +1,38 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set(CMAKE_SYSTEM_NAME Linux)
|
||||
set(CMAKE_SYSTEM_PROCESSOR aarch64)
|
||||
|
||||
set(CMAKE_FIND_ROOT_PATH /usr/aarch64-linux-gnu/)
|
||||
|
||||
set(TRT_PLATFORM_ID "aarch64")
|
||||
|
||||
set(CMAKE_C_COMPILER /usr/bin/aarch64-linux-gnu-gcc)
|
||||
set(CMAKE_CXX_COMPILER /usr/bin/aarch64-linux-gnu-g++)
|
||||
|
||||
set(CMAKE_C_FLAGS "" CACHE STRING "" FORCE)
|
||||
set(CMAKE_CXX_FLAGS "" CACHE STRING "" FORCE)
|
||||
|
||||
set(CMAKE_C_COMPILER_TARGET aarch64-linux-gnu)
|
||||
set(CMAKE_CXX_COMPILER_TARGET aarch64-linux-gnu)
|
||||
|
||||
set(CMAKE_C_COMPILER_FORCED TRUE)
|
||||
set(CMAKE_CXX_COMPILER_FORCED TRUE)
|
||||
|
||||
set(CMAKE_CUDA_COMPILER /usr/local/cuda/bin/nvcc)
|
||||
set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "" FORCE)
|
||||
@@ -0,0 +1,137 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Setup basic fixtures required for x86 -> QNX cross compile
|
||||
set(CMAKE_SYSTEM_NAME QNX)
|
||||
set(CMAKE_SYSTEM_PROCESSOR aarch64)
|
||||
set(CMAKE_C_COMPILER_TARGET gcc_ntoaarch64le)
|
||||
set(CMAKE_CXX_COMPILER_TARGET gcc_ntoaarch64le_cxx)
|
||||
|
||||
# Ensure that the QNX toolchain location was provided.
|
||||
if(NOT DEFINED ENV{QNX_HOST} OR NOT DEFINED ENV{QNX_TARGET})
|
||||
message(FATAL_ERROR "The environment variables QNX_HOST and QNX_TARGET must both be provided to build on QNX.")
|
||||
endif()
|
||||
|
||||
set(QNX_HOST $ENV{QNX_HOST})
|
||||
set(QNX_TARGET $ENV{QNX_TARGET})
|
||||
set(CMAKE_SYSROOT $ENV{QNX_TARGET})
|
||||
|
||||
if(NOT EXISTS ${QNX_HOST} OR NOT EXISTS ${QNX_TARGET})
|
||||
message(FATAL_ERROR "The QNX_HOST (${QNX_HOST}) or QNX_TARGET (${QNX_TARGET}) paths do not exist.")
|
||||
endif()
|
||||
|
||||
if(NOT DEFINED ENV{CUDA})
|
||||
message(FATAL_ERROR "The environment variable CUDA must be defined to specify the CUDA version to use for QNX builds.")
|
||||
endif()
|
||||
|
||||
if($ENV{CUDA} MATCHES "cuda-11.4")
|
||||
set(QNX_VERSION 7.1.0 CACHE STRING "")
|
||||
set(QNX_GCC_VERSION "8.3.0" CACHE STRING "")
|
||||
else()
|
||||
set(QNX_VERSION 8.0.0 CACHE STRING "")
|
||||
set(QNX_GCC_VERSION "12.2.0" CACHE STRING "")
|
||||
endif()
|
||||
|
||||
set(QNX_TRIPLE aarch64-unknown-nto-qnx${QNX_VERSION})
|
||||
|
||||
if($ENV{USE_QCC})
|
||||
set(CMAKE_C_COMPILER ${QNX_HOST}/usr/bin/qcc)
|
||||
set(CMAKE_CXX_COMPILER ${QNX_HOST}/usr/bin/q++)
|
||||
|
||||
# QCC includes threads in the system library.
|
||||
# CMake picks this up correctly for the main cross compiler, but not QCC.
|
||||
set(CMAKE_HAVE_LIBC_PTHREAD TRUE)
|
||||
|
||||
# The correct platform objcopy is automatically picked up when using the cross g++, but not when using q++.
|
||||
find_program(QNX_OBJCOPY_PATH
|
||||
NAMES ${QNX_TRIPLE}-objcopy
|
||||
PATHS ${QNX_HOST}/usr/bin
|
||||
NO_DEFAULT_PATH
|
||||
REQUIRED
|
||||
)
|
||||
set(CMAKE_OBJCOPY ${QNX_OBJCOPY_PATH})
|
||||
else()
|
||||
set(CMAKE_C_COMPILER ${QNX_HOST}/usr/bin/${QNX_TRIPLE}-gcc)
|
||||
set(CMAKE_CXX_COMPILER ${QNX_HOST}/usr/bin/${QNX_TRIPLE}-g++)
|
||||
endif()
|
||||
|
||||
# CUDA Host compilation needs to always use the underlying cross-compiler, and not Q++
|
||||
set(CMAKE_CUDA_HOST_COMPILER ${QNX_HOST}/usr/bin/${QNX_TRIPLE}-g++)
|
||||
|
||||
# WAR: Some of the cuda headers are randomly split into this "thor" dir.
|
||||
if(${QNX_VERSION} VERSION_GREATER_EQUAL "8.0.0")
|
||||
# include_directories(/usr/local/cuda-safe-${TRT_CUDA_VERSION}/thor/targets/aarch64-qnx/include)
|
||||
endif()
|
||||
|
||||
# These linker behaviors aren't setup by default for QNX
|
||||
# They work the same as they would on Linux
|
||||
set(CMAKE_LINK_LIBRARY_USING_WHOLE_ARCHIVE
|
||||
"-Wl,--whole-archive <LIBRARY> -Wl,--no-whole-archive"
|
||||
)
|
||||
set(CMAKE_LINK_LIBRARY_USING_WHOLE_ARCHIVE_SUPPORTED TRUE)
|
||||
|
||||
set(CMAKE_LINK_GROUP_USING_RESCAN
|
||||
"LINKER:--start-group"
|
||||
"LINKER:--end-group"
|
||||
)
|
||||
set(CMAKE_LINK_GROUP_USING_RESCAN_SUPPORTED TRUE)
|
||||
|
||||
# Setup required flags for QNX compilation.
|
||||
set(TRT_MAGIC_QNX_FLAGS "-D_XOPEN_SOURCE=700 -D_POSIX_C_SOURCE=2 -D_QNX_SOURCE -DQNX=1 -D__aarch64__")
|
||||
|
||||
# The QNX8 compiler supplies this define by default, but we need to add it on QNX7.
|
||||
if(${QNX_VERSION} VERSION_LESS "8.0.0")
|
||||
set(TRT_MAGIC_QNX_FLAGS "${TRT_MAGIC_QNX_FLAGS} -D__QNX__")
|
||||
endif()
|
||||
|
||||
set(CMAKE_C_FLAGS_INIT ${TRT_MAGIC_QNX_FLAGS})
|
||||
set(CMAKE_CXX_FLAGS_INIT ${TRT_MAGIC_QNX_FLAGS})
|
||||
set(CMAKE_CUDA_FLAGS_INIT ${TRT_MAGIC_QNX_FLAGS})
|
||||
|
||||
# librt is built-in on QNX.
|
||||
set(RT_LIB "")
|
||||
|
||||
# CUDA configuration
|
||||
if(NOT DEFINED CUDA_ROOT AND DEFINED ENV{CUDA_ROOT})
|
||||
set(CUDA_ROOT "$ENV{CUDA_ROOT}")
|
||||
endif()
|
||||
|
||||
if(NOT EXISTS ${CUDA_ROOT})
|
||||
message(FATAL_ERROR "CUDA_ROOT not defined or path '${CUDA_ROOT}' does not exist. CUDA is required for TensorRT.")
|
||||
endif()
|
||||
|
||||
if(NOT DEFINED CMAKE_CUDA_COMPILER)
|
||||
set(CMAKE_CUDA_COMPILER "${CUDA_ROOT}/bin/nvcc" CACHE FILEPATH "Path to nvcc compiler")
|
||||
endif()
|
||||
|
||||
# We need to explicitly populate `CMAKE_CUDA_FLAGS` with the initial flags so they propagate to the CMake Compiler Identification phase.
|
||||
# Otherwise, they will only be initialized afterwards, which will cause the identification to fail.
|
||||
set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS_INIT})
|
||||
|
||||
# The CUDA setup for TRT-OSS is a bit wonky, so we enable the includes globally.
|
||||
include_directories(BEFORE SYSTEM
|
||||
${CUDA_ROOT}/targets/aarch64-qnx/include
|
||||
/usr/include/aarch64-unknown-nto-qnx # TensorRT Safety Headers are installed here.
|
||||
)
|
||||
|
||||
# And, well, as another consequence of that weirdness, we need to ensure that the cuda libs are on the link path.
|
||||
add_link_options(
|
||||
"-L${CUDA_ROOT}/targets/aarch64-qnx/lib"
|
||||
"-L${CUDA_ROOT}/targets/aarch64-qnx/lib/stubs"
|
||||
)
|
||||
|
||||
# TRT-OSS expects this to be defined. Not really sure why.
|
||||
set(CUDART_LIB "cudart")
|
||||
@@ -0,0 +1,136 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set(CMAKE_SYSTEM_NAME QNX)
|
||||
set(CMAKE_SYSTEM_PROCESSOR aarch64)
|
||||
set(CMAKE_C_COMPILER_TARGET gcc_ntoaarch64le)
|
||||
set(CMAKE_CXX_COMPILER_TARGET gcc_ntoaarch64le_cxx)
|
||||
|
||||
# Ensure that the QNX toolchain location was provided.
|
||||
if(NOT DEFINED ENV{QNX_HOST} OR NOT DEFINED ENV{QNX_TARGET})
|
||||
message(FATAL_ERROR "The environment variables QNX_HOST and QNX_TARGET must both be provided to build on QNX.")
|
||||
endif()
|
||||
|
||||
set(QNX_HOST $ENV{QNX_HOST})
|
||||
set(QNX_TARGET $ENV{QNX_TARGET})
|
||||
set(CMAKE_SYSROOT $ENV{QNX_TARGET})
|
||||
|
||||
if(NOT EXISTS ${QNX_HOST} OR NOT EXISTS ${QNX_TARGET})
|
||||
message(FATAL_ERROR "The QNX_HOST (${QNX_HOST}) or QNX_TARGET (${QNX_TARGET}) paths do not exist.")
|
||||
endif()
|
||||
|
||||
if(NOT DEFINED ENV{CUDA})
|
||||
message(FATAL_ERROR "The environment variable CUDA must be defined to specify the CUDA version to use for QNX-Safe builds.")
|
||||
endif()
|
||||
|
||||
if($ENV{CUDA} MATCHES "cuda-11.4")
|
||||
set(QNX_VERSION 7.1.0 CACHE STRING "")
|
||||
set(QNX_GCC_VERSION "8.3.0" CACHE STRING "")
|
||||
else()
|
||||
set(QNX_VERSION 8.0.0 CACHE STRING "")
|
||||
set(QNX_GCC_VERSION "12.2.0" CACHE STRING "")
|
||||
endif()
|
||||
|
||||
set(QNX_TRIPLE aarch64-unknown-nto-qnx${QNX_VERSION})
|
||||
|
||||
set(CMAKE_C_COMPILER ${QNX_HOST}/usr/bin/${QNX_TRIPLE}-gcc)
|
||||
set(CMAKE_CXX_COMPILER ${QNX_HOST}/usr/bin/${QNX_TRIPLE}-g++)
|
||||
set(CMAKE_CUDA_HOST_COMPILER ${QNX_HOST}/usr/bin/${QNX_TRIPLE}-g++ CACHE FILEPATH "")
|
||||
|
||||
# These linker behaviors aren't setup by default for QNX
|
||||
# They work the same as they would on Linux
|
||||
set(CMAKE_LINK_LIBRARY_USING_WHOLE_ARCHIVE
|
||||
"-Wl,--whole-archive <LIBRARY> -Wl,--no-whole-archive"
|
||||
)
|
||||
set(CMAKE_LINK_LIBRARY_USING_WHOLE_ARCHIVE_SUPPORTED TRUE)
|
||||
|
||||
set(CMAKE_LINK_GROUP_USING_RESCAN
|
||||
"LINKER:--start-group"
|
||||
"LINKER:--end-group"
|
||||
)
|
||||
set(CMAKE_LINK_GROUP_USING_RESCAN_SUPPORTED TRUE)
|
||||
|
||||
# Setup required flags for QNX compilation.
|
||||
set(TRT_MAGIC_QNX_FLAGS "-D_XOPEN_SOURCE=700 -D_POSIX_C_SOURCE=2 -D_QNX_SOURCE -DQNX=1 -D__aarch64__ -DIS_QNX_SAFE=1")
|
||||
|
||||
# The QNX8 compiler supplies this define by default, but we need to add it on QNX7.
|
||||
if(${QNX_VERSION} VERSION_LESS "8.0.0")
|
||||
set(TRT_MAGIC_QNX_FLAGS "${TRT_MAGIC_QNX_FLAGS} -D__QNX__")
|
||||
endif()
|
||||
|
||||
# QNX-Specific include directories.
|
||||
include_directories(BEFORE SYSTEM
|
||||
${QNX_HOST}/usr/lib/gcc/${QNX_TRIPLE}/${QNX_GCC_VERSION}/include
|
||||
${QNX_TARGET}/usr/include/c++/${QNX_GCC_VERSION}/${QNX_TRIPLE}
|
||||
${QNX_TARGET}/usr/include
|
||||
/usr/include/aarch64-unknown-nto-qnx-safety # TensorRT Safety Headers are installed here.
|
||||
)
|
||||
|
||||
set(CMAKE_C_FLAGS_INIT ${TRT_MAGIC_QNX_FLAGS})
|
||||
set(CMAKE_CXX_FLAGS_INIT ${TRT_MAGIC_QNX_FLAGS})
|
||||
set(CMAKE_CUDA_FLAGS_INIT ${TRT_MAGIC_QNX_FLAGS})
|
||||
|
||||
# The CMake test compilation depends on linking against cudadevrt, which is unavailable in SafeCUDA.
|
||||
# This allows us to test that the compiler works without testing the behavior of the linker.
|
||||
set(CMAKE_TRY_COMPILE_TARGET_TYPE "STATIC_LIBRARY")
|
||||
|
||||
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
|
||||
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
|
||||
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
|
||||
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
|
||||
|
||||
# CUDA configuration
|
||||
if(NOT DEFINED CUDA_ROOT AND DEFINED ENV{CUDA_ROOT})
|
||||
set(CUDA_ROOT "$ENV{CUDA_ROOT}")
|
||||
endif()
|
||||
|
||||
if(NOT EXISTS ${CUDA_ROOT})
|
||||
message(FATAL_ERROR "CUDA_ROOT not defined or path '${CUDA_ROOT}' does not exist. CUDA is required for TensorRT.")
|
||||
endif()
|
||||
|
||||
if(NOT DEFINED CMAKE_CUDA_COMPILER)
|
||||
set(CMAKE_CUDA_COMPILER "${CUDA_ROOT}/bin/nvcc" CACHE FILEPATH "Path to nvcc compiler")
|
||||
endif()
|
||||
|
||||
# We need to set a couple additional flags to compile with SafeCUDA.
|
||||
# 1. SafeCUDA does not contain cudadevrt, so we need to disable it by setting --cudadevrt=none
|
||||
# 2. SafeCUDA depends on the QNX Slogger2 library, which must be linked via -lslog2. It is a system library for QNX-Safe.
|
||||
# 3. Use --target-directory to tell nvcc to use aarch64-qnx-safe instead of aarch64-qnx
|
||||
# This is critical to override nvcc's internal _TARGET_DIR_ variable
|
||||
set(CMAKE_CUDA_FLAGS_INIT "${CMAKE_CUDA_FLAGS_INIT} --cudadevrt=none -lslog2 --target-directory aarch64-qnx-safe -legacy-launch-seq")
|
||||
|
||||
# We need to explicitly populate `CMAKE_CUDA_FLAGS` with the initial flags so they propagate to the CMake Compiler Identification phase.
|
||||
# Otherwise, they will only be initialized afterwards, which will cause the identification to fail.
|
||||
set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS_INIT})
|
||||
|
||||
# The CUDA setup for TRT-OSS is a bit wonky, so we enable the includes globally.
|
||||
include_directories(BEFORE SYSTEM
|
||||
${CUDA_ROOT}/targets/aarch64-qnx-safe/include
|
||||
)
|
||||
|
||||
link_directories(
|
||||
${CUDA_ROOT}/targets/aarch64-qnx-safe/lib
|
||||
${CUDA_ROOT}/targets/aarch64-qnx-safe/lib/stubs
|
||||
)
|
||||
|
||||
add_link_options(
|
||||
"LINKER:-rpath-link=${CUDA_ROOT}/targets/aarch64-qnx-safe/lib"
|
||||
"LINKER:-rpath-link=${CUDA_ROOT}/targets/aarch64-qnx-safe/lib/stubs"
|
||||
)
|
||||
|
||||
# Disable CMake-based cudart support as we handle the linkage manually.
|
||||
set(CMAKE_CUDA_RUNTIME_LIBRARY "none" CACHE INTERNAL "The used CUDA runtime library type. Disabled by TRT as TRT manages the linkage itself.")
|
||||
@@ -0,0 +1,46 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set(CMAKE_SYSTEM_NAME WindowsStore)
|
||||
set(CMAKE_SYSTEM_VERSION 10.0)
|
||||
|
||||
set(CMAKE_C_COMPILER ${CC})
|
||||
set(CMAKE_CXX_COMPILER ${CC})
|
||||
|
||||
if(DEFINED CUDA_TOOLKIT)
|
||||
set(CUDAToolkit_ROOT ${CUDA_TOOLKIT})
|
||||
endif()
|
||||
|
||||
set(CMAKE_CUDA_COMPILER ${CUDAToolkit_ROOT}\\bin\\nvcc.exe)
|
||||
|
||||
set(CMAKE_C_COMPILER_FORCED TRUE)
|
||||
set(CMAKE_CXX_COMPILER_FORCED TRUE)
|
||||
set(CMAKE_CUDA_COMPILER_FORCED TRUE)
|
||||
|
||||
set(NV_TOOLS ${NV_TOOLS})
|
||||
set(W10_LIBRARY_SUFFIXES .lib .dll)
|
||||
set(W10_LINKER ${MSVC_COMPILER_DIR}\\bin\\amd64\\link)
|
||||
|
||||
set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_NVCC_COMPILER} CACHE STRING "" FORCE)
|
||||
|
||||
set(ADDITIONAL_PLATFORM_INCL_FLAGS "-I${MSVC_COMPILER_DIR}\\include -I${MSVC_COMPILER_DIR}\\..\\ucrt\\include")
|
||||
set(ADDITIONAL_PLATFORM_LIB_FLAGS ${ADDITIONAL_PLATFORM_LIB_FLAGS} "-LIBPATH:${NV_TOOLS}\\ddk\\wddmv2\\official\\17134\\Lib\\10.0.17134.0\\um\\x64")
|
||||
set(ADDITIONAL_PLATFORM_LIB_FLAGS ${ADDITIONAL_PLATFORM_LIB_FLAGS} "-LIBPATH:${MSVC_COMPILER_DIR}\\lib\\amd64" )
|
||||
set(ADDITIONAL_PLATFORM_LIB_FLAGS ${ADDITIONAL_PLATFORM_LIB_FLAGS} "-LIBPATH:${MSVC_COMPILER_DIR}\\..\\ucrt\\lib\\x64")
|
||||
set(ADDITIONAL_PLATFORM_LIB_FLAGS ${ADDITIONAL_PLATFORM_LIB_FLAGS} "-LIBPATH:${CUDAToolkit_ROOT}\\lib\\x64 cudart.lib")
|
||||
|
||||
set(TRT_PLATFORM_ID "win10")
|
||||
@@ -0,0 +1,28 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set(CMAKE_SYSTEM_NAME Linux)
|
||||
set(CMAKE_SYSTEM_PROCESSOR x86_64)
|
||||
|
||||
set(CMAKE_C_COMPILER gcc)
|
||||
set(CMAKE_CXX_COMPILER g++)
|
||||
|
||||
if(NOT DEFINED CUDAToolkit_ROOT)
|
||||
set(CUDAToolkit_ROOT /usr/local/cuda CACHE STRING "CUDA ROOT dir")
|
||||
endif()
|
||||
|
||||
set(TRT_PLATFORM_ID "x86_64")
|
||||
@@ -0,0 +1,28 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set(CMAKE_SYSTEM_NAME Linux)
|
||||
set(CMAKE_SYSTEM_PROCESSOR x86_64)
|
||||
|
||||
set(CMAKE_C_COMPILER /opt/rh/devtoolset-8/root/usr/bin/gcc)
|
||||
set(CMAKE_CXX_COMPILER /opt/rh/devtoolset-8/root/usr/bin/g++)
|
||||
|
||||
if(NOT DEFINED CUDAToolkit_ROOT)
|
||||
set(CUDAToolkit_ROOT /usr/local/cuda CACHE STRING "CUDA ROOT dir")
|
||||
endif()
|
||||
|
||||
set(TRT_PLATFORM_ID "x86_64")
|
||||
@@ -0,0 +1,5 @@
|
||||
*.onnx
|
||||
*.engine
|
||||
*.profile
|
||||
|
||||
**/__pycache__/
|
||||
@@ -0,0 +1,194 @@
|
||||
# DeBERTa Model Inference with TensorRT Disentangled Attention Optimizations
|
||||
|
||||
- [DeBERTa Model Inference with TensorRT Disentangled Attention Optimizations](#deberta-model-inference-with-tensorrt-disentangled-attention-optimizations)
|
||||
- [Background](#background)
|
||||
- [Performance Benchmark](#performance-benchmark)
|
||||
- [Environment Setup](#environment-setup)
|
||||
- [Step 1: PyTorch model to ONNX model](#step-1-pytorch-model-to-onnx-model)
|
||||
- [Step 2: Modify ONNX model for TensorRT engine building](#step-2-modify-onnx-model-for-tensorrt-engine-building)
|
||||
- [Step 3: Model inference with TensorRT (using Python TensorRT API or `trtexec`)](#step-3-model-inference-with-tensorrt-using-python-tensorrt-api-or-trtexec)
|
||||
- [Optional Step: Correctness check of model with and without plugin](#optional-step-correctness-check-of-model-with-and-without-plugin)
|
||||
- [Optional Step: Model inference with ONNX Runtime and TensorRT Execution Provider (Python API)](#optional-step-model-inference-with-onnx-runtime-and-tensorrt-execution-provider-python-api)
|
||||
|
||||
***
|
||||
|
||||
## Background
|
||||
A performance gap has been observed between Google's [BERT](https://arxiv.org/abs/1810.04805) design and Microsoft's [DeBERTa](https://arxiv.org/abs/2006.03654) design. The main reason of the gap is the disentangled attention design in DeBERTa triples the attention computation over BERT's regular attention. In addition to the extra matrix multiplications, the disentangled attention design also involves indirect memory accesses during the gather operations. In this regard, a [TensorRT plugin](https://github.com/NVIDIA/TensorRT/tree/main/plugin/disentangledAttentionPlugin) has been implemented to optimize DeBERTa's disentangled attention module, which is built-in since TensorRT 8.4 GA Update 2 (8.4.3) release.
|
||||
|
||||
This DeBERTa demo includes code and scripts for (i) exporting ONNX model from PyTorch, (ii) modifying ONNX model by inserting the plugin nodes, (iii) model inference options with TensorRT `trtexec` executable, TensorRT Python API, or ONNX Runtime with TensorRT execution provider, and (iv) measuring the correctness and performance of the optimized model.
|
||||
|
||||
The demo works with the [HuggingFace implementation](https://github.com/huggingface/transformers/tree/main/src/transformers/models/deberta_v2) of DeBERTa.
|
||||
|
||||
## Performance Benchmark
|
||||
Experiments of inference performance are measured on NVIDIA A100-80GB, T4, and V100-16GB, using TensorRT 8.4 GA Update 2 (8.4.3) + CUDA 11.6 and TensorRT 8.2 GA Update 3 (8.2.4) + CUDA 11.4. The application-based model configuration is: sequence length = 512/1024/2048, hidden size = 384, intermediate size = 1536, number of layers = 12, number of attention heads = 6, maximum relative distance = 256, vocabulary size = 128K, batch size = 1, with randomly initialized weights. Numbers are average latency of 100 inference runs. Speedup numbers are fastest TensorRT number over the PyTorch baseline.
|
||||
|
||||
| Sequence Length | Model Latency (ms) | | TensorRT 8.4 GA Update 2 (8.4.3), CUDA 11.6 | | | TensorRT 8.2 GA Update 3 (8.2.4), CUDA 11.4 | |
|
||||
| :-------------- | :----------------------------- | :-------: | :-----------------------------------------: | :-------: | :-------: | :-----------------------------------------: | :-------: |
|
||||
| | | A100-80GB | T4 | V100-16GB | A100-80GB | T4 | V100-16GB |
|
||||
| 512 | PyTorch (FP32/TF32) | 23.7 | 35.6 | 53.1 | 24.6 | 34.0 | 53.1 |
|
||||
| | PyTorch (FP16) | 21.5 | 22.7 | 43.5 | 22.6 | 20.6 | 43.5 |
|
||||
| | TensorRT (FP32/TF32) | 3.9 | 12.4 | 7.2 | 4.3 | 12.3 | 5.6 |
|
||||
| | TensorRT (FP16) | **1.6** | 6.2 | **3.9** | **1.8** | 6.1 | **3.1** |
|
||||
| | TensorRT w/ plugin (FP32/TF32) | 4.3 | 12.9 | 6.9 | 4.8 | 13.1 | 6.9 |
|
||||
| | TensorRT w/ plugin (FP16) | 1.9 | **5.6** | 4.0 | 2.1 | **6.0** | 4.2 |
|
||||
| | **Speedup** | **14.8** | **6.4** | **13.6** | **13.7** | **5.7** | **17.1** |
|
||||
| | | | | | | | |
|
||||
| 1024 | PyTorch (FP32/TF32) | 35.7 | 82.8 | 65.4 | 35.8 | 83.7 | 65.4 |
|
||||
| | PyTorch (FP16) | 35.1 | 53.8 | 59.4 | 35.6 | 52.9 | 59.4 |
|
||||
| | TensorRT (FP32/TF32) | 8.3 | 31.3 | 15.7 | 8.8 | 34.7 | 12.8 |
|
||||
| | TensorRT (FP16) | 3.8 | 16.3 | 8.3 | 3.7 | 18.4 | 7.4 |
|
||||
| | TensorRT w/ plugin (FP32/TF32) | 7.9 | 31.1 | 14.3 | 9.0 | 32.8 | 13.5 |
|
||||
| | TensorRT w/ plugin (FP16) | **2.8** | **12.4** | **7.3** | **3.3** | **13.6** | **6.8** |
|
||||
| | **Speedup** | **12.8** | **6.7** | **9.0** | **10.8** | **6.2** | **9.6** |
|
||||
| | | | | | | | |
|
||||
| 2048 | PyTorch (FP32/TF32) | 84.8 | 261.3 | 236.3 | 84.8 | 263.1 | 236.3 |
|
||||
| | PyTorch (FP16) | 79.4 | 178.2 | 205.5 | 76.0 | 181.8 | 205.5 |
|
||||
| | TensorRT (FP32/TF32) | 22.2 | 109.1 | 39.1 | 22.6 | 109.8 | 35.0 |
|
||||
| | TensorRT (FP16) | 10.2 | 56.2 | 23.5 | 10.8 | 62.9 | 21.0 |
|
||||
| | TensorRT w/ plugin (FP32/TF32) | 20.4 | 96.7 | 38.6 | 21.2 | 95.7 | 35.6 |
|
||||
| | TensorRT w/ plugin (FP16) | **7.6** | **44.1** | **21.0** | **8.3** | **44.6** | **18.1** |
|
||||
| | **Speedup** | **11.2** | **5.9** | **11.3** | **10.2** | **5.9** | **13.1** |
|
||||
|
||||
In addition, a pre-trained model `microsoft/deberta-v3-xsmall` was tested, which configuration is similar to sequencen length = 512 model above. And `microsoft/deberta-v3-large` (sequence length = 512) performance on TensorRT 8.4 GA Update 2 (8.4.3) is also added from recent experiments.
|
||||
|
||||
| Variant | Model Latency (ms) | | TensorRT 8.4 GA Update 2 (8.4.3), CUDA 11.6 | | | TensorRT 8.2 GA Update 3 (8.2.4), CUDA 11.4 | |
|
||||
| :------------------ | :----------------------------- | :-------: | :-----------------------------------------: | :-------: | :-------: | :-----------------------------------------: | :-------: |
|
||||
| | | A100-80GB | T4 | V100-16GB | A100-80GB | T4 | V100-16GB |
|
||||
| `deberta-v3-xsmall` | PyTorch (FP32/TF32) | 30.1 | 40.6 | 57.1 | 28.9 | 39.2 | 57.1 |
|
||||
| | PyTorch (FP16) | 27.6 | 26.3 | 47.9 | 26.3 | 24.8 | 47.9 |
|
||||
| | TensorRT (FP32/TF32) | 4.1 | 12.4 | 7.6 | 4.4 | 12.8 | 5.8 |
|
||||
| | TensorRT (FP16) | 1.8 | 6.2 | **3.7** | **1.9** | 6.4 | **3.1** |
|
||||
| | TensorRT w/ plugin (FP32/TF32) | 4.3 | 12.9 | 7.7 | 4.8 | 13.6 | 6.9 |
|
||||
| | TensorRT w/ plugin (FP16) | **1.8** | **5.6** | 4.7 | 2.1 | **6.0** | 4.1 |
|
||||
| | **Speedup** | **16.7** | **7.3** | **15.4** | **15.2** | **6.5** | **18.4** |
|
||||
| | | | | | | | |
|
||||
| `deberta-v3-large` | PyTorch (FP32/TF32) | 51.6 | 40.6 | 100.0 | - | - | - |
|
||||
| | PyTorch (FP16) | 52.6 | 26.3 | 96.5 | - | - | - |
|
||||
| | TensorRT (FP32/TF32) | 31.1 | 32.3 | 43.2 | - | - | - |
|
||||
| | TensorRT (FP16) | 7.8 | 16.0 | **13.9** | - | - | - |
|
||||
| | TensorRT w/ plugin (FP32/TF32) | 30.9 | 32.8 | 44.9 | - | - | - |
|
||||
| | TensorRT w/ plugin (FP16) | **7.3** | **13.5** | 14.4 | - | - | - |
|
||||
| | **Speedup** | **7.1** | **3.0** | **7.2** | - | - | - |
|
||||
|
||||
Note that the performance gap between BERT's self-attention and DeBERTa's disentangled attention mainly comes from the additional `Gather` and `Transpose` operations in the attention design, and such gap becomes most significant when the maximum input sequence length is long (e.g., 2048). The fastest inference times are labeled as bold in the table above. In summary, for short sequence length applications, regular TensorRT inference might be sufficient, while for longer sequence length applications, the plugin optimizations are preferred and can be utilized to further improve the inference latency. Also, to get maximum speedup, using FP16 precision for inference is recommended.
|
||||
|
||||
## Environment Setup
|
||||
It is recommended to use docker for reproducing the following steps. Follow the setup steps in TensorRT OSS [README](https://github.com/NVIDIA/TensorRT#setting-up-the-build-environment) to build and launch the container and build OSS:
|
||||
|
||||
**Example: Ubuntu 20.04 on x86-64 with cuda-12.8 (default)**
|
||||
```bash
|
||||
# Download this TensorRT OSS repo
|
||||
git clone -b main https://github.com/nvidia/TensorRT TensorRT
|
||||
cd TensorRT
|
||||
git submodule update --init --recursive
|
||||
|
||||
## at root of TensorRT OSS
|
||||
# build container
|
||||
./docker/build.sh --file docker/ubuntu-20.04.Dockerfile --tag tensorrt-ubuntu20.04-cuda12.8
|
||||
|
||||
# launch container
|
||||
./docker/launch.sh --tag tensorrt-ubuntu20.04-cuda12.8 --gpus all
|
||||
|
||||
## now inside container
|
||||
# build OSS (only required for pre-8.4.3 TensorRT versions)
|
||||
cd $TRT_OSSPATH
|
||||
mkdir -p build && cd build
|
||||
cmake .. -DTRT_LIB_DIR=$TRT_LIBPATH -DTRT_OUT_DIR=`pwd`/out
|
||||
make -j$(nproc)
|
||||
|
||||
# polygraphy bin location & trtexec bin location
|
||||
export PATH="~/.local/bin":"${TRT_OSSPATH}/build/out":$PATH
|
||||
|
||||
# navigate to the demo folder install additional dependencies (note PyTorch 1.11.0 is recommended for onnx to export properly)
|
||||
cd $TRT_OSSPATH/demo/DeBERTa
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
> NOTE:
|
||||
1. `sudo` password for Ubuntu build containers is 'nvidia'.
|
||||
2. The DeBERTa plugin is only built-in after TensorRT 8.4 GA Update 2 (8.4.3) release. For pre-8.4.3 versions, you need to build TensorRT OSS from source and link the shared libraries with TensorRT build.
|
||||
3. For ONNX Runtime deployment, the associated changes for the plugin are only built-in after 1.12 release. For pre-1.12 versions, you need to [build ONNX Runtime from source with TensorRT execution provider](https://onnxruntime.ai/docs/build/eps.html#tensorrt).
|
||||
4. TensorRT OSS docker container is designed for use cases when you need to build OSS repository from source. After TensorRT 8.4 GA Update 2 (8.4.3) release, the most convenient way is to use the corresponding `22.08-py3` docker image at [TensorRT NGC container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/tensorrt) when it is released, without the need to build from dockerfile. But for now, please follow the instructions above to use the OSS docker image.
|
||||
|
||||
## Step 1: PyTorch model to ONNX model
|
||||
```bash
|
||||
python deberta_pytorch2onnx.py --filename ./test/deberta.onnx [--variant microsoft/deberta-v3-xsmall] [--seq-len 2048]
|
||||
```
|
||||
|
||||
This will export the DeBERTa model from HuggingFace's DeBERTa-V2 implementation into ONNX format, with user given file name. Optionally, specify DeBERTa variant to export, such as `--variant microsoft/deberta-v3-xsmall`, or specify the maximum sequence length configuration for testing, such as `--seq-len 2048`. Models specified by `--variant` are with pre-trained weights, while models specified by `--seq-len` are with randomly initialized weights. Note that `--variant` and `--seq-len` cannot be used together because pre-trained models have pre-defined max sequence length.
|
||||
|
||||
## Step 2: Modify ONNX model for TensorRT engine building
|
||||
```bash
|
||||
python deberta_onnx_modify.py ./test/deberta.onnx # generates original TRT-compatible model, `*_original.onnx`
|
||||
|
||||
python deberta_onnx_modify.py ./test/deberta.onnx --plugin # generates TRT-compatible model with plugin nodes, `*_plugin.onnx`
|
||||
```
|
||||
|
||||
The original HuggingFace implementation has uint8 Cast operations that TensorRT doesn't support, which needs to be removed from the ONNX model. After this step, the ONNX model can run in TensorRT. Without passing any flags, the script will save the model with Cast nodes removed, by default named with `_original` suffix.
|
||||
|
||||
Further, to use the DeBERTa plugin optimizations, the disentangled attention subgraph needs to be replaced by node named `DisentangledAttention_TRT`. By passing `--plugin` flag, the script will save the model with Cast nodes removed and plugin nodes replaced, by default named with `_plugin` suffix.
|
||||
|
||||
The benefits of the DeBERTa plugin optimizations can be demonstrated by comparing the latency of original model and plugin model.
|
||||
|
||||
## Step 3: Model inference with TensorRT (using Python TensorRT API or `trtexec`)
|
||||
```bash
|
||||
# build and test the original DeBERTa model (baseline)
|
||||
python deberta_tensorrt_inference.py --onnx=./test/deberta_original.onnx --build fp16 --test fp16
|
||||
|
||||
# build and test the optimized DeBERTa model with plugin
|
||||
python deberta_tensorrt_inference.py --onnx=./test/deberta_plugin.onnx --build fp16 --test fp16
|
||||
```
|
||||
|
||||
This will build and test the original and optimized DeBERTa models. `--build` to build the engine from ONNX model, and `--test` to measure the optimized latency. TensorRT engine of different precisions (`--fp32/--tf32/--fp16`) can be built. Engine files are saved as `**/[Model name]_[GPU name]_[Precision].engine`.
|
||||
|
||||
> NOTE:
|
||||
1. To get maximum speedup, using `--fp16` is recommended. Also, it was observed the plugin optimizations demonstrate more speedup when the input sequence length is long, such as 2048.
|
||||
2. TF32 is only effective on Ampere and later devices.
|
||||
3. TensorRT optimization profile in `deberta_tensorrt_inference.py` is set by default for batch size of 1 usage. For batch usage, the optional and maximum profile should be changed.
|
||||
4. TensorRT engines are specific to the exact GPU device they were built on, as well as the platform and the TensorRT version. On the same machine, building is needed only once and the engine can be used for repeated testing.
|
||||
|
||||
For `trtexec` test, it is recommended to used the [TensorRT NGC container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/tensorrt) where the executable is built-in. The DeBERTa plugin support is available since docker image `22.08-py3`, which contains TensorRT 8.4 GA Update 2 (8.4.3) pre-installed. For earlier images before `22.08`, skip this `trtexec` test.
|
||||
|
||||
`trtexec` command for sequence length = 2048 model is given as an example:
|
||||
```bash
|
||||
trtexec --onnx=./test/deberta_plugin.onnx --workspace=4096 --explicitBatch --optShapes=input_ids:1x2048,attention_mask:1x2048 --iterations=10 --warmUp=10 --noDataTransfers --fp16
|
||||
```
|
||||
|
||||
## Optional Step: Correctness check of model with and without plugin
|
||||
```bash
|
||||
# prepare the ONNX models with intermediate output nodes (this will save two new onnx models with suffix `*_correctness_check_original.onnx` and `*_correctness_check_plugin.onnx`)
|
||||
python deberta_onnx_modify.py ./test/deberta.onnx --correctness-check
|
||||
|
||||
# build the ONNX models with intermediate outputs for comparison
|
||||
python deberta_tensorrt_inference.py --onnx=./test/deberta_correctness_check_original.onnx --build fp16
|
||||
python deberta_tensorrt_inference.py --onnx=./test/deberta_correctness_check_plugin.onnx --build fp16
|
||||
|
||||
# run correctness check (specify the root model name with --onnx)
|
||||
python deberta_tensorrt_inference.py --onnx=./test/deberta --correctness-check fp16
|
||||
```
|
||||
|
||||
Correctness check requires intermediate outputs from the model, thus it is necessary to modify the ONNX graph and add intermediate output nodes. The correctness check was added at the location of plugin outputs in each layer. The metrics are average and maximum of the element-wise absolute error. Note that for FP16 precision with 10 significance bits, absolute error in the order of 1e-2 and 1e-3 is expected, and for FP32 precision with 23 significance bits, 1e-6 to 1e-7 is expected. Refer to [Machine Epsilon](https://en.wikipedia.org/wiki/Machine_epsilon) for details.
|
||||
|
||||
Example output for FP16 correctness check between original DeBERTa model and plugin optimized DeBERTa model:
|
||||
```bash
|
||||
[Layer 0 Element-wise Check] Avgerage absolute error: 2.152580e-03, Maximum absolute error: 3.010559e-02. 1e-2~1e-3 expected for FP16 (10 significance bits) and 1e-6~1e-7 expected for FP32 (23 significance bits)
|
||||
...
|
||||
...
|
||||
[Layer 12 Element-wise Check] Avgerage absolute error: 6.198883e-05, Maximum absolute error: 1.220703e-04. 1e-2~1e-3 expected for FP16 (10 significance bits) and 1e-6~1e-7 expected for FP32 (23 significance bits)
|
||||
```
|
||||
|
||||
## Optional Step: Model inference with ONNX Runtime and TensorRT Execution Provider (Python API)
|
||||
```bash
|
||||
python deberta_ort_inference.py --onnx=./test/deberta_original.onnx --test fp16
|
||||
|
||||
python deberta_ort_inference.py --onnx=./test/deberta_plugin.onnx --test fp16
|
||||
|
||||
python deberta_ort_inference.py --onnx=./test/deberta --correctness-check fp16 # for correctness check
|
||||
```
|
||||
|
||||
In addition to TensorRT inference, [ONNX Runtime](https://github.com/microsoft/onnxruntime) with TensorRT Execution Provider can also be used as the inference framework. The DeBERTa TensorRT plugin is officially supported in onnxruntime since version 1.12. For earlier releases of onnxruntime, skip this step.
|
||||
|
||||
The results can be cross-validated between TensorRT and onnxruntime. For example, the [Polygraphy](https://github.com/NVIDIA/TensorRT/tree/master/tools/Polygraphy) tool can be used for easy comparison:
|
||||
|
||||
```bash
|
||||
polygraphy run ./test/deberta_original.onnx --trt --onnxrt --workspace=4000000000
|
||||
```
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import numpy as np
|
||||
import logging
|
||||
from cuda.bindings import driver as cuda, runtime as cudart
|
||||
|
||||
class CudaStreamContext:
|
||||
"""CUDA stream lifecycle management with context manager support"""
|
||||
def __init__(self):
|
||||
"""Initialize CUDA stream"""
|
||||
self._stream = cuda_call(cudart.cudaStreamCreate())
|
||||
|
||||
def __enter__(self):
|
||||
"""Create CUDA stream when entering context (if not already created)"""
|
||||
if self._stream is None:
|
||||
self._stream = cuda_call(cudart.cudaStreamCreate())
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Destroy CUDA stream when exiting context"""
|
||||
self.free()
|
||||
|
||||
@property
|
||||
def stream(self) -> cudart.cudaStream_t:
|
||||
if self._stream is None:
|
||||
raise RuntimeError("Stream not created. Use 'with' statement.")
|
||||
return self._stream
|
||||
|
||||
def synchronize(self):
|
||||
"""Synchronize the stream"""
|
||||
if self._stream is None:
|
||||
raise RuntimeError("Stream not created. Use 'with' statement.")
|
||||
cuda_call(cudart.cudaStreamSynchronize(self._stream))
|
||||
|
||||
def free(self):
|
||||
"""Explicitly free the CUDA stream"""
|
||||
if self._stream is not None:
|
||||
try:
|
||||
cuda_call(cudart.cudaStreamDestroy(self._stream))
|
||||
self._stream = None
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to destroy CUDA stream: {e}")
|
||||
|
||||
def __del__(self):
|
||||
"""Cleanup stream on destruction"""
|
||||
if hasattr(self, '_stream') and self._stream is not None:
|
||||
self.free()
|
||||
|
||||
def __str__(self):
|
||||
return f"CudaStreamContext: {self._stream}"
|
||||
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
def cuda_call(call):
|
||||
"""Helper function to make CUDA calls and check for errors"""
|
||||
def _cudaGetErrorEnum(error):
|
||||
if isinstance(error, cuda.CUresult):
|
||||
err, name = cuda.cuGetErrorName(error)
|
||||
return name if err == cuda.CUresult.CUDA_SUCCESS else "<unknown>"
|
||||
elif isinstance(error, cudart.cudaError_t):
|
||||
return cudart.cudaGetErrorName(error)[1]
|
||||
else:
|
||||
raise RuntimeError("Unknown error type: {}".format(error))
|
||||
|
||||
err, res = call[0], call[1:]
|
||||
if err.value:
|
||||
raise RuntimeError(
|
||||
"CUDA error code={}({})".format(
|
||||
err.value, _cudaGetErrorEnum(err)
|
||||
)
|
||||
)
|
||||
if len(res) == 1:
|
||||
return res[0]
|
||||
elif len(res) == 0:
|
||||
return None
|
||||
else:
|
||||
return res
|
||||
|
||||
def getComputeCapacity(devID=0):
|
||||
major = cuda_call(cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrComputeCapabilityMajor, devID))
|
||||
minor = cuda_call(cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrComputeCapabilityMinor, devID))
|
||||
return (major, minor)
|
||||
|
||||
def memcpy_host_to_device_async(device_ptr: int, host_arr: np.ndarray, stream):
|
||||
"""Wrapper for async host-to-device memory copy"""
|
||||
cuda_call(cudart.cudaMemcpyAsync(device_ptr, host_arr.ctypes.data, host_arr.nbytes, cudart.cudaMemcpyKind.cudaMemcpyHostToDevice, stream))
|
||||
|
||||
|
||||
def memcpy_device_to_host_async(host_arr: np.ndarray, device_ptr: int, stream):
|
||||
"""Wrapper for async device-to-host memory copy"""
|
||||
cuda_call(cudart.cudaMemcpyAsync(host_arr.ctypes.data, device_ptr, host_arr.nbytes, cudart.cudaMemcpyKind.cudaMemcpyDeviceToHost, stream))
|
||||
|
||||
def memcpy_device_to_device_async(dst_device_ptr: int, src_device_ptr: int, nbytes: int, stream):
|
||||
"""Wrapper for async device-to-device memory copy"""
|
||||
cuda_call(cudart.cudaMemcpyAsync(dst_device_ptr, src_device_ptr, nbytes, cudart.cudaMemcpyKind.cudaMemcpyDeviceToDevice, stream))
|
||||
|
||||
def memcpy_host_to_device(device_ptr: int, host_arr: np.ndarray):
|
||||
"""Wrapper for synchronous host-to-device memory copy"""
|
||||
cuda_call(cudart.cudaMemcpy(device_ptr, host_arr.ctypes.data, host_arr.nbytes, cudart.cudaMemcpyKind.cudaMemcpyHostToDevice))
|
||||
|
||||
|
||||
def memcpy_device_to_host(host_arr: np.ndarray, device_ptr: int):
|
||||
"""Wrapper for synchronous device-to-host memory copy"""
|
||||
cuda_call(cudart.cudaMemcpy(host_arr.ctypes.data, device_ptr, host_arr.nbytes, cudart.cudaMemcpyKind.cudaMemcpyDeviceToHost))
|
||||
|
||||
def memcpy_device_to_device(dst_device_ptr: int, src_device_ptr: int, nbytes: int):
|
||||
"""Wrapper for synchronous device-to-device memory copy"""
|
||||
cuda_call(cudart.cudaMemcpy(dst_device_ptr, src_device_ptr, nbytes, cudart.cudaMemcpyKind.cudaMemcpyDeviceToDevice))
|
||||
|
||||
# Initialize CUDA
|
||||
cuda_call(cudart.cudaFree(0))
|
||||
@@ -0,0 +1,227 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
'''
|
||||
Modify original ONNX exported from HuggingFace to for TensorRT engine building.
|
||||
|
||||
The original HuggingFace implementation has uint8 Cast operations that TensorRT doesn't support, which needs to be removed from the ONNX model. After this step, the ONNX model can run in TensorRT.
|
||||
|
||||
Further, to use the DeBERTa plugin optimizations, the disentangled attention module needs to be replaced by node named `DisentangledAttention_TRT`.
|
||||
|
||||
Optional: generate model that has per-layer intermediate outputs for correctness check purpose.
|
||||
|
||||
These modifications are automated in this script.
|
||||
|
||||
Usage:
|
||||
python deberta_onnx_modify.py xx.onnx # for original TRT-compatible model, `xx_original.onnx`
|
||||
python deberta_onnx_modify.py xx.onnx --plugin # for TRT-compatible model with plugin nodes, `xx_plugin.onnx`
|
||||
python deberta_onnx_modify.py xx.onnx --correctness-check # for correctness check
|
||||
'''
|
||||
|
||||
import onnx
|
||||
from onnx import TensorProto
|
||||
import onnx_graphsurgeon as gs
|
||||
import argparse, os
|
||||
import numpy as np
|
||||
|
||||
parser = argparse.ArgumentParser(description="Modify DeBERTa ONNX model to prepare for TensorRT engine building. If none of --plugin or --correctness-check flag is passed, it will just save the uint8 cast removed model.")
|
||||
parser.add_argument('input', type=str, help='Path to the input ONNX model')
|
||||
parser.add_argument('--plugin', action='store_true', help="Generate model with plugin")
|
||||
parser.add_argument('--correctness-check', action='store_true', help="Generate model that has per-layer intermediate outputs for correctness check purpose")
|
||||
parser.add_argument('--output', type=str, help="Path to the output ONNX model. If not set, default to the input file name with a suffix of '_original' or '_plugin' ")
|
||||
args = parser.parse_args()
|
||||
|
||||
model_input = args.input
|
||||
use_plugin = args.plugin
|
||||
correctness_check = args.correctness_check
|
||||
|
||||
if args.output is None:
|
||||
model_output = os.path.splitext(model_input)[0] + ("_plugin" if use_plugin else "_original") + os.path.splitext(model_input)[-1]
|
||||
else:
|
||||
model_output = args.output
|
||||
|
||||
def remove_uint8_cast(graph):
|
||||
'''
|
||||
Remove all uint8 Cast nodes since TRT doesn't support UINT8 cast op.
|
||||
Ref: https://github.com/NVIDIA/TensorRT/tree/main/tools/onnx-graphsurgeon/examples/06_removing_nodes
|
||||
'''
|
||||
nodes = [node for node in graph.nodes if node.op == 'Cast' and node.attrs["to"] == TensorProto.UINT8] # find by op name and attribute
|
||||
|
||||
for node in nodes:
|
||||
# [ONNX's Cast operator](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Cast) will exactly have 1 input and 1 output
|
||||
# reconnect tensors
|
||||
input_node = node.i()
|
||||
input_node.outputs = node.outputs
|
||||
node.outputs.clear()
|
||||
|
||||
# an alternative way is to just not cast to uint8
|
||||
# node.attrs["to"] = TensorProto.INT64
|
||||
|
||||
return graph
|
||||
|
||||
@gs.Graph.register()
|
||||
def insert_disentangled_attention(self, inputs, outputs, factor, span):
|
||||
'''
|
||||
Fuse disentangled attention module (Add + Gather + Gather + Transpose + Add + Div)
|
||||
|
||||
inputs: list of plugin inputs
|
||||
outputs: list of plugin outputs
|
||||
factor: scaling factor of disentangled attention, sqrt(3d), converted from a division factor to a multiplying factor
|
||||
span: relative distance span, k
|
||||
'''
|
||||
# disconnect previous output from flow (the previous subgraph still exists but is effectively dead since it has no link to an output tensor, and thus will be cleaned up)
|
||||
[out.inputs.clear() for out in outputs]
|
||||
# add plugin layer
|
||||
attrs = {
|
||||
"factor": 1/factor,
|
||||
"span": span
|
||||
}
|
||||
self.layer(op='DisentangledAttention_TRT', inputs=inputs, outputs=outputs, attrs=attrs)
|
||||
|
||||
def insert_disentangled_attention_all(graph):
|
||||
'''
|
||||
Insert disentangled attention plugin nodes for all layers
|
||||
'''
|
||||
nodes = [node for node in graph.nodes if node.op == 'GatherElements'] # find entry points by gatherelements op
|
||||
assert len(nodes) % 2 == 0, "No. of GatherElements nodes is not an even number!"
|
||||
|
||||
layers = [(nodes[2*i+0], nodes[2*i+1]) for i in range(len(nodes)//2)] # 2 gatherelements in 1 layer
|
||||
for l, (left,right) in enumerate(layers):
|
||||
print(f"Fusing layer {l}")
|
||||
|
||||
# CAVEAT! MUST cast to list() when setting the inputs & outputs. graphsurgeon's default for X.inputs and X.outputs is `onnx_graphsurgeon.util.misc.SynchronizedList`, i.e. 2-way node-tensor updating mechanism. If not cast, when we remove the input nodes of a tensor, the tensor itself will be removed as well...
|
||||
|
||||
# inputs: (data0, data1, data2), input tensors for c2c add and 2 gathers
|
||||
inputs = list(left.o().o().o().o().i().inputs)[0:1] + list(left.inputs)[0:1] + list(right.inputs)[0:1]
|
||||
|
||||
# outputs: (result), output tensors after adding 3 gather results
|
||||
outputs = list(left.o().o().o().o().outputs)
|
||||
|
||||
# constants: scaling factor, relative distance span
|
||||
factor = left.o().inputs[1].inputs[0].attrs["value"].values.item()
|
||||
span = right.i(1,0).i().i().i().inputs[1].inputs[0].attrs["value"].values.item()
|
||||
|
||||
# insert plugin layer
|
||||
graph.insert_disentangled_attention(inputs, outputs, factor, span)
|
||||
|
||||
return graph
|
||||
|
||||
def correctness_check_models(graph):
|
||||
'''
|
||||
Add output nodes at the plugin exit point for both the original model and the model with plugin
|
||||
'''
|
||||
|
||||
seq_len = graph.inputs[0].shape[1]
|
||||
|
||||
## for original graph
|
||||
# make a copy of the graph first
|
||||
graph_raw = graph.copy()
|
||||
nodes = [node for node in graph_raw.nodes if node.op == 'GatherElements'] # find by gatherelements op
|
||||
assert len(nodes) % 2 == 0, "No. of GatherElements nodes is not an even number!"
|
||||
|
||||
layers = [(nodes[2*i+0], nodes[2*i+1]) for i in range(len(nodes)//2)] # 2 gatherelements in 1 layer
|
||||
original_output_all = []
|
||||
for l, (left,right) in enumerate(layers):
|
||||
# outputs: (result), output tensors after adding 3 gather results
|
||||
# add the output tensor to the graph outputs list. Don't create any new tensor!
|
||||
end_node = left.o().o().o().o()
|
||||
end_node.outputs[0].dtype = graph_raw.outputs[0].dtype # need to explicitly specify dtype and shape of graph output tensor
|
||||
end_node.outputs[0].shape = ['batch_size*num_heads', seq_len, seq_len]
|
||||
original_output_all.append(end_node.outputs[0])
|
||||
|
||||
graph_raw.outputs = graph_raw.outputs + original_output_all # add plugin outputs to graph output
|
||||
|
||||
## for modified graph with plugin
|
||||
nodes = [node for node in graph.nodes if node.op == 'GatherElements'] # find by gatherelements op
|
||||
assert len(nodes) % 2 == 0, "No. of GatherElements nodes is not an even number!"
|
||||
|
||||
layers = [(nodes[2*i+0], nodes[2*i+1]) for i in range(len(nodes)//2)] # 2 gatherelements in 1 layer
|
||||
plugin_output_all = []
|
||||
for l, (left,right) in enumerate(layers):
|
||||
# inputs: (data0, data1, data2), input tensors for c2c add and 2 gathers
|
||||
inputs = list(left.o().o().o().o().i().inputs)[0:1] + list(left.inputs)[0:1] + list(right.inputs)[0:1]
|
||||
# outputs: (result), output tensors after adding 3 gather results
|
||||
outputs = list(left.o().o().o().o().outputs)
|
||||
end_node = left.o().o().o().o()
|
||||
end_node.outputs[0].dtype = graph.outputs[0].dtype # need to explicitly specify dtype and shape of graph output tensor
|
||||
end_node.outputs[0].shape = ['batch_size*num_heads', seq_len, seq_len]
|
||||
plugin_output_all.append(end_node.outputs[0]) # add to graph output (outside this loop)
|
||||
|
||||
# constants: scaling factor, relative distance span
|
||||
factor = left.o().inputs[1].inputs[0].attrs["value"].values.item()
|
||||
span = right.i(1,0).i().i().i().inputs[1].inputs[0].attrs["value"].values.item()
|
||||
|
||||
# insert plugin layer
|
||||
graph.insert_disentangled_attention(inputs, outputs, factor, span)
|
||||
|
||||
graph.outputs = graph.outputs + plugin_output_all # add plugin outputs to graph output
|
||||
|
||||
return graph_raw, graph
|
||||
|
||||
def check_model(model_name):
|
||||
# Load the ONNX model
|
||||
model = onnx.load(model_name)
|
||||
|
||||
# Check that the model is well formed
|
||||
onnx.checker.check_model(model)
|
||||
|
||||
# load onnx
|
||||
graph = gs.import_onnx(onnx.load(model_input))
|
||||
|
||||
# first, remove uint8 cast nodes
|
||||
graph = remove_uint8_cast(graph)
|
||||
|
||||
if use_plugin:
|
||||
# save the modified model with plugin nodes
|
||||
|
||||
# replace Add + Gather + Gather + Transpose + Add + Div (c2c and c2p and p2c) with DisentangledAttention_TRT node
|
||||
graph = insert_disentangled_attention_all(graph)
|
||||
|
||||
# remove unused nodes, and topologically sort the graph.
|
||||
graph.cleanup().toposort()
|
||||
|
||||
# export the onnx graph from graphsurgeon
|
||||
onnx.save_model(gs.export_onnx(graph), model_output)
|
||||
print(f"Saving modified model to {model_output}")
|
||||
|
||||
# don't check model because 'DisentangledAttention_TRT' is not a registered op
|
||||
|
||||
elif correctness_check:
|
||||
# correctness check, save two models (original and w/ plugin) with intermediate output nodes inserted
|
||||
graph_raw, graph = correctness_check_models(graph)
|
||||
|
||||
# remove unused nodes, and topologically sort the graph.
|
||||
graph_raw.cleanup().toposort()
|
||||
graph.cleanup().toposort()
|
||||
|
||||
# export the onnx graph from graphsurgeon
|
||||
model_output1 = os.path.splitext(model_input)[0] + "_correctness_check_original" + os.path.splitext(model_input)[-1]
|
||||
model_output2 = os.path.splitext(model_input)[0] + "_correctness_check_plugin" + os.path.splitext(model_input)[-1]
|
||||
onnx.save_model(gs.export_onnx(graph_raw), model_output1)
|
||||
onnx.save_model(gs.export_onnx(graph), model_output2)
|
||||
|
||||
print(f"Saving models for correctness check to {model_output1} (original) and {model_output2} (with plugin)")
|
||||
|
||||
check_model(model_output1)
|
||||
# don't check model_output2 because 'DisentangledAttention_TRT' is not a registered op
|
||||
|
||||
else:
|
||||
# no flag passed, save model with just uint8 cast removed
|
||||
graph.cleanup().toposort()
|
||||
onnx.save_model(gs.export_onnx(graph), model_output)
|
||||
print(f"Saving modified model to {model_output}")
|
||||
check_model(model_output)
|
||||
@@ -0,0 +1,201 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
Test ORT-TRT engine of DeBERTa model. Different precisions are supported.
|
||||
|
||||
Usage:
|
||||
Test model inference time:
|
||||
- python deberta_ort_inference.py --onnx=./test/deberta.onnx --test fp16
|
||||
|
||||
Correctness check by comparing original model and model with plugin:
|
||||
- python deberta_ort_inference.py --onnx=./test/deberta --correctness-check fp16
|
||||
|
||||
Notes:
|
||||
- supported precisions are fp32/fp16. For test, you can specify more than one precisions, and TensorRT engine of each precision will be built sequentially.
|
||||
- engine files are saved at `./engine_cache/[Model name]_[GPU name]_[Precision]/`. Note that TensorRT engine is specific to both GPU architecture and TensorRT version.
|
||||
- if in --correctness-check mode, the argument for --onnx is the stem name for the model without .onnx extension.
|
||||
"""
|
||||
|
||||
import os, argparse
|
||||
import onnxruntime as ort
|
||||
import numpy as np
|
||||
import torch
|
||||
from time import time
|
||||
|
||||
ENGINE_PATH = './test'
|
||||
if not os.path.exists(ENGINE_PATH):
|
||||
os.makedirs(ENGINE_PATH)
|
||||
|
||||
def GPU_ABBREV(name):
|
||||
'''
|
||||
Map GPU device query name to abbreviation.
|
||||
|
||||
::param str name Device name from torch.cuda.get_device_name().
|
||||
::return str GPU abbreviation.
|
||||
'''
|
||||
|
||||
GPU_LIST = [
|
||||
'V100',
|
||||
'TITAN',
|
||||
'T4',
|
||||
'A100',
|
||||
'A10G',
|
||||
'A10'
|
||||
]
|
||||
# Partial list, can be extended. The order of A100, A10G, A10 matters. They're put in a way to not detect substring A10 as A100
|
||||
|
||||
for i in GPU_LIST:
|
||||
if i in name:
|
||||
return i
|
||||
|
||||
return 'GPU' # for names not in the partial list, use 'GPU' as default
|
||||
|
||||
gpu_name = GPU_ABBREV(torch.cuda.get_device_name())
|
||||
|
||||
VALID_PRECISION = [
|
||||
'fp32',
|
||||
'fp16',
|
||||
]
|
||||
|
||||
parser = argparse.ArgumentParser(description="Build and test TensorRT engine.")
|
||||
parser.add_argument('--onnx', required=True, help='ONNX model path (or filename stem if in correctness check mode).')
|
||||
parser.add_argument('--test', nargs='+', help='Test ORT-TRT engine in precision fp32/fp16. You can list multiple precisions to test all of them.')
|
||||
parser.add_argument('--correctness-check', nargs='+', help='Correctness check for original & plugin TRT engines in precision fp32/fp16. You can list multiple precisions to check all of them.')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
ONNX_MODEL = args.onnx
|
||||
MODEL_STEM = os.path.splitext(args.onnx)[0].split('/')[-1]
|
||||
TEST = args.test
|
||||
CORRECTNESS = args.correctness_check
|
||||
|
||||
if TEST:
|
||||
for i in TEST:
|
||||
if i not in VALID_PRECISION:
|
||||
parser.error(f'Unsupported precision {i}')
|
||||
if CORRECTNESS:
|
||||
for i in CORRECTNESS:
|
||||
if i not in VALID_PRECISION:
|
||||
parser.error(f'Unsupported precision {i}')
|
||||
|
||||
def test_engine():
|
||||
|
||||
for precision in TEST:
|
||||
|
||||
engine_cachepath = '/'.join([ENGINE_PATH, '_'.join([MODEL_STEM, gpu_name, precision, 'ort'])])
|
||||
|
||||
providers = [
|
||||
('TensorrtExecutionProvider', {
|
||||
'trt_max_workspace_size': 2147483648,
|
||||
'trt_fp16_enable': precision == 'fp16',
|
||||
'trt_engine_cache_enable': True,
|
||||
'trt_engine_cache_path': engine_cachepath
|
||||
}),
|
||||
'CUDAExecutionProvider'] # EP order indicates priority
|
||||
|
||||
so = ort.SessionOptions()
|
||||
|
||||
sess = ort.InferenceSession(ONNX_MODEL, sess_options=so, providers=providers)
|
||||
|
||||
print(f'Running inference on engine {engine_cachepath}')
|
||||
|
||||
## psuedo-random input test
|
||||
batch_size = 1
|
||||
seq_len = sess.get_inputs()[0].shape[1]
|
||||
vocab = 128203
|
||||
input_ids = torch.randint(0, vocab, (batch_size, seq_len), dtype=torch.long)
|
||||
attention_mask = torch.randint(0, 2, (batch_size, seq_len), dtype=torch.long)
|
||||
inputs = {
|
||||
'input_ids': input_ids.numpy(),
|
||||
'attention_mask': attention_mask.numpy()
|
||||
}
|
||||
|
||||
outputs = sess.run(None, inputs)
|
||||
|
||||
nreps = 100
|
||||
start_time = time()
|
||||
for _ in range(nreps):
|
||||
sess.run(None, inputs)
|
||||
end_time = time()
|
||||
|
||||
duration = end_time - start_time
|
||||
print(f'Average Inference time (ms) of {nreps} runs: {duration/nreps*1000:.3f}. For more accurate test, please use the onnxruntime_perf_test commands.')
|
||||
|
||||
def correctness_check_engines():
|
||||
|
||||
for precision in CORRECTNESS:
|
||||
|
||||
engine_cachepath1 = '/'.join([ENGINE_PATH, '_'.join([MODEL_STEM, 'original', gpu_name, precision, 'ort'])])
|
||||
engine_cachepath2 = '/'.join([ENGINE_PATH, '_'.join([MODEL_STEM, 'plugin', gpu_name, precision, 'ort'])])
|
||||
|
||||
if not os.path.exists(engine_cachepath1) or not os.path.exists(engine_cachepath2):
|
||||
print('At least one of the original and/or plugin engines do not exist. Please build them first by --test')
|
||||
return
|
||||
|
||||
print(f'Running inference on original engine {engine_cachepath1} and plugin engine {engine_cachepath2}')
|
||||
|
||||
so = ort.SessionOptions()
|
||||
|
||||
providers1 = [
|
||||
('TensorrtExecutionProvider', {
|
||||
'trt_max_workspace_size': 2147483648,
|
||||
'trt_fp16_enable': precision == 'fp16',
|
||||
'trt_engine_cache_enable': True,
|
||||
'trt_engine_cache_path': engine_cachepath1
|
||||
}),
|
||||
'CUDAExecutionProvider']
|
||||
|
||||
providers2 = [
|
||||
('TensorrtExecutionProvider', {
|
||||
'trt_max_workspace_size': 2147483648,
|
||||
'trt_fp16_enable': precision == 'fp16',
|
||||
'trt_engine_cache_enable': True,
|
||||
'trt_engine_cache_path': engine_cachepath2
|
||||
}),
|
||||
'CUDAExecutionProvider']
|
||||
|
||||
sess1 = ort.InferenceSession(ONNX_MODEL+'_original.onnx', sess_options=so, providers=providers1)
|
||||
sess2 = ort.InferenceSession(ONNX_MODEL+'_plugin.onnx', sess_options=so, providers=providers2)
|
||||
|
||||
## psuedo-random input test
|
||||
batch_size = 1
|
||||
seq_len = sess1.get_inputs()[0].shape[1]
|
||||
vocab = 128203
|
||||
input_ids = torch.randint(0, vocab, (batch_size, seq_len), dtype=torch.long)
|
||||
attention_mask = torch.randint(0, 2, (batch_size, seq_len), dtype=torch.long)
|
||||
inputs = {
|
||||
'input_ids': input_ids.numpy(),
|
||||
'attention_mask': attention_mask.numpy()
|
||||
}
|
||||
|
||||
outputs1 = sess1.run(None, inputs)
|
||||
outputs2 = sess2.run(None, inputs)
|
||||
|
||||
for i in range(len(outputs1)):
|
||||
avg_abs_error = np.sum(np.abs(outputs1[i] - outputs2[i])) / outputs1[i].size
|
||||
max_abs_error = np.max(np.abs(outputs1[i] - outputs2[i]))
|
||||
print(f"Output {i}:")
|
||||
print("onnx model (original): ", outputs1[i])
|
||||
print("onnx model (plugin): ", outputs2[i])
|
||||
print(f"[Output {i} Element-wise Check] Avgerage absolute error: {avg_abs_error:e}, Maximum absolute error: {max_abs_error:e}. 1e-2~1e-3 expected for FP16 (10 significance bits) and 1e-6~1e-7 expected for FP32 (23 significance bits) " ) # machine epsilon for different precisions: https://en.wikipedia.org/wiki/Machine_epsilon
|
||||
|
||||
if TEST:
|
||||
test_engine()
|
||||
|
||||
if CORRECTNESS:
|
||||
correctness_check_engines()
|
||||
@@ -0,0 +1,134 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
'''
|
||||
Generate HuggingFace DeBERTa (V2) model with different configurations (e.g., sequence length, hidden size, No. of layers, No. of heads, etc.) and export in ONNX format
|
||||
|
||||
Usage:
|
||||
python deberta_pytorch2onnx.py [--filename xx.onnx] [--variant microsoft/deberta-xx] [--seq-len xx]
|
||||
'''
|
||||
|
||||
import os, time, argparse
|
||||
from transformers import DebertaV2Tokenizer, DebertaV2Config, DebertaV2ForSequenceClassification
|
||||
# DEBERTA V2 implementation, https://github.com/huggingface/transformers/blob/master/src/transformers/models/deberta_v2/modeling_deberta_v2.py
|
||||
import torch, onnxruntime as ort, numpy as np
|
||||
|
||||
parser = argparse.ArgumentParser(description="Generate HuggingFace DeBERTa (V2) model with different configurations and export in ONNX format. This will save the model under the same directory as 'deberta_seqxxx_hf.onnx'.")
|
||||
parser.add_argument('--filename', type=str, help='Path to the save the ONNX model')
|
||||
parser.add_argument('--variant', type=str, default=None, help='DeBERTa variant name. Such as microsoft/deberta-v3-xsmall')
|
||||
parser.add_argument('--seq-len', type=int, default=None, help='Specify maximum sequence length. Note: --variant and --seq-len cannot be used together. Pre-trained models have pre-defined sequence length')
|
||||
|
||||
args = parser.parse_args()
|
||||
onnx_filename = args.filename
|
||||
model_variant = args.variant
|
||||
sequence_length = args.seq_len
|
||||
|
||||
assert not args.variant or (args.variant and not args.seq_len), "--variant and --seq-len cannot be used together!"
|
||||
assert torch.cuda.is_available(), "CUDA not available!"
|
||||
|
||||
def randomize_model(model):
|
||||
for module_ in model.named_modules():
|
||||
if isinstance(module_[1],(torch.nn.Linear, torch.nn.Embedding)):
|
||||
module_[1].weight.data.normal_(mean=0.0, std=model.config.initializer_range)
|
||||
elif isinstance(module_[1], torch.nn.LayerNorm):
|
||||
module_[1].bias.data.zero_()
|
||||
module_[1].weight.data.fill_(1.0)
|
||||
if isinstance(module_[1], torch.nn.Linear) and module_[1].bias is not None:
|
||||
module_[1].bias.data.zero_()
|
||||
return model
|
||||
|
||||
def export():
|
||||
parent_dir = os.path.dirname(onnx_filename)
|
||||
if not os.path.exists(parent_dir):
|
||||
os.makedirs(parent_dir)
|
||||
|
||||
if model_variant is None:
|
||||
# default model hyper-params
|
||||
batch_size = 1
|
||||
seq_len = 2048 if sequence_length is None else sequence_length
|
||||
max_position_embeddings = 512 if seq_len <= 512 else seq_len # maximum sequence length that this model might ever be used with. By default 512. otherwise error https://github.com/huggingface/transformers/issues/4542
|
||||
vocab_size = 128203
|
||||
hidden_size = 384
|
||||
layers = 12
|
||||
heads = 6
|
||||
intermediate_size = hidden_size*4 # feed forward layer dimension
|
||||
type_vocab_size = 0
|
||||
# relative attention
|
||||
relative_attention=True
|
||||
max_relative_positions = 256 # k
|
||||
pos_att_type = ["p2c", "c2p"]
|
||||
|
||||
deberta_config = DebertaV2Config(vocab_size=vocab_size, hidden_size=hidden_size, num_hidden_layers=layers, num_attention_heads=heads, intermediate_size=intermediate_size, type_vocab_size=type_vocab_size, max_position_embeddings=max_position_embeddings, relative_attention=relative_attention, max_relative_positions=max_relative_positions, pos_att_type=pos_att_type)
|
||||
deberta_model = DebertaV2ForSequenceClassification(deberta_config)
|
||||
deberta_model = randomize_model(deberta_model)
|
||||
else:
|
||||
deberta_model = DebertaV2ForSequenceClassification.from_pretrained(model_variant)
|
||||
deberta_config = DebertaV2Config.from_pretrained(model_variant)
|
||||
|
||||
batch_size = 1
|
||||
seq_len = deberta_config.max_position_embeddings
|
||||
vocab_size = deberta_config.vocab_size
|
||||
|
||||
deberta_model.cuda().eval()
|
||||
|
||||
# input/output
|
||||
gpu = torch.device('cuda')
|
||||
input_ids = torch.randint(0, vocab_size, (batch_size, seq_len), dtype=torch.long, device=gpu)
|
||||
attention_mask = torch.randint(0, 2, (batch_size, seq_len), dtype=torch.long, device=gpu)
|
||||
input_names = ['input_ids', 'attention_mask']
|
||||
output_names = ['output']
|
||||
dynamic_axes={'input_ids' : {0 : 'batch_size'},
|
||||
'attention_mask' : {0 : 'batch_size'},
|
||||
'output' : {0 : 'batch_size'}}
|
||||
|
||||
# ONNX export
|
||||
torch.onnx.export(deberta_model, # model
|
||||
(input_ids, attention_mask), # model inputs
|
||||
onnx_filename,
|
||||
export_params=True,
|
||||
opset_version=13,
|
||||
do_constant_folding=True,
|
||||
input_names = input_names,
|
||||
output_names = output_names,
|
||||
dynamic_axes = dynamic_axes)
|
||||
|
||||
# full precision inference
|
||||
num_trials = 10
|
||||
|
||||
start = time.time()
|
||||
for i in range(num_trials):
|
||||
results = deberta_model(input_ids, attention_mask)
|
||||
end = time.time()
|
||||
|
||||
print("Average PyTorch FP32(TF32) time: {:.2f} ms".format((end - start)/num_trials*1000))
|
||||
|
||||
# half precision inference (do this after onnx export, otherwise the export ONNX model is with FP16 weights...)
|
||||
deberta_model_fp16 = deberta_model.half()
|
||||
start = time.time()
|
||||
for i in range(num_trials):
|
||||
results = deberta_model_fp16(input_ids, attention_mask)
|
||||
end = time.time()
|
||||
|
||||
print("Average PyTorch FP16 time: {:.2f} ms".format((end - start)/num_trials*1000))
|
||||
|
||||
# model size
|
||||
total_params = sum(param.numel() for param in deberta_model.parameters())
|
||||
print("Total # of params: ", total_params)
|
||||
print("Maximum sequence length: ", seq_len)
|
||||
|
||||
if __name__ == "__main__":
|
||||
export()
|
||||
@@ -0,0 +1,401 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
Build and test TensorRT engines generated from the DeBERTa model. Different precisions are supported.
|
||||
|
||||
Usage:
|
||||
Build and test a model:
|
||||
- build: python deberta_tensorrt_inference.py --onnx=xx.onnx --build fp16 # build TRT engines
|
||||
- test: python deberta_tensorrt_inference.py --onnx=xx.onnx --test fp16 # test will measure the inference time
|
||||
- build and test: python deberta_tensorrt_inference.py --onnx=xx.onnx --build fp16 --test fp16
|
||||
|
||||
Correctness check is done by comparing engines generated from the original model and the plugin model:
|
||||
- [1] export ONNX model with extra output nodes: python deberta_onnx_modify.py xx.onnx --correctness-check
|
||||
- [2] build original model: python deberta_tensorrt_inference.py --onnx=xx_correctness_check_original.onnx --build fp16
|
||||
- [3] build plugin model: python deberta_tensorrt_inference.py --onnx=xx_correctness_check_plugin.onnx --build fp16
|
||||
- [4] correctness check: python deberta_tensorrt_inference.py --onnx=deberta --correctness_check fp16
|
||||
|
||||
Notes:
|
||||
- supported precisions are fp32/tf32/fp16. For both --build and --test, you can specify more than one precisions, and TensorRT engines of each precision will be built sequentially.
|
||||
- engine files are saved as `**/[Model name]_[GPU name]_[Precision].engine`. Note that TensorRT engines are specific to both GPU architecture and TensorRT version, and therefore are not compatible cross-version nor cross-device.
|
||||
- in --correctness-check mode, the argument for --onnx is the `root` name for the models [root]_correctness_check_original/plugin.onnx
|
||||
"""
|
||||
|
||||
import torch
|
||||
import tensorrt as trt
|
||||
import os, sys, argparse
|
||||
import numpy as np
|
||||
from time import time
|
||||
from cuda.bindings import driver as cuda, runtime as cudart
|
||||
from cuda_utils import (
|
||||
cuda_call,
|
||||
CudaStreamContext,
|
||||
memcpy_host_to_device_async,
|
||||
memcpy_device_to_host_async,
|
||||
memcpy_host_to_device,
|
||||
memcpy_device_to_device_async,
|
||||
memcpy_device_to_device,
|
||||
)
|
||||
|
||||
TRT_VERSION = int(trt.__version__[:3].replace('.','')) # e.g., version 8.4.1.5 becomes 84
|
||||
|
||||
def GPU_ABBREV(name):
|
||||
'''
|
||||
Map GPU device query name to abbreviation.
|
||||
|
||||
::param str name Device name from torch.cuda.get_device_name().
|
||||
::return str GPU abbreviation.
|
||||
'''
|
||||
|
||||
GPU_LIST = [
|
||||
'V100',
|
||||
'TITAN',
|
||||
'T4',
|
||||
'A100',
|
||||
'A10G',
|
||||
'A10'
|
||||
]
|
||||
# Partial list, can be extended. The order of A100, A10G, A10 matters. They're put in a way to not detect substring A10 as A100
|
||||
|
||||
for i in GPU_LIST:
|
||||
if i in name:
|
||||
return i
|
||||
|
||||
return 'GPU' # for names not in the partial list, use 'GPU' as default
|
||||
|
||||
gpu_name = GPU_ABBREV(torch.cuda.get_device_name())
|
||||
|
||||
VALID_PRECISION = [
|
||||
'fp32',
|
||||
'tf32',
|
||||
'fp16'
|
||||
]
|
||||
|
||||
parser = argparse.ArgumentParser(description="Build and test TensorRT engine.")
|
||||
parser.add_argument('--onnx', required=True, help='ONNX model path (or filename stem if in correctness check mode).')
|
||||
parser.add_argument('--build', nargs='+', help='Build TRT engine in precision fp32/tf32/fp16. You can list multiple precisions to build all of them.')
|
||||
parser.add_argument('--test', nargs='+', help='Test TRT engine in precision fp32/tf32/fp16. You can list multiple precisions to test all of them.')
|
||||
parser.add_argument('--correctness-check', nargs='+', help='Correctness check for original & plugin TRT engines in precision fp32/tf32/fp16. You can list multiple precisions to check all of them.')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
ONNX_MODEL = args.onnx
|
||||
MODEL_NAME = os.path.splitext(args.onnx)[0]
|
||||
BUILD = args.build
|
||||
TEST = args.test
|
||||
CORRECTNESS = args.correctness_check
|
||||
|
||||
if not (args.build or args.test or args.correctness_check):
|
||||
parser.error('Please specify --build and/or --test and/or --correctness-check' )
|
||||
|
||||
if BUILD:
|
||||
for i in BUILD:
|
||||
if i not in VALID_PRECISION:
|
||||
parser.error(f'Unsupported precision {i}')
|
||||
if TEST:
|
||||
for i in TEST:
|
||||
if i not in VALID_PRECISION:
|
||||
parser.error(f'Unsupported precision {i}')
|
||||
if CORRECTNESS:
|
||||
for i in CORRECTNESS:
|
||||
if i not in VALID_PRECISION:
|
||||
parser.error(f'Unsupported precision {i}')
|
||||
|
||||
class TRTModel:
|
||||
'''
|
||||
Generic class to run a TRT engine by specifying engine path and giving input data.
|
||||
'''
|
||||
class HostDeviceMem(object):
|
||||
'''
|
||||
Helper class to record host-device memory pointer pairs
|
||||
'''
|
||||
def __init__(self, host_mem, device_mem):
|
||||
self.host = host_mem
|
||||
self.device = device_mem
|
||||
|
||||
def __str__(self):
|
||||
return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device)
|
||||
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
def __init__(self, engine_path):
|
||||
self.engine_path = engine_path
|
||||
self.logger = trt.Logger(trt.Logger.WARNING)
|
||||
self.runtime = trt.Runtime(self.logger)
|
||||
|
||||
# load and deserialize TRT engine
|
||||
self.engine = self.load_engine()
|
||||
|
||||
# allocate input/output memory buffers
|
||||
self.inputs, self.outputs, self.bindings, self.stream = self.allocate_buffers(self.engine)
|
||||
|
||||
# create context
|
||||
self.context = self.engine.create_execution_context()
|
||||
|
||||
# Dict of NumPy dtype -> torch dtype (when the correspondence exists). From: https://github.com/pytorch/pytorch/blob/e180ca652f8a38c479a3eff1080efe69cbc11621/torch/testing/_internal/common_utils.py#L349
|
||||
self.numpy_to_torch_dtype_dict = {
|
||||
bool : torch.bool,
|
||||
np.uint8 : torch.uint8,
|
||||
np.int8 : torch.int8,
|
||||
np.int16 : torch.int16,
|
||||
np.int32 : torch.int32,
|
||||
np.int64 : torch.int64,
|
||||
np.float16 : torch.float16,
|
||||
np.float32 : torch.float32,
|
||||
np.float64 : torch.float64,
|
||||
np.complex64 : torch.complex64,
|
||||
np.complex128 : torch.complex128
|
||||
}
|
||||
|
||||
def load_engine(self):
|
||||
with open(self.engine_path, 'rb') as f:
|
||||
engine = self.runtime.deserialize_cuda_engine(f.read())
|
||||
return engine
|
||||
|
||||
def allocate_buffers(self, engine):
|
||||
'''
|
||||
Allocates all buffers required for an engine, i.e. host/device inputs/outputs.
|
||||
'''
|
||||
inputs = []
|
||||
outputs = []
|
||||
bindings = []
|
||||
stream = CudaStreamContext()
|
||||
|
||||
for i in range(engine.num_io_tensors):
|
||||
tensor_name = engine.get_tensor_name(i)
|
||||
size = trt.volume(engine.get_tensor_shape(tensor_name))
|
||||
dtype = trt.nptype(engine.get_tensor_dtype(tensor_name))
|
||||
|
||||
# Allocate host and device buffers
|
||||
host_mem = np.empty(size, dtype)
|
||||
device_mem = cuda_call(cudart.cudaMalloc(host_mem.nbytes))
|
||||
|
||||
# Append the device buffer address to device bindings. When cast to int, it's a linear index into the context's memory (like memory address).
|
||||
bindings.append(int(device_mem))
|
||||
|
||||
# Append to the appropriate input/output list.
|
||||
if engine.get_tensor_mode(tensor_name) == trt.TensorIOMode.INPUT:
|
||||
inputs.append(self.HostDeviceMem(host_mem, device_mem))
|
||||
else:
|
||||
outputs.append(self.HostDeviceMem(host_mem, device_mem))
|
||||
|
||||
return inputs, outputs, bindings, stream
|
||||
|
||||
def __call__(self, model_inputs: list, timing=False):
|
||||
'''
|
||||
Inference step (like forward() in PyTorch).
|
||||
|
||||
model_inputs: list of numpy array or list of torch.Tensor (on GPU)
|
||||
'''
|
||||
NUMPY = False
|
||||
TORCH = False
|
||||
if isinstance(model_inputs[0], np.ndarray):
|
||||
NUMPY = True
|
||||
elif torch.is_tensor(model_inputs[0]):
|
||||
TORCH = True
|
||||
else:
|
||||
assert False, 'Unsupported input data format!'
|
||||
|
||||
# batch size consistency check
|
||||
if NUMPY:
|
||||
batch_size = np.unique(np.array([i.shape[0] for i in model_inputs]))
|
||||
elif TORCH:
|
||||
batch_size = np.unique(np.array([i.size(dim=0) for i in model_inputs]))
|
||||
assert len(batch_size) == 1, 'Input batch sizes are not consistent!'
|
||||
batch_size = batch_size[0]
|
||||
|
||||
for i, model_input in enumerate(model_inputs):
|
||||
binding_name = self.engine.get_tensor_name(i) # i-th input/output name
|
||||
binding_dtype = trt.nptype(self.engine.get_tensor_dtype(binding_name)) # trt can only tell to numpy dtype
|
||||
|
||||
# input type cast
|
||||
if NUMPY:
|
||||
model_input = model_input.astype(binding_dtype)
|
||||
elif TORCH:
|
||||
model_input = model_input.to(self.numpy_to_torch_dtype_dict[binding_dtype])
|
||||
|
||||
if NUMPY:
|
||||
# fill host memory with flattened input data
|
||||
np.copyto(self.inputs[i].host, model_input.ravel())
|
||||
elif TORCH:
|
||||
nbytes = model_input.element_size() * model_input.nelement()
|
||||
if timing:
|
||||
memcpy_device_to_device(self.inputs[i].device, model_input.data_ptr(), nbytes)
|
||||
else:
|
||||
# for Torch GPU tensor it's easier, can just do Device to Device copy
|
||||
memcpy_device_to_device_async(self.inputs[i].device, model_input.data_ptr(), nbytes, self.stream.stream)
|
||||
|
||||
if NUMPY:
|
||||
if timing:
|
||||
[memcpy_host_to_device(inp.device, inp.host) for inp in self.inputs]
|
||||
else:
|
||||
# input, Host to Device
|
||||
[memcpy_host_to_device_async(inp.device, inp.host, self.stream.stream) for inp in self.inputs]
|
||||
|
||||
for i in range(self.engine.num_io_tensors):
|
||||
self.context.set_tensor_address(self.engine.get_tensor_name(i), self.bindings[i])
|
||||
|
||||
duration = 0
|
||||
if timing:
|
||||
start_time = time()
|
||||
self.context.execute_v2(bindings=self.bindings)
|
||||
end_time = time()
|
||||
duration = end_time - start_time
|
||||
else:
|
||||
# run inference
|
||||
self.context.execute_async_v3(stream_handle=self.stream.stream)
|
||||
|
||||
if timing:
|
||||
[cuda_call(cudart.cudaMemcpy(out.host.ctypes.data, out.device, out.host.nbytes, cudart.cudaMemcpyKind.cudaMemcpyDeviceToHost)) for out in self.outputs]
|
||||
else:
|
||||
# output, Device to Host
|
||||
[memcpy_device_to_host_async(out.host, out.device, self.stream.stream) for out in self.outputs]
|
||||
|
||||
if not timing:
|
||||
# synchronize to ensure completion of async calls
|
||||
self.stream.synchronize()
|
||||
|
||||
if NUMPY:
|
||||
return [out.host.reshape(batch_size,-1) for out in self.outputs], duration
|
||||
elif TORCH:
|
||||
return [torch.from_numpy(out.host.reshape(batch_size,-1)) for out in self.outputs], duration
|
||||
|
||||
def build_engine():
|
||||
|
||||
TRT_LOGGER = trt.Logger(trt.Logger.INFO)
|
||||
TRT_BUILDER = trt.Builder(TRT_LOGGER)
|
||||
|
||||
for precision in BUILD:
|
||||
engine_filename = '_'.join([MODEL_NAME, gpu_name, precision]) + '.engine'
|
||||
if os.path.exists(engine_filename):
|
||||
print(f'Engine file {engine_filename} exists. Skip building...')
|
||||
continue
|
||||
|
||||
print(f'Building {precision} engine of {MODEL_NAME} model on {gpu_name} GPU...')
|
||||
|
||||
## parse ONNX model
|
||||
network_creation_flag = 0
|
||||
if "EXPLICIT_BATCH" in trt.NetworkDefinitionCreationFlag.__members__.keys():
|
||||
network_creation_flag = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)
|
||||
network = TRT_BUILDER.create_network(network_creation_flag)
|
||||
onnx_parser = trt.OnnxParser(network, TRT_LOGGER)
|
||||
parse_success = onnx_parser.parse_from_file(ONNX_MODEL)
|
||||
for idx in range(onnx_parser.num_errors):
|
||||
print(onnx_parser.get_error(idx))
|
||||
if not parse_success:
|
||||
sys.exit('ONNX model parsing failed')
|
||||
|
||||
## build TRT engine (configuration options at: https://docs.nvidia.com/deeplearning/tensorrt/api/python_api/infer/Core/BuilderConfig.html#ibuilderconfig)
|
||||
config = TRT_BUILDER.create_builder_config()
|
||||
|
||||
seq_len = network.get_input(0).shape[1]
|
||||
|
||||
# handle dynamic shape (min/opt/max): https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#work_dynamic_shapes
|
||||
# by default batch dim set as 1 for all min/opt/max. If there are batch need, change the value for opt and max accordingly
|
||||
profile = TRT_BUILDER.create_optimization_profile()
|
||||
profile.set_shape("input_ids", (1,seq_len), (1,seq_len), (1,seq_len))
|
||||
profile.set_shape("attention_mask", (1,seq_len), (1,seq_len), (1,seq_len))
|
||||
config.add_optimization_profile(profile)
|
||||
config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 4096 * (1 << 20)) # 4096 MiB
|
||||
|
||||
# precision
|
||||
if precision == 'fp32':
|
||||
config.clear_flag(trt.BuilderFlag.TF32) # TF32 enabled by default, need to clear flag
|
||||
elif precision == 'tf32':
|
||||
pass
|
||||
elif precision == 'fp16':
|
||||
config.set_flag(trt.BuilderFlag.FP16)
|
||||
|
||||
# build
|
||||
serialized_engine = TRT_BUILDER.build_serialized_network(network, config)
|
||||
|
||||
## save TRT engine
|
||||
with open(engine_filename, 'wb') as f:
|
||||
f.write(serialized_engine)
|
||||
print(f'Engine is saved to {engine_filename}')
|
||||
|
||||
def test_engine():
|
||||
|
||||
for precision in TEST:
|
||||
## load and deserialize TRT engine
|
||||
engine_filename = '_'.join([MODEL_NAME, gpu_name, precision]) + '.engine'
|
||||
print(f'Running inference on engine {engine_filename}')
|
||||
|
||||
model = TRTModel(engine_filename)
|
||||
|
||||
## psuedo-random input test
|
||||
batch_size = 1
|
||||
seq_len = model.engine.get_tensor_shape(model.engine.get_tensor_name(0))[1]
|
||||
vocab = 128203
|
||||
gpu = torch.device('cuda')
|
||||
torch.manual_seed(0) # make sure in each test the seed are the same
|
||||
input_ids = torch.randint(0, vocab, (batch_size, seq_len), dtype=torch.long, device=gpu)
|
||||
attention_mask = torch.randint(0, 2, (batch_size, seq_len), dtype=torch.long, device=gpu)
|
||||
inputs = [input_ids, attention_mask]
|
||||
outputs, duration = model(inputs, timing=True)
|
||||
|
||||
nreps = 100
|
||||
duration_total = 0
|
||||
for _ in range(nreps):
|
||||
outputs, duration = model(inputs, timing=True)
|
||||
duration_total += duration
|
||||
|
||||
print(f'Average Inference time (ms) of {nreps} runs: {duration_total/nreps*1000:.3f}')
|
||||
|
||||
def correctness_check_engines():
|
||||
for precision in CORRECTNESS:
|
||||
## load and deserialize TRT engine
|
||||
engine_filename1 = '_'.join([ONNX_MODEL, 'correctness_check_original', gpu_name, precision]) + '.engine'
|
||||
engine_filename2 = '_'.join([ONNX_MODEL, 'correctness_check_plugin', gpu_name, precision]) + '.engine'
|
||||
|
||||
assert os.path.exists(engine_filename1), f'Engine file {engine_filename1} does not exist. Please build the engine first by --build'
|
||||
assert os.path.exists(engine_filename2), f'Engine file {engine_filename2} does not exist. Please build the engine first by --build'
|
||||
|
||||
print(f'Running inference on original engine {engine_filename1} and plugin engine {engine_filename2}')
|
||||
|
||||
model1 = TRTModel(engine_filename1)
|
||||
model2 = TRTModel(engine_filename2)
|
||||
|
||||
## psuedo-random input test
|
||||
batch_size = 1
|
||||
seq_len = model1.engine.get_tensor_shape(model1.engine.get_tensor_name(0))[1]
|
||||
vocab = 128203
|
||||
gpu = torch.device('cuda')
|
||||
# torch.manual_seed(0) # make sure in each test the seed are the same
|
||||
input_ids = torch.randint(0, vocab, (batch_size, seq_len), dtype=torch.long, device=gpu)
|
||||
attention_mask = torch.randint(0, 2, (batch_size, seq_len), dtype=torch.long, device=gpu)
|
||||
inputs = [input_ids, attention_mask]
|
||||
|
||||
outputs1, _ = model1(inputs)
|
||||
outputs2, _ = model2(inputs)
|
||||
|
||||
# element-wise and layer-wise output comparison
|
||||
for i in range(len(outputs1)):
|
||||
avg_abs_error = torch.sum(torch.abs(torch.sub(outputs1[i], outputs2[i]))) / torch.numel(outputs1[i])
|
||||
max_abs_error = torch.max(torch.abs(torch.sub(outputs1[i], outputs2[i])))
|
||||
print(f"[Layer {i} Element-wise Check] Avgerage absolute error: {avg_abs_error.item():e}, Maximum absolute error: {max_abs_error.item():e}. 1e-2~1e-3 expected for FP16 (10 significance bits) and 1e-6~1e-7 expected for FP32 (23 significance bits)" ) # machine epsilon for different precisions: https://en.wikipedia.org/wiki/Machine_epsilon
|
||||
|
||||
if BUILD:
|
||||
build_engine()
|
||||
|
||||
if TEST:
|
||||
test_engine()
|
||||
|
||||
if CORRECTNESS:
|
||||
correctness_check_engines()
|
||||
@@ -0,0 +1,24 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
-f https://download.pytorch.org/whl/cu113/torch_stable.html
|
||||
torch==1.11.0+cu113
|
||||
transformers==4.18.0
|
||||
onnxruntime-gpu>=1.12
|
||||
argparse
|
||||
--extra-index-url https://pypi.ngc.nvidia.com
|
||||
polygraphy
|
||||
@@ -0,0 +1,6 @@
|
||||
__pycache__/
|
||||
onnx/
|
||||
engine/
|
||||
output/
|
||||
pytorch_model/
|
||||
artifacts_cache/
|
||||
Executable
+538
@@ -0,0 +1,538 @@
|
||||
# Introduction
|
||||
|
||||
This demo application ("demoDiffusion") showcases the acceleration of Stable Diffusion and ControlNet pipeline using TensorRT.
|
||||
|
||||
# Setup
|
||||
|
||||
### Clone the TensorRT OSS repository
|
||||
|
||||
```bash
|
||||
git clone git@github.com:NVIDIA/TensorRT.git -b release/11.0 --single-branch
|
||||
cd TensorRT
|
||||
```
|
||||
|
||||
### Launch NVIDIA pytorch container
|
||||
|
||||
Install nvidia-docker using [these intructions](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker).
|
||||
|
||||
```bash
|
||||
# Create a directory for persistent dependencies
|
||||
mkdir -p deps
|
||||
|
||||
# Launch container with volume mounts
|
||||
docker run --rm -it --gpus all \
|
||||
-v $PWD:/workspace \
|
||||
-v $PWD/deps:/workspace/deps \
|
||||
nvcr.io/nvidia/pytorch:26.03-py3 /bin/bash
|
||||
```
|
||||
|
||||
> **NOTE:** Mounting `/workspace/deps` as a volume ensures dependencies persist across container restarts. After initial installation, subsequent container launches will reuse the installed dependencies.
|
||||
|
||||
NOTE: The demo supports CUDA>=13.0
|
||||
|
||||
### Install the required packages
|
||||
|
||||
This demo uses a family-based dependency management system. Install dependencies for the model families you want to use:
|
||||
|
||||
**Install all dependencies (recommended for first-time users):**
|
||||
```bash
|
||||
python3 setup.py all
|
||||
```
|
||||
|
||||
**Or install specific model families:**
|
||||
```bash
|
||||
# SD family: SD 1.4, SDXL, SD3, SD3.5, SVD (Stable Video Diffusion), Stable Cascade
|
||||
python3 setup.py sd
|
||||
|
||||
# Flux family: Flux.1-dev, Flux.1-schnell, Flux.1-Canny, Flux.1-Depth, Flux.1-Kontext
|
||||
python3 setup.py flux
|
||||
|
||||
# Cosmos family: Cosmos-Predict2 text2image, video2world
|
||||
python3 setup.py cosmos
|
||||
```
|
||||
|
||||
**Additional options:**
|
||||
```bash
|
||||
# Force reinstall even if already installed
|
||||
python3 setup.py all --force
|
||||
|
||||
# Install dependencies to a custom location
|
||||
# Option 1 (recommended): set the env var and install
|
||||
export TENSORRT_DIFFUSION_DEPS_ROOT=/custom/path/deps
|
||||
python3 setup.py all
|
||||
|
||||
# Option 2: install to a path without changing this shell
|
||||
# Remember to export the env var in the environment that runs the demos,
|
||||
# so deps.configure() can find the custom path.
|
||||
python3 setup.py all --deps-root /custom/path/deps
|
||||
# Then, before running any demo scripts:
|
||||
export TENSORRT_DIFFUSION_DEPS_ROOT=/custom/path/deps
|
||||
```
|
||||
|
||||
**Check installation status:**
|
||||
```bash
|
||||
python3 -c "from demo_diffusion import deps; deps.print_status()"
|
||||
```
|
||||
|
||||
Check your installed TensorRT version using:
|
||||
```bash
|
||||
python3 -c 'import tensorrt; print(tensorrt.__version__)'
|
||||
```
|
||||
|
||||
> NOTE: Alternatively, you can download and install TensorRT packages from [NVIDIA TensorRT Developer Zone](https://developer.nvidia.com/tensorrt).
|
||||
|
||||
> NOTE: demoDiffusion has been tested on systems with NVIDIA H100, A100, L40, T4, and RTX4090 GPUs, and the following software configuration.
|
||||
|
||||
|
||||
# Running demoDiffusion
|
||||
|
||||
### Review usage instructions for the supported pipelines
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img.py --help
|
||||
python3 demo_img2img.py --help
|
||||
python3 demo_controlnet.py --help
|
||||
python3 demo_txt2img_xl.py --help
|
||||
python3 demo_txt2img_flux.py --help
|
||||
python3 demo_txt2vid_wan.py --help
|
||||
```
|
||||
|
||||
### HuggingFace user access token
|
||||
|
||||
To download model checkpoints for the Stable Diffusion pipelines, obtain a `read` access token to HuggingFace Hub. See [instructions](https://huggingface.co/docs/hub/security-tokens).
|
||||
|
||||
```bash
|
||||
export HF_TOKEN=<your access token>
|
||||
```
|
||||
|
||||
### Generate an image guided by a text prompt
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN
|
||||
```
|
||||
|
||||
### Faster Text-to-image using SD1.4 INT8 & FP8 quantization using ModelOpt
|
||||
|
||||
Run the below command to generate an image with SD1.4 in INT8
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --int8
|
||||
```
|
||||
|
||||
Run the below command to generate an image with SD1.4 in FP8. (FP8 is only supported on Hopper and Ada.)
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --fp8
|
||||
```
|
||||
|
||||
### Generate an image guided by an initial image and a text prompt
|
||||
|
||||
```bash
|
||||
wget https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg -O sketch-mountains-input.jpg
|
||||
|
||||
python3 demo_img2img.py "A fantasy landscape, trending on artstation" --hf-token=$HF_TOKEN --input-image=sketch-mountains-input.jpg
|
||||
```
|
||||
|
||||
### Generate an image with ControlNet guided by image(s) and text prompt(s)
|
||||
|
||||
```bash
|
||||
python3 demo_controlnet.py "Stormtrooper's lecture in beautiful lecture hall" --controlnet-type depth --hf-token=$HF_TOKEN --denoising-steps 20 --onnx-dir=onnx-cnet-depth --engine-dir=engine-cnet-depth
|
||||
```
|
||||
|
||||
> NOTE: `--input-image` must be a pre-processed image corresponding to `--controlnet-type`. If unspecified, a sample image will be downloaded. Supported controlnet types include: `canny`, `depth`, `hed`, `mlsd`, `normal`, `openpose`, `scribble`, and `seg`.
|
||||
|
||||
Examples:
|
||||
<img src="https://drive.google.com/uc?export=view&id=17ub3MVSQHp26ty-wioNX6iQQ-nAveYSV" alt= “” width="800" height="400">
|
||||
|
||||
#### Combining multiple conditionings
|
||||
|
||||
Multiple ControlNet types can also be specified to combine the conditionings. While specifying multiple conditionings, controlnet scales should also be provided. The scales signify the importance of each conditioning in relation with the other. For example, to condition using `openpose` and `canny` with scales of 1.0 and 0.8 respectively, the arguments provided would be `--controlnet-type openpose canny` and `--controlnet-scale 1.0 0.8`. Note that the number of controlnet scales provided should match the number of controlnet types.
|
||||
|
||||
### Generate an image with Stable Diffusion XL guided by a single text prompt
|
||||
|
||||
> **NOTE:** SDXL and later Stable Diffusion models require sd dependencies to be installed. Install with: `python3 setup.py sd`
|
||||
|
||||
Run the below command to generate an image with Stable Diffusion XL
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img_xl.py "a photo of an astronaut riding a horse on mars" --hf-token=$HF_TOKEN --version=xl-1.0
|
||||
```
|
||||
|
||||
The optional refiner model may be enabled by specifying `--enable-refiner` and separate directories for storing refiner onnx and engine files using `--onnx-refiner-dir` and `--engine-refiner-dir` respectively.
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img_xl.py "a photo of an astronaut riding a horse on mars" --hf-token=$HF_TOKEN --version=xl-1.0 --enable-refiner --onnx-refiner-dir=onnx-refiner --engine-refiner-dir=engine-refiner
|
||||
```
|
||||
|
||||
### Generate an image with Stable Diffusion XL with ControlNet guided by an image and a text prompt
|
||||
|
||||
```bash
|
||||
python3 demo_controlnet.py "A beautiful bird with rainbow colors" --controlnet-type canny --hf-token=$HF_TOKEN --denoising-steps 20 --onnx-dir=onnx-cnet --engine-dir=engine-cnet --version xl-1.0
|
||||
```
|
||||
|
||||
> NOTE: Currently only `--controlnet-type canny` is supported. `--input-image` must be a pre-processed image corresponding to `--controlnet-type canny`. If unspecified, a sample image will be downloaded.
|
||||
|
||||
> NOTE: FP8 quantization (`--fp8`) is supported.
|
||||
|
||||
### Generate an image guided by a text prompt, and using specified LoRA model weight updates
|
||||
|
||||
```bash
|
||||
# FP16
|
||||
python3 demo_txt2img_xl.py "Picture of a rustic Italian village with Olive trees and mountains" --version=xl-1.0 --lora-path "ostris/crayon_style_lora_sdxl" "ostris/watercolor_style_lora_sdxl" --lora-weight 0.3 0.7 --onnx-dir onnx-sdxl-lora --engine-dir engine-sdxl-lora --build-enable-refit
|
||||
|
||||
# FP8
|
||||
python3 demo_txt2img_xl.py "Picture of a rustic Italian village with Olive trees and mountains" --version=xl-1.0 --lora-path "ostris/crayon_style_lora_sdxl" "ostris/watercolor_style_lora_sdxl" --lora-weight 0.3 0.7 --onnx-dir onnx-sdxl-lora --engine-dir engine-sdxl-lora --fp8
|
||||
```
|
||||
|
||||
### Faster Text-to-image using SDXL INT8 & FP8 quantization using ModelOpt
|
||||
|
||||
Run the below command to generate an image with Stable Diffusion XL in INT8
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img_xl.py "a photo of an astronaut riding a horse on mars" --version xl-1.0 --onnx-dir onnx-sdxl --engine-dir engine-sdxl --int8
|
||||
```
|
||||
|
||||
Run the below command to generate an image with Stable Diffusion XL in FP8. (FP8 is only supported on Hopper and Ada.)
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img_xl.py "a photo of an astronaut riding a horse on mars" --version xl-1.0 --onnx-dir onnx-sdxl --engine-dir engine-sdxl --fp8
|
||||
```
|
||||
|
||||
> Note that INT8 & FP8 quantization is only supported for SDXL, and won't work with LoRA weights. FP8 quantization is only supported on Hopper and Ada. Some prompts may produce better inputs with fewer denoising steps (e.g. `--denoising-steps 20`) but this will repeat the calibration, ONNX export, and engine building processes for the U-Net.
|
||||
|
||||
For step-by-step tutorials to run INT8 & FP8 inference on stable diffusion models, please refer to examples in [TensorRT ModelOpt diffusers sample](https://github.com/NVIDIA/TensorRT-Model-Optimizer/tree/main/diffusers).
|
||||
|
||||
### Faster Text-to-Image using SDXL Turbo
|
||||
|
||||
Produce coherent images in just 1 step. Note: SDXL Turbo works best for 512x512 resolution, EulerA scheduler and classifier-free-guidance disabled.
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img_xl.py "Einstein" --version xl-turbo --onnx-dir onnx-sdxl-turbo --engine-dir engine-sdxl-turbo --denoising-steps 1 --scheduler EulerA --guidance-scale 0.0 --width 512 --height 512
|
||||
```
|
||||
|
||||
### Generate an image guided by a text prompt using Stable Diffusion 3 and its variants
|
||||
|
||||
Run the command below to generate an image using Stable Diffusion 3 and Stable Diffusion 3.5
|
||||
|
||||
```bash
|
||||
# Stable Diffusion 3
|
||||
python3 demo_txt2img_sd3.py "A vibrant street wall covered in colorful graffiti, the centerpiece spells \"SD3 MEDIUM\", in a storm of colors" --version sd3 --hf-token=$HF_TOKEN
|
||||
|
||||
# Stable Diffusion 3.5-medium
|
||||
python3 demo_txt2img_sd35.py "a beautiful photograph of Mt. Fuji during cherry blossom" --version=3.5-medium --denoising-steps=30 --guidance-scale 3.5 --hf-token=$HF_TOKEN --bf16 --download-onnx-models
|
||||
|
||||
# Stable Diffusion 3.5-large
|
||||
python3 demo_txt2img_sd35.py "a beautiful photograph of Mt. Fuji during cherry blossom" --version=3.5-large --denoising-steps=30 --guidance-scale 3.5 --hf-token=$HF_TOKEN --bf16 --download-onnx-models
|
||||
|
||||
# Stable Diffusion 3.5-large FP8
|
||||
python3 demo_txt2img_sd35.py "a beautiful photograph of Mt. Fuji during cherry blossom" --version=3.5-large --denoising-steps=30 --guidance-scale 3.5 --hf-token=$HF_TOKEN --fp8 --download-onnx-models --onnx-dir onnx_35_fp8/ --engine-dir engine_35_fp8/
|
||||
```
|
||||
|
||||
You can also specify an input image conditioning as shown below
|
||||
|
||||
```bash
|
||||
wget https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png -O dog-on-bench.png
|
||||
|
||||
# Stable Diffusion 3
|
||||
python3 demo_txt2img_sd3.py "dog wearing a sweater and a blue collar" --version sd3 --input-image dog-on-bench.png --hf-token=$HF_TOKEN
|
||||
```
|
||||
|
||||
Note that a denosing-percentage is applied to the number of denoising-steps when an input image conditioning is provided. Its default value is set to 0.6. This parameter can be updated using `--denoising-percentage`
|
||||
|
||||
### Generate an image with Stable Diffusion v3.5-large with ControlNet guided by an image and a text prompt
|
||||
|
||||
```bash
|
||||
# Depth BF16
|
||||
python3 demo_controlnet_sd35.py "a photo of a man" --controlnet-type depth --hf-token=$HF_TOKEN --denoising-steps 40 --guidance-scale 4.5 --bf16 --download-onnx-models --low-vram
|
||||
|
||||
# Depth FP8
|
||||
python3 demo_controlnet_sd35.py "a photo of a man" --version=3.5-large --fp8 --controlnet-type depth --download-onnx-models --denoising-steps=40 --guidance-scale 4.5 --hf-token=$HF_TOKEN --low-vram
|
||||
|
||||
# Canny BF16
|
||||
python3 demo_controlnet_sd35.py "A Night time photo taken by Leica M11, portrait of a Japanese woman in a kimono, looking at the camera, Cherry blossoms" --controlnet-type canny --hf-token=$HF_TOKEN --denoising-steps 60 --guidance-scale 3.5 --bf16 --download-onnx-models --low-vram
|
||||
|
||||
# Canny FP8
|
||||
python3 demo_controlnet_sd35.py "A Night time photo taken by Leica M11, portrait of a Japanese woman in a kimono, looking at the camera, Cherry blossoms" --version=3.5-large --fp8 --controlnet-type canny --hf-token=$HF_TOKEN --denoising-steps 60 --guidance-scale 3.5 --low-vram --download-onnx-models
|
||||
|
||||
# Blur
|
||||
python3 demo_controlnet_sd35.py "generated ai art, a tiny, lost rubber ducky in an action shot close-up, surfing the humongous waves, inside the tube, in the style of Kelly Slater" --controlnet-type blur --hf-token=$HF_TOKEN --denoising-steps 60 --guidance-scale 3.5 --bf16 --download-onnx-models --low-vram
|
||||
```
|
||||
|
||||
### Generate a video guided by an initial image using Stable Video Diffusion
|
||||
|
||||
Download the pre-exported ONNX model
|
||||
|
||||
```bash
|
||||
pip install -U "huggingface_hub[cli]"
|
||||
hf download stabilityai/stable-video-diffusion-img2vid-xt-1-1-tensorrt --local-dir onnx-svd-xt-1-1
|
||||
```
|
||||
|
||||
SVD-XT-1.1 (25 frames at resolution 576x1024)
|
||||
|
||||
```bash
|
||||
python3 demo_img2vid.py --version svd-xt-1.1 --onnx-dir onnx-svd-xt-1-1 --engine-dir engine-svd-xt-1-1 --hf-token=$HF_TOKEN
|
||||
```
|
||||
|
||||
Run the command below to generate a video in FP8.
|
||||
|
||||
```bash
|
||||
python3 demo_img2vid.py --version svd-xt-1.1 --onnx-dir onnx-svd-xt-1-1 --engine-dir engine-svd-xt-1-1 --hf-token=$HF_TOKEN --fp8
|
||||
```
|
||||
|
||||
> NOTE: There is a bug in HuggingFace, you can workaround with following this [PR](https://github.com/huggingface/diffusers/pull/6562/files)
|
||||
|
||||
```
|
||||
if torch.is_tensor(num_frames):
|
||||
num_frames = num_frames.item()
|
||||
emb = emb.repeat_interleave(num_frames, dim=0)
|
||||
```
|
||||
|
||||
You may also specify a custom conditioning image using `--input-image`:
|
||||
|
||||
```bash
|
||||
python3 demo_img2vid.py --version svd-xt-1.1 --onnx-dir onnx-svd-xt-1-1 --engine-dir engine-svd-xt-1-1 --input-image https://www.hdcarwallpapers.com/walls/2018_chevrolet_camaro_zl1_nascar_race_car_2-HD.jpg --hf-token=$HF_TOKEN
|
||||
```
|
||||
|
||||
NOTE: The min and max guidance scales are configured using --min-guidance-scale and --max-guidance-scale respectively.
|
||||
|
||||
### Generate an image guided by a text prompt using Stable Cascade
|
||||
|
||||
Run the below command to generate an image using Stable Cascade
|
||||
|
||||
```bash
|
||||
python3 demo_stable_cascade.py --onnx-opset=16 "Anthropomorphic cat dressed as a pilot" --onnx-dir onnx-sc --engine-dir engine-sc
|
||||
```
|
||||
|
||||
The lite versions of the models are also supported using the command below
|
||||
|
||||
```bash
|
||||
python3 demo_stable_cascade.py --onnx-opset=16 "Anthropomorphic cat dressed as a pilot" --onnx-dir onnx-sc-lite --engine-dir engine-sc-lite --lite
|
||||
```
|
||||
|
||||
> NOTE: The pipeline is only enabled for the BF16 model weights
|
||||
|
||||
> NOTE: The pipeline only supports ONNX export using Opset 16.
|
||||
|
||||
> NOTE: The denoising steps and guidance scale for the Prior and Decoder models are configured using --prior-denoising-steps, --prior-guidance-scale, --decoder-denoising-steps, and --decoder-guidance-scale respectively.
|
||||
|
||||
### Generating Images with Flux
|
||||
|
||||
> **NOTE:** Flux models require Flux family dependencies. Install with: `python3 setup.py flux`
|
||||
|
||||
#### 1. Generate an Image from a Text Prompt
|
||||
|
||||
##### Run Flux.1-Dev
|
||||
|
||||
NOTE: Pass `--download-onnx-models` to avoid native ONNX export and download the ONNX models from [Black Forest Labs' collection](https://huggingface.co/collections/black-forest-labs/flux1-onnx-679d06b7579583bd84c8ef83). It is only supported for BF16, FP8, and FP4 pipelines.
|
||||
|
||||
```bash
|
||||
# FP16 (requires >48GB VRAM for native export)
|
||||
python3 demo_txt2img_flux.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN
|
||||
|
||||
# BF16
|
||||
python3 demo_txt2img_flux.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --bf16 --download-onnx-models
|
||||
|
||||
# FP8
|
||||
python3 demo_txt2img_flux.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --quantization-level 4 --fp8 --download-onnx-models
|
||||
|
||||
# FP4
|
||||
python3 demo_txt2img_flux.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --fp4 --download-onnx-models
|
||||
```
|
||||
|
||||
##### Run Flux.1-Schnell
|
||||
|
||||
```bash
|
||||
# FP16 (requires >48GB VRAM for native export)
|
||||
python3 demo_txt2img_flux.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --version="flux.1-schnell"
|
||||
|
||||
# BF16
|
||||
python3 demo_txt2img_flux.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --version="flux.1-schnell" --bf16 --download-onnx-models
|
||||
|
||||
# FP8
|
||||
python3 demo_txt2img_flux.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --version="flux.1-schnell" --quantization-level 4 --fp8 --download-onnx-models
|
||||
|
||||
# FP4
|
||||
python3 demo_txt2img_flux.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --version="flux.1-schnell" --fp4 --download-onnx-models
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### 2. Generate an Image from an Initial Image + Text Prompt
|
||||
|
||||
Download an example input image:
|
||||
|
||||
```bash
|
||||
wget "https://miro.medium.com/v2/resize:fit:640/format:webp/1*iD8mUonHMgnlP0qrSx3qPg.png" -O yellow.png
|
||||
```
|
||||
|
||||
Run the image-to-image pipeline:
|
||||
|
||||
```bash
|
||||
python3 demo_img2img_flux.py "A home with 2 floors and windows. The front door is purple" --hf-token=$HF_TOKEN --input-image yellow.png --image-strength 0.95 --bf16 --onnx-dir onnx-flux-dev/bf16 --engine-dir engine-flux-dev/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### 3. Generate an Image Using Flux ControlNet
|
||||
|
||||
##### Download the Control Image
|
||||
|
||||
```bash
|
||||
wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/robot.png
|
||||
```
|
||||
|
||||
##### Calibration Data for native ONNX export (FP8 Pipeline)
|
||||
|
||||
FP8 ControlNet pipelines require downloading a calibration dataset and providing the path. You can use the datasets provided by Black Forest Labs here: [depth](https://drive.google.com/file/d/1DFfhOSrTlKfvBFLcD2vAALwwH4jSGdGk/view) | [canny](https://drive.google.com/file/d/1dRoxOL-vy3tSAesyqBSJoUWsbkMwv3en/view)
|
||||
|
||||
You can use the `--calibraton-dataset` flag to specify the path, which is set to `./{depth/canny}-eval/benchmark` by default if not provided. Note that the dataset should have `inputs/` and `prompts/` underneath the provided path, matching the format of the BFL dataset.
|
||||
|
||||
##### Depth ControlNet
|
||||
|
||||
```bash
|
||||
# BF16
|
||||
python3 demo_img2img_flux.py "A robot made of exotic candies and chocolates of different kinds. The background is filled with confetti and celebratory gifts." --version="flux.1-dev-depth" --hf-token=$HF_TOKEN --guidance-scale 10 --control-image robot.png --bf16 --denoising-steps 30 --download-onnx-models
|
||||
|
||||
# FP8 using pre-exported ONNX models
|
||||
python3 demo_img2img_flux.py "A robot made of exotic candies" --version="flux.1-dev-depth" --hf-token=$HF_TOKEN --guidance-scale 10 --control-image robot.png --fp8 --denoising-steps 30 --download-onnx-models --build-static-batch --quantization-level 4
|
||||
|
||||
# FP8 using native ONNX export
|
||||
rm -rf onnx/* engine/* && python3 demo_img2img_flux.py "A robot made of exotic candies" --version="flux.1-dev-depth" --hf-token=$HF_TOKEN --guidance-scale 10 --control-image robot.png --quantization-level 4 --fp8 --denoising-steps 30
|
||||
|
||||
# FP4
|
||||
python3 demo_img2img_flux.py "A robot made of exotic candies" --version="flux.1-dev-depth" --hf-token=$HF_TOKEN --guidance-scale 10 --control-image robot.png --fp4 --denoising-steps 30 --download-onnx-models --build-static-batch
|
||||
```
|
||||
|
||||
##### Canny ControlNet
|
||||
|
||||
```bash
|
||||
# BF16
|
||||
python3 demo_img2img_flux.py "a robot made out of gold" --version="flux.1-dev-canny" --hf-token=$HF_TOKEN --guidance-scale 30 --control-image robot.png --bf16 --denoising-steps 30 --download-onnx-models
|
||||
|
||||
# FP8 using pre-exported ONNX models
|
||||
python3 demo_img2img_flux.py "a robot made out of gold" --version="flux.1-dev-canny" --hf-token=$HF_TOKEN --guidance-scale 30 --control-image robot.png --fp8 --denoising-steps 30 --download-onnx-models --build-static-batch --quantization-level 4
|
||||
|
||||
# FP8 using native ONNX export
|
||||
rm -rf onnx/* engine/* && python3 demo_img2img_flux.py "a robot made out of gold" --version="flux.1-dev-canny" --hf-token=$HF_TOKEN --guidance-scale 30 --control-image robot.png --quantization-level 4 --fp8 --denoising-steps 30 --calibration-dataset {custom/dataset/path}
|
||||
|
||||
# FP4
|
||||
python3 demo_img2img_flux.py "a robot made out of gold" --version="flux.1-dev-canny" --hf-token=$HF_TOKEN --guidance-scale 30 --control-image robot.png --fp4 --denoising-steps 30 --download-onnx-models --build-static-batch
|
||||
```
|
||||
|
||||
#### 4. Generate an Image Using Flux LoRA
|
||||
|
||||
FLUX supports loading LoRA for Flux.1-Dev and Flux.1-Schnell. Make sure the target lora is compatible with the transformer model. Below is an example of using a [water color Flux LoRA](https://huggingface.co/SebastianBodza/flux_lora_aquarel_watercolor)
|
||||
|
||||
```bash
|
||||
# FP16
|
||||
python3 demo_txt2img_flux.py "A painting of a barista creating an intricate latte art design, with the 'Coffee Creations' logo skillfully formed within the latte foam. In a watercolor style, AQUACOLTOK. White background." --hf-token=$HF_TOKEN --lora-path "SebastianBodza/flux_lora_aquarel_watercolor" --lora-weight 1.0 --onnx-dir=onnx-flux-lora --engine-dir=engine-flux-lora
|
||||
|
||||
# FP8
|
||||
python3 demo_txt2img_flux.py "A painting of a barista creating an intricate latte art design, with the 'Coffee Creations' logo skillfully formed within the latte foam. In a watercolor style, AQUACOLTOK. White background." --hf-token=$HF_TOKEN --lora-path "SebastianBodza/flux_lora_aquarel_watercolor" --lora-weight 1.0 --onnx-dir=onnx-flux-lora --engine-dir=engine-flux-lora --fp8
|
||||
```
|
||||
|
||||
#### 5. Edit an Image using Flux Kontext
|
||||
|
||||
```bash
|
||||
wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png
|
||||
|
||||
# BF16
|
||||
python3 demo_img2img_flux.py "Add a hat to the cat" --version="flux.1-kontext-dev" --hf-token=$HF_TOKEN --guidance-scale 2.5 --kontext-image cat.png --denoising-steps 28 --bf16 --onnx-dir onnx-kontext --engine-dir engine-kontext --download-onnx-models
|
||||
|
||||
# FP8
|
||||
python3 demo_img2img_flux.py "Add a hat to the cat" --version="flux.1-kontext-dev" --hf-token=$HF_TOKEN --guidance-scale 2.5 --kontext-image cat.png --denoising-steps 28 --fp8 --onnx-dir onnx-kontext-fp8 --engine-dir engine-kontext-fp8 --download-onnx-models --quantization-level 4
|
||||
|
||||
# FP4
|
||||
python3 demo_img2img_flux.py "Add a hat to the cat" --version="flux.1-kontext-dev" --hf-token=$HF_TOKEN --guidance-scale 2.5 --kontext-image cat.png --denoising-steps 28 --fp4 --onnx-dir onnx-kontext-fp4 --engine-dir engine-kontext-fp4 --download-onnx-models
|
||||
```
|
||||
---
|
||||
|
||||
#### 5. Export ONNX Models Only (Skip Inference)
|
||||
|
||||
Use the `--onnx-export-only` flag to export ONNX models on a higher-VRAM device. The exported ONNX models can be used on a device with lower VRAM for the engine build and inference steps.
|
||||
|
||||
```bash
|
||||
python3 demo_txt2img_flux.py "a beautiful photograph of Mt. Fuji during cherry blossom" --hf-token=$HF_TOKEN --onnx-export-only
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### 6. Running Flux on GPUs with Limited Memory
|
||||
|
||||
##### Optimization Flags
|
||||
|
||||
- `--low-vram`: Enables model-offloading for reduced VRAM usage.
|
||||
- `--ws`: Enables weight streaming in TensorRT engines.
|
||||
- `--t5-ws-percentage` and `--transformer-ws-percentage`: Set runtime weight streaming budgets.
|
||||
- `--build-static-batch`: Build all engines using static batch sizes to lower the required activation memory. This will limit supported batch size of these engines for inference to the value specified by `--batch-size`.
|
||||
|
||||
##### FLUX VRAM Requirements Table
|
||||
|
||||
Memory usage captured below excludes the ONNX export step, and assumes use of the `--build-static-batch` flag to reduce activation VRAM usage. Users can either use [pre-exported ONNX models](README.md#download-pre-exported-models-recommended-for-48gb-vram) or export the models separately on a higher-VRAM device using [--onnx-export-only](README.md#4-export-onnx-models-only-skip-inference).
|
||||
|
||||
| Precision | Default VRAM Usage | With `--low-vram` |
|
||||
| --------- | ------------------ | ----------------- |
|
||||
| FP16 | 39.3 GB | 23.9 GB |
|
||||
| BF16 | 35.7 GB | 23.9 GB |
|
||||
| FP8 | 24.6 GB | 14.9 GB |
|
||||
| FP4 | 21.67 GB | 11.1 GB |
|
||||
|
||||
NOTE: The FP8 and FP4 Pipelines are supported on Hopper/Ada/Blackwell devices only. The FP4 pipeline is most performant on Blackwell devices.
|
||||
|
||||
|
||||
### Run Cosmos2 World Foundation Models
|
||||
|
||||
> **NOTE:** Cosmos models require Cosmos family dependencies. Install with: `python3 setup.py cosmos`
|
||||
|
||||
Select the prompts and export them as below
|
||||
|
||||
```bash
|
||||
export PROMPT="A close-up shot captures a vibrant yellow scrubber vigorously working on a grimy plate, its bristles moving in circular motions to lift stubborn grease and food residue. The dish, once covered in remnants of a hearty meal, gradually reveals its original glossy surface. Suds form and bubble around the scrubber, creating a satisfying visual of cleanliness in progress. The sound of scrubbing fills the air, accompanied by the gentle clinking of the dish against the sink. As the scrubber continues its task, the dish transforms, gleaming under the bright kitchen lights, symbolizing the triumph of cleanliness over mess."
|
||||
|
||||
export NEGATIVE_PROMPT="The video captures a series of frames showing ugly scenes, static with no motion, motion blur, over-saturation, shaky footage, low resolution, grainy texture, pixelated images, poorly lit areas, underexposed and overexposed scenes, poor color balance, washed out colors, choppy sequences, jerky movements, low frame rate, artifacting, color banding, unnatural transitions, outdated special effects, fake elements, unconvincing visuals, poorly edited content, jump cuts, visual noise, and flickering. Overall, the video is of poor quality."
|
||||
```
|
||||
|
||||
#### 1. Generate an Image from a Text Prompt
|
||||
|
||||
##### Run Cosmos-Predict2-2B-Text2Image
|
||||
|
||||
```bash
|
||||
# BF16
|
||||
python3 demo_txt2image_cosmos.py "$PROMPT" --negative-prompt="$NEGATIVE_PROMPT" --hf-token=$HF_TOKEN
|
||||
```
|
||||
|
||||
#### 2. Generate a Video guided by an Initial Video Conditioning and a Text Prompt
|
||||
|
||||
##### Run Cosmos-Predict2-2B-Video2World (only PyTorch backend enabled)
|
||||
|
||||
```bash
|
||||
# BF16
|
||||
python3 demo_vid2world_cosmos.py "$PROMPT" --negative-prompt="$NEGATIVE_PROMPT" --hf-token=$HF_TOKEN
|
||||
```
|
||||
|
||||
|
||||
### Specify Custom Paths for ONNX models and TensorRT engines (FLUX, Stable Diffusion 3.5 and Cosmos only)
|
||||
|
||||
Custom override paths to pre-exported ONNX model files can be provided using `--custom-onnx-paths`. These ONNX models are directly used to build TRT engines without further optimization on the ONNX graphs. Paths should be a comma-separated list of <model_name>:<path> pairs. For example: `--custom-onnx-paths=transformer:/path/to/transformer.onnx,vae:/path/to/vae.onnx`. Call <PipelineClass>.get_model_names(...) for the list of supported model names.
|
||||
|
||||
Custom override paths to pre-built engine files can be provided using `--custom-engine-paths`. Paths should be a comma-separated list of <model_name>:<path> pairs. For example: `--custom-onnx-paths=transformer:/path/to/transformer.plan,vae:/path/to/vae.plan`.
|
||||
|
||||
### Generate a video from a text prompt using Wan
|
||||
|
||||
Run the below command to generate 81 frames of video at 720×1280 resolution using Wan 2.2. Due to the high memory requirements of this model, it is recommended to enable `--low-vram` and use a Blackwell device.
|
||||
|
||||
```bash
|
||||
# Default (81 frames, 720x1280) with --low-vram enabled
|
||||
python3 demo_txt2vid_wan.py "A serene bamboo forest with sunlight filtering through the leaves" --hf-token=$HF_TOKEN --low-vram
|
||||
|
||||
# Adjust denoising steps, guidance scales, negative prompt, seed, warmup runs
|
||||
python3 demo_txt2vid_wan.py "Ocean waves crashing on a beach at sunset" --hf-token=$HF_TOKEN --low-vram --denoising-steps 50 --guidance-scale 4.5 --guidance-scale-2 3.5 --negative-prompt "blurry, low quality, static" --seed 42 --num-warmup-runs 0
|
||||
```
|
||||
|
||||
## Configuration options
|
||||
|
||||
- Noise scheduler can be set using `--scheduler <scheduler>`. Note: not all schedulers are available for every version.
|
||||
- To accelerate engine building time use `--timing-cache <path to cache file>`. The cache file will be created if it does not already exist. Note that performance may degrade if cache files are used across multiple GPU targets. It is recommended to use timing caches only during development. To achieve the best perfromance in deployment, please build engines without timing cache.
|
||||
- Specify new directories for storing onnx and engine files when switching between versions, LoRAs, ControlNets, etc. This can be done using `--onnx-dir <new onnx dir>` and `--engine-dir <new engine dir>`.
|
||||
- Inference performance can be improved by enabling [CUDA graphs](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#cuda-graphs) using `--use-cuda-graph`. Enabling CUDA graphs requires fixed input shapes, so this flag must be combined with `--build-static-batch` and cannot be combined with `--build-dynamic-shape`.
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 482 KiB |
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,161 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Configure dependencies before any external imports
|
||||
from demo_diffusion import deps
|
||||
deps.configure("sd")
|
||||
|
||||
import argparse
|
||||
|
||||
import controlnet_aux
|
||||
import torch
|
||||
from cuda.bindings import runtime as cudart
|
||||
from PIL import Image
|
||||
|
||||
from demo_diffusion import dd_argparse
|
||||
from demo_diffusion import image as image_module
|
||||
from demo_diffusion import pipeline as pipeline_module
|
||||
|
||||
|
||||
def parseArgs():
|
||||
parser = argparse.ArgumentParser(description="Options for Stable Diffusion ControlNet Demo", conflict_handler='resolve')
|
||||
parser = dd_argparse.add_arguments(parser)
|
||||
parser.add_argument('--scheduler', type=str, default="UniPC", choices=["DDIM", "DPM", "EulerA", "LMSD", "PNDM", "UniPC"], help="Scheduler for diffusion process")
|
||||
parser.add_argument('--input-image', nargs = '+', type=str, default=[], help="Path to the input image/images already prepared for ControlNet modality. For example: canny edged image for canny ControlNet, not just regular rgb image")
|
||||
parser.add_argument('--controlnet-type', nargs='+', type=str, default=["canny"], help="Controlnet type, can be `None`, `str` or `str` list from ['canny', 'depth', 'hed', 'mlsd', 'normal', 'openpose', 'scribble', 'seg']")
|
||||
parser.add_argument('--controlnet-scale', nargs='+', type=float, default=[1.0], help="The outputs of the controlnet are multiplied by `controlnet_scale` before they are added to the residual in the original unet, can be `None`, `float` or `float` list")
|
||||
return parser.parse_args()
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("[I] Initializing StableDiffusion controlnet demo using TensorRT")
|
||||
args = parseArgs()
|
||||
|
||||
# Controlnet configuration
|
||||
if not isinstance(args.controlnet_type, list):
|
||||
raise ValueError(f"`--controlnet-type` must be of type `str` or `str` list, but is {type(args.controlnet_type)}")
|
||||
|
||||
# Controlnet configuration
|
||||
if not isinstance(args.controlnet_scale, list):
|
||||
raise ValueError(f"`--controlnet-scale`` must be of type `float` or `float` list, but is {type(args.controlnet_scale)}")
|
||||
|
||||
# Check number of ControlNets to ControlNet scales
|
||||
if len(args.controlnet_type) != len(args.controlnet_scale):
|
||||
raise ValueError(f"Numbers of ControlNets {len(args.controlnet_type)} should be equal to number of ControlNet scales {len(args.controlnet_scale)}.")
|
||||
|
||||
# Convert controlnet scales to tensor
|
||||
controlnet_scale = torch.FloatTensor(args.controlnet_scale)
|
||||
|
||||
# Check images
|
||||
input_images = []
|
||||
if len(args.input_image) > 0:
|
||||
for image in args.input_image:
|
||||
input_images.append(Image.open(image))
|
||||
else:
|
||||
for controlnet in args.controlnet_type:
|
||||
if controlnet == "canny":
|
||||
if args.version == "xl-1.0":
|
||||
canny_image = image_module.download_image(
|
||||
"https://huggingface.co/diffusers/controlnet-canny-sdxl-1.0/resolve/main/out_bird.png"
|
||||
)
|
||||
# "out_bird.png" has 5 images combined in a row. We pick the first image which is the input image.
|
||||
canny_image = canny_image.crop((0, 0, canny_image.width / 5, canny_image.height))
|
||||
elif args.version == "1.5":
|
||||
canny_image = image_module.download_image(
|
||||
"https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png"
|
||||
)
|
||||
canny_image = controlnet_aux.CannyDetector()(canny_image)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"This demo supports ControlNets for v1.4 and SDXL base pipelines only. Version provided: {args.version}"
|
||||
)
|
||||
input_images.append(canny_image.resize((args.width, args.height)))
|
||||
elif controlnet == "normal":
|
||||
normal_image = image_module.download_image(
|
||||
"https://huggingface.co/lllyasviel/sd-controlnet-normal/resolve/main/images/toy.png"
|
||||
)
|
||||
normal_image = controlnet_aux.NormalBaeDetector.from_pretrained("lllyasviel/Annotators")(normal_image)
|
||||
input_images.append(normal_image.resize((args.width, args.height)))
|
||||
elif controlnet == "depth":
|
||||
depth_image = image_module.download_image(
|
||||
"https://huggingface.co/lllyasviel/sd-controlnet-depth/resolve/main/images/stormtrooper.png"
|
||||
)
|
||||
depth_image = controlnet_aux.LeresDetector.from_pretrained("lllyasviel/Annotators")(depth_image)
|
||||
input_images.append(depth_image.resize((args.width, args.height)))
|
||||
elif controlnet == "hed":
|
||||
hed_image = image_module.download_image(
|
||||
"https://huggingface.co/lllyasviel/sd-controlnet-hed/resolve/main/images/man.png"
|
||||
)
|
||||
hed_image = controlnet_aux.HEDdetector.from_pretrained("lllyasviel/Annotators")(hed_image)
|
||||
input_images.append(hed_image.resize((args.width, args.height)))
|
||||
elif controlnet == "mlsd":
|
||||
mlsd_image = image_module.download_image(
|
||||
"https://huggingface.co/lllyasviel/sd-controlnet-mlsd/resolve/main/images/room.png"
|
||||
)
|
||||
mlsd_image = controlnet_aux.MLSDdetector.from_pretrained("lllyasviel/Annotators")(mlsd_image)
|
||||
input_images.append(mlsd_image.resize((args.width, args.height)))
|
||||
elif controlnet == "openpose":
|
||||
openpose_image = image_module.download_image(
|
||||
"https://huggingface.co/lllyasviel/sd-controlnet-openpose/resolve/main/images/pose.png"
|
||||
)
|
||||
openpose_image = controlnet_aux.OpenposeDetector.from_pretrained("lllyasviel/Annotators")(openpose_image)
|
||||
input_images.append(openpose_image.resize((args.width, args.height)))
|
||||
elif controlnet == "scribble":
|
||||
scribble_image = image_module.download_image(
|
||||
"https://huggingface.co/lllyasviel/sd-controlnet-scribble/resolve/main/images/bag.png"
|
||||
)
|
||||
scribble_image = controlnet_aux.HEDdetector.from_pretrained("lllyasviel/Annotators")(scribble_image, scribble=True)
|
||||
input_images.append(scribble_image.resize((args.width, args.height)))
|
||||
elif controlnet == "seg":
|
||||
seg_image = image_module.download_image(
|
||||
"https://huggingface.co/lllyasviel/sd-controlnet-seg/resolve/main/images/house.png"
|
||||
)
|
||||
seg_image = controlnet_aux.SamDetector.from_pretrained("ybelkada/segment-anything", subfolder="checkpoints")(seg_image)
|
||||
input_images.append(seg_image.resize((args.width, args.height)))
|
||||
else:
|
||||
raise ValueError(f"You should implement the conditonal image of this controlnet: {controlnet}")
|
||||
assert len(input_images) > 0
|
||||
|
||||
kwargs_init_pipeline, kwargs_load_engine, args_run_demo = dd_argparse.process_pipeline_args(args)
|
||||
|
||||
# Initialize demo
|
||||
demo = pipeline_module.StableDiffusionPipeline(
|
||||
pipeline_type=(
|
||||
pipeline_module.PIPELINE_TYPE.CONTROLNET
|
||||
if args.version != "xl-1.0"
|
||||
else pipeline_module.PIPELINE_TYPE.XL_CONTROLNET
|
||||
),
|
||||
controlnets=args.controlnet_type,
|
||||
**kwargs_init_pipeline,
|
||||
)
|
||||
|
||||
# Load TensorRT engines and pytorch modules
|
||||
demo.loadEngines(
|
||||
args.engine_dir,
|
||||
args.framework_model_dir,
|
||||
args.onnx_dir,
|
||||
**kwargs_load_engine)
|
||||
|
||||
# Load resources
|
||||
_, shared_device_memory = cudart.cudaMalloc(demo.calculateMaxDeviceMemory())
|
||||
demo.activateEngines(shared_device_memory)
|
||||
demo.loadResources(args.height, args.width, args.batch_size, args.seed)
|
||||
|
||||
# Run inference
|
||||
demo_kwargs = {'input_image': input_images, 'controlnet_scales': controlnet_scale}
|
||||
demo.run(*args_run_demo, **demo_kwargs)
|
||||
|
||||
demo.teardown()
|
||||
@@ -0,0 +1,191 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Configure dependencies before any external imports
|
||||
from demo_diffusion import deps
|
||||
deps.configure("sd")
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
from cuda.bindings import runtime as cudart
|
||||
from PIL import Image
|
||||
|
||||
from demo_diffusion import dd_argparse
|
||||
from demo_diffusion import image as image_module
|
||||
from demo_diffusion import pipeline as pipeline_module
|
||||
|
||||
|
||||
def parseArgs():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Options for Stable Diffusion 3.5-large ControlNet Demo", conflict_handler="resolve"
|
||||
)
|
||||
parser = dd_argparse.add_arguments(parser)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
type=str,
|
||||
default="3.5-large",
|
||||
choices={"3.5-large"},
|
||||
help="Version of Stable Diffusion 3.5",
|
||||
)
|
||||
parser.add_argument("--height", type=int, default=1024, help="Height of image to generate (must be multiple of 8)")
|
||||
parser.add_argument("--width", type=int, default=1024, help="Height of image to generate (must be multiple of 8)")
|
||||
parser.add_argument(
|
||||
"--max-sequence-length",
|
||||
type=int,
|
||||
default=256,
|
||||
help="Maximum sequence length to use with the prompt.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--control-image",
|
||||
nargs="+",
|
||||
type=str,
|
||||
default=[],
|
||||
help="Path to the input image/images already prepared for ControlNet modality. For example: canny edged image for canny ControlNet, not just regular rgb image",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--controlnet-type",
|
||||
type=str,
|
||||
default="canny",
|
||||
help="Controlnet type (single type only), can be 'canny', 'depth', 'blur', etc.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--controlnet-scale",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="The outputs of the controlnet are multiplied by `controlnet_scale` before they are added to the residual in the original Transformer",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def process_demo_args(args):
|
||||
batch_size = args.batch_size
|
||||
prompt = args.prompt
|
||||
negative_prompt = args.negative_prompt
|
||||
# Process prompt
|
||||
if not isinstance(prompt, list):
|
||||
raise ValueError(f"`prompt` must be of type `str` list, but is {type(prompt)}")
|
||||
prompt = prompt * batch_size
|
||||
|
||||
if not isinstance(negative_prompt, list):
|
||||
raise ValueError(f"`--negative-prompt` must be of type `str` list, but is {type(negative_prompt)}")
|
||||
if len(negative_prompt) == 1:
|
||||
negative_prompt = negative_prompt * batch_size
|
||||
|
||||
if args.height % 8 != 0 or args.width % 8 != 0:
|
||||
raise ValueError(
|
||||
f"Image height and width have to be divisible by 8 but specified as: {args.image_height} and {args.width}."
|
||||
)
|
||||
|
||||
max_batch_size = 4
|
||||
if args.batch_size > max_batch_size:
|
||||
raise ValueError(f"Batch size {args.batch_size} is larger than allowed {max_batch_size}.")
|
||||
|
||||
if args.use_cuda_graph and (not args.build_static_batch or args.build_dynamic_shape):
|
||||
raise ValueError(
|
||||
"Using CUDA graph requires static dimensions. Enable `--build-static-batch` and do not specify `--build-dynamic-shape`"
|
||||
)
|
||||
|
||||
# Controlnet configuration
|
||||
if not isinstance(args.controlnet_type, str):
|
||||
raise ValueError(f"`--controlnet-type` must be of type `str`, but is {type(args.controlnet_type)}")
|
||||
|
||||
# Controlnet configuration
|
||||
if not isinstance(args.controlnet_scale, float):
|
||||
raise ValueError(f"`--controlnet-scale` must be of type `float`, but is {type(args.controlnet_scale)}")
|
||||
|
||||
# Convert controlnet scales to tensor
|
||||
controlnet_scale = torch.tensor(args.controlnet_scale)
|
||||
|
||||
# Check images
|
||||
input_images = []
|
||||
if len(args.control_image) > 0:
|
||||
for image in args.control_image:
|
||||
input_images.append(Image.open(image))
|
||||
else:
|
||||
if args.controlnet_type == "canny":
|
||||
canny_image = image_module.download_image(
|
||||
"https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/canny.png"
|
||||
)
|
||||
input_images.append(canny_image.resize((args.width, args.height)))
|
||||
elif args.controlnet_type == "depth":
|
||||
depth_image = image_module.download_image(
|
||||
"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/marigold/marigold_einstein_lcm_depth.png"
|
||||
)
|
||||
input_images.append(depth_image.resize((args.width, args.height)))
|
||||
elif args.controlnet_type == "blur":
|
||||
blur_image = image_module.download_image(
|
||||
"https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/blur.png"
|
||||
)
|
||||
input_images.append(blur_image.resize((args.width, args.height)))
|
||||
else:
|
||||
raise ValueError(f"You should implement the conditonal image of this controlnet: {args.controlnet_type}")
|
||||
assert len(input_images) > 0
|
||||
|
||||
kwargs_run_demo = {
|
||||
"prompt": prompt,
|
||||
"negative_prompt": negative_prompt,
|
||||
"height": args.height,
|
||||
"width": args.width,
|
||||
"control_image": input_images,
|
||||
"controlnet_scale": controlnet_scale,
|
||||
"batch_count": args.batch_count,
|
||||
"num_warmup_runs": args.num_warmup_runs,
|
||||
"use_cuda_graph": args.use_cuda_graph,
|
||||
}
|
||||
|
||||
return kwargs_run_demo
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("[I] Initializing StableDiffusion ControlNet demo using TensorRT")
|
||||
args = parseArgs()
|
||||
|
||||
# Initialize demo
|
||||
_, kwargs_load_engine, _ = dd_argparse.process_pipeline_args(args)
|
||||
kwargs_run_demo = process_demo_args(args)
|
||||
|
||||
# Initialize demo
|
||||
demo = pipeline_module.StableDiffusion35Pipeline.FromArgs(
|
||||
args,
|
||||
pipeline_type=pipeline_module.PIPELINE_TYPE.CONTROLNET,
|
||||
)
|
||||
|
||||
# Load TensorRT engines and pytorch modules
|
||||
demo.load_engines(
|
||||
framework_model_dir=args.framework_model_dir,
|
||||
**kwargs_load_engine,
|
||||
)
|
||||
|
||||
if demo.low_vram:
|
||||
demo.device_memory_sizes = demo.get_device_memory_sizes()
|
||||
else:
|
||||
_, shared_device_memory = cudart.cudaMalloc(demo.calculate_max_device_memory())
|
||||
demo.activate_engines(shared_device_memory)
|
||||
|
||||
# Load resources
|
||||
demo.load_resources(
|
||||
image_height=args.height,
|
||||
image_width=args.width,
|
||||
batch_size=args.batch_size,
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
# Run inference
|
||||
demo.run(**kwargs_run_demo)
|
||||
|
||||
demo.teardown()
|
||||
@@ -0,0 +1,465 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
# Define valid optimization levels for TensorRT engine build
|
||||
VALID_OPTIMIZATION_LEVELS = list(range(6))
|
||||
|
||||
|
||||
def parse_key_value_pairs(string: str) -> Dict[str, str]:
|
||||
"""Parse a string of comma-separated key-value pairs into a dictionary.
|
||||
|
||||
Args:
|
||||
string (str): A string of comma-separated key-value pairs.
|
||||
|
||||
Returns:
|
||||
Dict[str, str]: Parsed dictionary of key-value pairs.
|
||||
|
||||
Example:
|
||||
>>> parse_key_value_pairs("key1:value1,key2:value2")
|
||||
{"key1": "value1", "key2": "value2"}
|
||||
"""
|
||||
parsed = {}
|
||||
|
||||
for key_value_pair in string.split(","):
|
||||
if not key_value_pair:
|
||||
continue
|
||||
|
||||
key_value_pair = key_value_pair.split(":")
|
||||
if len(key_value_pair) != 2:
|
||||
raise argparse.ArgumentTypeError(f"Invalid key-value pair: {key_value_pair}. Must have length 2.")
|
||||
key, value = key_value_pair
|
||||
parsed[key] = value
|
||||
|
||||
return parsed
|
||||
|
||||
|
||||
def add_arguments(parser):
|
||||
# Stable Diffusion configuration
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
type=str,
|
||||
default="1.4",
|
||||
choices=(
|
||||
"1.4",
|
||||
"dreamshaper-7",
|
||||
"xl-1.0",
|
||||
"xl-turbo",
|
||||
"svd-xt-1.1",
|
||||
"sd3",
|
||||
"3.5-medium",
|
||||
"3.5-large",
|
||||
"cascade",
|
||||
"flux.1-dev",
|
||||
"flux.1-schnell",
|
||||
"flux.1-dev-canny",
|
||||
"flux.1-dev-depth",
|
||||
"flux.1-kontext-dev",
|
||||
"cosmos-predict2-2b-text2image",
|
||||
"cosmos-predict2-14b-text2image",
|
||||
"cosmos-predict2-2b-video2world",
|
||||
"cosmos-predict2-14b-video2world",
|
||||
),
|
||||
help="Version of Stable Diffusion",
|
||||
)
|
||||
parser.add_argument("prompt", nargs="*", help="Text prompt(s) to guide image generation")
|
||||
parser.add_argument(
|
||||
"--negative-prompt", nargs="*", default=[""], help="The negative prompt(s) to guide the image generation."
|
||||
)
|
||||
parser.add_argument("--batch-size", type=int, default=1, choices=[1, 2, 4], help="Batch size (repeat prompt)")
|
||||
parser.add_argument(
|
||||
"--batch-count", type=int, default=1, help="Number of images to generate in sequence, one at a time."
|
||||
)
|
||||
parser.add_argument("--height", type=int, default=512, help="Height of image to generate (must be multiple of 8)")
|
||||
parser.add_argument("--width", type=int, default=512, help="Height of image to generate (must be multiple of 8)")
|
||||
parser.add_argument("--denoising-steps", type=int, default=30, help="Number of denoising steps")
|
||||
parser.add_argument(
|
||||
"--scheduler",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=("DDIM", "DDPM", "EulerA", "Euler", "LCM", "LMSD", "PNDM", "UniPC", "DDPMWuerstchen", "FlowMatchEuler"),
|
||||
help="Scheduler for diffusion process",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--guidance-scale",
|
||||
type=float,
|
||||
default=7.5,
|
||||
help="Value of classifier-free guidance scale (must be greater than 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-scale",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Controls how much to influence the outputs with the LoRA parameters. (must between 0 and 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-weight",
|
||||
type=float,
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="The LoRA adapter(s) weights to use with the UNet. (must between 0 and 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-path",
|
||||
type=str,
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="Path to LoRA adaptor. Ex: 'latent-consistency/lcm-lora-sdv1-5'",
|
||||
)
|
||||
parser.add_argument("--bf16", action="store_true", help="Run pipeline in BFloat16 precision")
|
||||
|
||||
# ONNX export
|
||||
parser.add_argument(
|
||||
"--onnx-opset",
|
||||
type=int,
|
||||
default=19,
|
||||
choices=range(7, 24),
|
||||
help="Select ONNX opset version to target for exported models",
|
||||
)
|
||||
parser.add_argument("--onnx-dir", default="onnx", help="Output directory for ONNX export")
|
||||
parser.add_argument(
|
||||
"--custom-onnx-paths",
|
||||
type=parse_key_value_pairs,
|
||||
help=(
|
||||
"[FLUX, Stable Diffusion 3.5-large, Cosmos only] Custom override paths to pre-exported ONNX model files. These ONNX models are directly used to "
|
||||
"build TRT engines without further optimization on the ONNX graphs. Paths should be a comma-separated list "
|
||||
"of <model_name>:<path> pairs. For example: "
|
||||
"--custom-onnx-paths=transformer:/path/to/transformer.onnx,vae:/path/to/vae.onnx. Call "
|
||||
"<PipelineClass>.get_model_names(...) for the list of supported model names."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--onnx-export-only",
|
||||
action="store_true",
|
||||
help="If set, only performs the export of models to ONNX, skipping engine build and inference.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--download-onnx-models",
|
||||
action="store_true",
|
||||
help=("[FLUX and Stable Diffusion 3.5-large only] Download pre-exported ONNX models"),
|
||||
)
|
||||
|
||||
# Framework model ckpt
|
||||
parser.add_argument("--framework-model-dir", default="pytorch_model", help="Directory for HF saved models")
|
||||
|
||||
# TensorRT engine build
|
||||
parser.add_argument("--engine-dir", default="engine", help="Output directory for TensorRT engines")
|
||||
parser.add_argument(
|
||||
"--custom-engine-paths",
|
||||
type=parse_key_value_pairs,
|
||||
help=(
|
||||
"[FLUX only] Custom override paths to pre-built engine files. Paths should be a comma-separated list of "
|
||||
"<model_name>:<path> pairs. For example: "
|
||||
"--custom-onnx-paths=transformer:/path/to/transformer.plan,vae:/path/to/vae.plan. Call "
|
||||
"<PipelineClass>.get_model_names(...) for the list of supported model names."
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--optimization-level",
|
||||
type=int,
|
||||
default=None,
|
||||
help=f"Set the builder optimization level to build the engine with. A higher level allows TensorRT to spend more building time for more optimization options. Must be one of {VALID_OPTIMIZATION_LEVELS}.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-static-batch", action="store_true", help="Build TensorRT engines with fixed batch size."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-dynamic-shape", action="store_true", help="Build TensorRT engines with dynamic image shapes."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-enable-refit", action="store_true", help="Enable Refit option in TensorRT engines during build."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-all-tactics", action="store_true", help="Build TensorRT engines using all tactic sources."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timing-cache", default=None, type=str, help="Path to the precached timing measurements to accelerate build."
|
||||
)
|
||||
parser.add_argument("--ws", action="store_true", help="Build TensorRT engines with weight streaming enabled.")
|
||||
|
||||
# Quantization configuration.
|
||||
parser.add_argument("--int8", action="store_true", help="Apply int8 quantization.")
|
||||
parser.add_argument("--fp8", action="store_true", help="Apply fp8 quantization.")
|
||||
parser.add_argument("--fp4", action="store_true", help="Apply fp4 quantization.")
|
||||
parser.add_argument(
|
||||
"--quantization-level",
|
||||
type=float,
|
||||
default=0.0,
|
||||
choices=[0.0, 1.0, 2.0, 2.5, 3.0, 4.0],
|
||||
help="int8/fp8 quantization level, 1: CNN, 2: CNN + FFN, 2.5: CNN + FFN + QKV, 3: CNN + Almost all Linear (Including FFN, QKV, Proj and others), 4: CNN + Almost all Linear + fMHA, 0: Default to 2.5 for int8 and 4.0 for fp8.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quantization-percentile",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Control quantization scaling factors (amax) collecting range, where the minimum amax in range(n_steps * percentile) will be collected. Recommendation: 1.0.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quantization-alpha",
|
||||
type=float,
|
||||
default=0.8,
|
||||
help="The alpha parameter for SmoothQuant quantization used for linear layers. Recommendation: 0.8 for SDXL.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-size",
|
||||
type=int,
|
||||
default=32,
|
||||
help="The number of steps to use for calibrating the model for quantization. Recommendation: 32, 64, 128 for SDXL",
|
||||
)
|
||||
|
||||
# Inference
|
||||
parser.add_argument(
|
||||
"--num-warmup-runs", type=int, default=5, help="Number of warmup runs before benchmarking performance"
|
||||
)
|
||||
parser.add_argument("--use-cuda-graph", action="store_true", help="Enable cuda graph")
|
||||
parser.add_argument("--nvtx-profile", action="store_true", help="Enable NVTX markers for performance profiling")
|
||||
parser.add_argument(
|
||||
"--torch-inference",
|
||||
default="",
|
||||
help="Run inference with PyTorch (using specified compilation mode) instead of TensorRT.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--torch-fallback",
|
||||
default=None,
|
||||
type=str,
|
||||
help="[FLUX, SD3.5, and Wan] Comma separated list of models to be inferenced using PyTorch instead of TRT. For example --torch-fallback text_encoder,transformer,transformer_2. If --torch-inference set, this parameter will be ignored.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--low-vram",
|
||||
action="store_true",
|
||||
help="[FLUX, SD3.5, and Wan] Optimize for low VRAM usage, possibly at the expense of inference performance. Disabled by default.",
|
||||
)
|
||||
parser.add_argument("--seed", type=int, default=None, help="Seed for random generator to get consistent results")
|
||||
parser.add_argument("--output-dir", default="output", help="Output directory for logs and image artifacts")
|
||||
parser.add_argument("--hf-token", type=str, help="HuggingFace API access token for downloading model checkpoints")
|
||||
parser.add_argument("-v", "--verbose", action="store_true", help="Show verbose output")
|
||||
return parser
|
||||
|
||||
|
||||
def process_pipeline_args(args: argparse.Namespace) -> Tuple[Dict[str, Any], Dict[str, Any], Tuple]:
|
||||
"""Validate parsed arguments and process argument values.
|
||||
|
||||
Some argument values are resolved or overwritten during processing.
|
||||
|
||||
Args:
|
||||
args (argparse.Namespace): Parsed argument. This is modified in-place.
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: Keyword arguments for initializing a pipeline. This is only used in legacy pipelines that do not
|
||||
have factory methods `FromArgs` that construct the pipeline directly from the parsed argument.
|
||||
Dict[str, Any]: Keyword arguments for calling the `.load_engine` method of the pipeline.
|
||||
Tuple: Arguments for calling the `.run` method of the pipeline.
|
||||
"""
|
||||
|
||||
# GPU device info
|
||||
device_info = torch.cuda.get_device_properties(0)
|
||||
sm_version = device_info.major * 10 + device_info.minor
|
||||
|
||||
is_flux = args.version.startswith("flux")
|
||||
is_sd35 = args.version.startswith("3.5")
|
||||
is_wan = args.version.startswith("wan")
|
||||
is_cosmos = args.version.startswith("cosmos")
|
||||
|
||||
if args.height % 8 != 0 or args.width % 8 != 0:
|
||||
raise ValueError(
|
||||
f"Image height and width have to be divisible by 8 but specified as: {args.image_height} and {args.width}."
|
||||
)
|
||||
|
||||
# Handle batch size
|
||||
max_batch_size = 4
|
||||
if args.batch_size > max_batch_size:
|
||||
raise ValueError(f"Batch size {args.batch_size} is larger than allowed {max_batch_size}.")
|
||||
|
||||
if args.use_cuda_graph and (not args.build_static_batch or args.build_dynamic_shape):
|
||||
raise ValueError(
|
||||
"Using CUDA graph requires static dimensions. Enable `--build-static-batch` and do not specify `--build-dynamic-shape`"
|
||||
)
|
||||
|
||||
# TensorRT builder optimization level
|
||||
if args.optimization_level is None:
|
||||
# optimization level set to 3 for all Flux pipelines to reduce GPU memory usage
|
||||
if args.int8 or args.fp8 and not is_flux:
|
||||
args.optimization_level = 4
|
||||
else:
|
||||
args.optimization_level = 3
|
||||
|
||||
if args.optimization_level not in VALID_OPTIMIZATION_LEVELS:
|
||||
raise ValueError(
|
||||
f"Optimization level {args.optimization_level} not valid. Valid values are: {VALID_OPTIMIZATION_LEVELS}"
|
||||
)
|
||||
|
||||
# Quantized pipeline
|
||||
# int8 support
|
||||
if args.int8 and not any(args.version.startswith(prefix) for prefix in ("xl", "1.4")):
|
||||
raise ValueError("int8 quantization is only supported for SDXL and SD1.4 pipelines.")
|
||||
|
||||
# fp8 support validation
|
||||
if args.fp8:
|
||||
# Check version compatibility
|
||||
supported_versions = ("xl", "1.4", "3.5-large")
|
||||
if not (any(args.version.startswith(prefix) for prefix in supported_versions) or is_flux):
|
||||
raise ValueError(
|
||||
"fp8 quantization is only supported for SDXL, SD1.4, SD3.5-large and FLUX pipelines."
|
||||
)
|
||||
|
||||
# Check controlnet compatibility
|
||||
if getattr(args, "controlnet_type", None) is not None:
|
||||
if args.version not in ("xl-1.0", "3.5-large"):
|
||||
raise ValueError("fp8 controlnet quantization is only supported for SDXL and SD3.5-large.")
|
||||
if args.version == "3.5-large" and args.controlnet_type == "blur":
|
||||
raise ValueError("Blur controlnet type is not supported for SD3.5.")
|
||||
# Check for conflicting quantization
|
||||
if args.int8:
|
||||
raise ValueError("Cannot apply both int8 and fp8 quantization, please choose only one.")
|
||||
|
||||
# Check GPU compute capability
|
||||
if sm_version < 89:
|
||||
raise ValueError(
|
||||
f"Cannot apply FP8 quantization for GPU with compute capability {sm_version / 10.0}. A minimum compute capability of 8.9 is required."
|
||||
)
|
||||
|
||||
# Check SD3.5-large specific requirement
|
||||
if args.version == "3.5-large" and not args.download_onnx_models:
|
||||
raise ValueError(
|
||||
"Native FP8 quantization is not supported for SD3.5-large. Please pass --download-onnx-models."
|
||||
)
|
||||
|
||||
# TensorRT ModelOpt quantization level
|
||||
if args.quantization_level == 0.0:
|
||||
def override_quant_level(level: float, dtype_str: str):
|
||||
args.quantization_level = level
|
||||
print(f"[W] The default quantization level has been set to {level} for {dtype_str}.")
|
||||
|
||||
if args.fp8:
|
||||
# L4 fp8 fMHA on Hopper not yet enabled.
|
||||
if sm_version == 90 and is_flux:
|
||||
override_quant_level(3.0, "FP8")
|
||||
else:
|
||||
override_quant_level(3.0 if args.version == "1.4" else 4.0, "FP8")
|
||||
|
||||
elif args.int8:
|
||||
override_quant_level(3.0, "INT8")
|
||||
|
||||
if args.version.startswith("flux") and args.quantization_level == 3.0 and args.download_onnx_models:
|
||||
raise ValueError(
|
||||
"Transformer ONNX model for Quantization level 3 is not available for download. Please export the quantized Transformer model natively with the removal of --download-onnx-models."
|
||||
)
|
||||
if args.fp4:
|
||||
# FP4 precision is only supported for the Flux pipeline
|
||||
assert is_flux, "FP4 precision is only supported for the Flux pipeline"
|
||||
|
||||
# Handle LoRA
|
||||
# FLUX canny and depth official LoRAs are not supported because they modify the transformer architecture, conflicting with refit
|
||||
if args.lora_path and not any(args.version.startswith(prefix) for prefix in ("xl", "flux.1-dev", "flux.1-schnell")):
|
||||
raise ValueError("LoRA adapter support is only supported for SDXL, FLUX.1-dev and FLUX.1-schnell pipelines")
|
||||
|
||||
if args.lora_weight:
|
||||
for weight in (weight for weight in args.lora_weight if not 0 <= weight <= 1):
|
||||
raise ValueError(f"LoRA adapter weights must be between 0 and 1, provided {weight}")
|
||||
|
||||
if not 0 <= args.lora_scale <= 1:
|
||||
raise ValueError(f"LoRA scale value must be between 0 and 1, provided {args.lora_scale}")
|
||||
|
||||
# Force lora merge when fp8 or int8 is used with LoRA
|
||||
if args.build_enable_refit and args.lora_path and (args.int8 or args.fp8):
|
||||
raise ValueError(
|
||||
"Engine refit should not be enabled for quantized models with LoRA. ModelOpt recommends fusing the LoRA to the model before quantization. \
|
||||
See https://github.com/NVIDIA/TensorRT-Model-Optimizer/tree/main/examples/diffusers/quantization#lora"
|
||||
)
|
||||
|
||||
# Torch-fallback and Torch-inference
|
||||
if args.torch_fallback and not args.torch_inference:
|
||||
assert (
|
||||
is_flux or is_sd35 or is_wan or is_cosmos
|
||||
), "PyTorch Fallback is only supported for Flux, Stable Diffusion 3.5, Wan and Cosmos pipelines."
|
||||
args.torch_fallback = args.torch_fallback.split(",")
|
||||
|
||||
if args.torch_fallback and args.torch_inference:
|
||||
print(
|
||||
"[W] All models will run in PyTorch when --torch-inference is set. Parameter --torch-fallback will be ignored."
|
||||
)
|
||||
args.torch_fallback = None
|
||||
|
||||
# low-vram
|
||||
if args.low_vram:
|
||||
assert (
|
||||
is_flux or is_sd35 or is_wan or is_cosmos
|
||||
), "low-vram mode is only supported for Flux, Stable Diffusion 3.5, Wan and Cosmos pipelines."
|
||||
|
||||
# Disable SDXL LCM pipeline
|
||||
if args.version == "xl-1.0" and args.scheduler == "LCM":
|
||||
raise ValueError("SDXL pipeline does not support the LCM scheduler currently. Please use a different scheduler.")
|
||||
|
||||
# Pack arguments
|
||||
kwargs_init_pipeline = {
|
||||
"version": args.version,
|
||||
"max_batch_size": max_batch_size,
|
||||
"denoising_steps": args.denoising_steps,
|
||||
"scheduler": args.scheduler,
|
||||
"guidance_scale": args.guidance_scale,
|
||||
"output_dir": args.output_dir,
|
||||
"hf_token": args.hf_token,
|
||||
"verbose": args.verbose,
|
||||
"nvtx_profile": args.nvtx_profile,
|
||||
"use_cuda_graph": args.use_cuda_graph,
|
||||
"lora_scale": args.lora_scale,
|
||||
"lora_weight": args.lora_weight,
|
||||
"lora_path": args.lora_path,
|
||||
"framework_model_dir": args.framework_model_dir,
|
||||
"torch_inference": args.torch_inference,
|
||||
}
|
||||
|
||||
kwargs_load_engine = {
|
||||
"onnx_opset": args.onnx_opset,
|
||||
"opt_batch_size": args.batch_size,
|
||||
"opt_image_height": args.height,
|
||||
"opt_image_width": args.width,
|
||||
"optimization_level": args.optimization_level,
|
||||
"static_batch": args.build_static_batch,
|
||||
"static_shape": not args.build_dynamic_shape,
|
||||
"enable_all_tactics": args.build_all_tactics,
|
||||
"enable_refit": args.build_enable_refit,
|
||||
"timing_cache": args.timing_cache,
|
||||
"int8": args.int8,
|
||||
"fp8": args.fp8,
|
||||
"fp4": args.fp4,
|
||||
"quantization_level": args.quantization_level,
|
||||
"quantization_percentile": args.quantization_percentile,
|
||||
"quantization_alpha": args.quantization_alpha,
|
||||
"calibration_size": args.calibration_size,
|
||||
"onnx_export_only": args.onnx_export_only,
|
||||
"download_onnx_models": args.download_onnx_models,
|
||||
}
|
||||
|
||||
args_run_demo = (
|
||||
args.prompt,
|
||||
args.negative_prompt,
|
||||
args.height,
|
||||
args.width,
|
||||
args.batch_size,
|
||||
args.batch_count,
|
||||
args.num_warmup_runs,
|
||||
args.use_cuda_graph,
|
||||
)
|
||||
|
||||
return kwargs_init_pipeline, kwargs_load_engine, args_run_demo
|
||||
@@ -0,0 +1,270 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""Dependency management for TensorRT diffusion demos.
|
||||
|
||||
Adds the right group's site-packages to sys.path so imports work.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Marker file to indicate successful installation
|
||||
# Must match the marker file used by setup.py
|
||||
INSTALL_COMPLETE_MARKER = ".install_complete"
|
||||
|
||||
# Descriptions for user-facing messages
|
||||
GROUP_DESCRIPTIONS = {
|
||||
"sd": "SD family (SD 1.4, SDXL, SD3, SD3.5, SVD, Stable Cascade)",
|
||||
"flux": "Flux family (Black Forest Labs)",
|
||||
"cosmos": "Cosmos family (NVIDIA), Wan2.2 T2V",
|
||||
}
|
||||
|
||||
# Valid dependency groups
|
||||
VALID_GROUPS = list(GROUP_DESCRIPTIONS.keys())
|
||||
|
||||
__all__ = [
|
||||
"configure",
|
||||
"get_configured_groups",
|
||||
"print_status",
|
||||
]
|
||||
|
||||
|
||||
def _resolve_deps_root(deps_root: str | None) -> str:
|
||||
"""Resolve the dependency root path with precedence: arg > env > default."""
|
||||
if deps_root is not None:
|
||||
return deps_root
|
||||
return os.environ.get("TENSORRT_DIFFUSION_DEPS_ROOT", "/workspace/deps")
|
||||
|
||||
def _prepend_env_path(var: str, path: str, clean_root: str | None = None):
|
||||
"""Prepend `path` to an os.pathsep-separated env var so child processes
|
||||
(e.g. the `polygraphy` CLI) inherit the group's dependencies. If
|
||||
`clean_root` is given, drop existing entries under it first."""
|
||||
def _norm(p: str) -> str:
|
||||
return os.path.abspath(os.path.expanduser(p))
|
||||
|
||||
existing = [p for p in os.environ.get(var, "").split(os.pathsep) if p]
|
||||
if clean_root is not None:
|
||||
root_abs = _norm(clean_root).rstrip(os.sep) + os.sep
|
||||
existing = [p for p in existing if not _norm(p).startswith(root_abs)]
|
||||
existing = [p for p in existing if _norm(p) != _norm(path)]
|
||||
os.environ[var] = os.pathsep.join([path, *existing])
|
||||
|
||||
|
||||
def _clean_diffusion_paths(deps_root: str = "/workspace/deps"):
|
||||
"""Drop any existing paths under deps_root from sys.path."""
|
||||
# Filter out any paths that live under deps_root (robust to path forms)
|
||||
root_abs = os.path.abspath(os.path.expanduser(deps_root)).rstrip(os.sep) + os.sep
|
||||
sys.path[:] = [
|
||||
p for p in sys.path
|
||||
if not os.path.abspath(os.path.expanduser(p)).startswith(root_abs)
|
||||
]
|
||||
|
||||
|
||||
def configure(
|
||||
group: str,
|
||||
deps_root: str | None = None,
|
||||
verbose: bool = False,
|
||||
clean: bool = True,
|
||||
fallback: bool = False
|
||||
):
|
||||
"""
|
||||
Configure sys.path to use dependencies from the specified group.
|
||||
|
||||
This function should be called at the top of each demo script, before
|
||||
any other imports that depend on external packages.
|
||||
|
||||
Args:
|
||||
group: Dependency group name ("sd", "flux", or "cosmos")
|
||||
deps_root: Root directory where dependencies are installed.
|
||||
If None, uses TENSORRT_DIFFUSION_DEPS_ROOT environment variable,
|
||||
or defaults to "/workspace/deps"
|
||||
verbose: Print configuration info (default: False)
|
||||
clean: Remove other dependency group paths first (default: True)
|
||||
This prevents sys.path inflation in long-running processes.
|
||||
fallback: If True, gracefully handle missing dependencies instead of raising
|
||||
RuntimeError. Useful for test environments using requirements.txt.
|
||||
If False but USE_REQUIREMENTS=1 is set,
|
||||
fallback will be automatically enabled.
|
||||
|
||||
Example:
|
||||
from demo_diffusion import deps
|
||||
deps.configure("flux")
|
||||
|
||||
# Or with custom location
|
||||
deps.configure("flux", deps_root="/custom/path/deps")
|
||||
|
||||
# Or via environment variable
|
||||
# export TENSORRT_DIFFUSION_DEPS_ROOT=/custom/path/deps
|
||||
# deps.configure("flux")
|
||||
|
||||
# For test environments using requirements.txt (installed via setup.sh)
|
||||
# export USE_REQUIREMENTS=1
|
||||
# deps.configure("sd") # Will automatically use fallback mode
|
||||
"""
|
||||
if group not in VALID_GROUPS:
|
||||
raise ValueError(
|
||||
f"Invalid dependency group: '{group}'\n"
|
||||
f"Valid groups: {', '.join(sorted(VALID_GROUPS))}"
|
||||
)
|
||||
|
||||
# Auto-enable fallback mode if using requirements.txt installation
|
||||
if not fallback and os.environ.get("USE_REQUIREMENTS") == "1":
|
||||
fallback = True
|
||||
if verbose:
|
||||
print("Detected USE_REQUIREMENTS=1, enabling fallback mode")
|
||||
|
||||
# Resolve deps_root consistently across the module
|
||||
deps_root = _resolve_deps_root(deps_root)
|
||||
|
||||
# Clean old dependency paths first (prevents inflation)
|
||||
if clean:
|
||||
_clean_diffusion_paths(deps_root)
|
||||
|
||||
# Determine Python version
|
||||
python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
|
||||
|
||||
# Construct path to site-packages for this group
|
||||
deps_path = os.path.join(
|
||||
deps_root,
|
||||
group,
|
||||
"lib",
|
||||
f"python{python_version}",
|
||||
"site-packages"
|
||||
)
|
||||
|
||||
# Check if installation is complete
|
||||
group_dir = os.path.join(deps_root, group)
|
||||
marker_file = os.path.join(group_dir, INSTALL_COMPLETE_MARKER)
|
||||
|
||||
if os.path.exists(deps_path) and os.path.exists(marker_file):
|
||||
# Insert at the beginning to override any system packages
|
||||
sys.path.insert(0, deps_path)
|
||||
|
||||
# Mirror onto the environment so child processes (e.g. the polygraphy
|
||||
# CLI that engine builds shell out to) use these deps too. When clean,
|
||||
# drop other groups' entries under deps_root, matching sys.path above.
|
||||
clean_root = deps_root if clean else None
|
||||
_prepend_env_path("PYTHONPATH", deps_path, clean_root=clean_root)
|
||||
_prepend_env_path("PATH", os.path.join(group_dir, "bin"), clean_root=clean_root)
|
||||
|
||||
if verbose:
|
||||
description = GROUP_DESCRIPTIONS.get(group, group)
|
||||
print(f"Configured dependencies: {description}")
|
||||
print(f" Path: {deps_path}")
|
||||
else:
|
||||
# Dependencies not found or installation incomplete
|
||||
description = GROUP_DESCRIPTIONS.get(group, group)
|
||||
|
||||
# Check if it's an incomplete installation
|
||||
if os.path.exists(group_dir) and not os.path.exists(marker_file):
|
||||
error_msg = (
|
||||
f"Dependencies for '{group}' are incomplete!\n"
|
||||
f" Location: {group_dir}\n"
|
||||
f" Description: {description}\n"
|
||||
f" A previous installation failed or was interrupted.\n"
|
||||
f" To fix, run:\n"
|
||||
f" python setup.py {group}\n"
|
||||
f" (This will clean up and reinstall)"
|
||||
)
|
||||
if fallback:
|
||||
if verbose:
|
||||
print(f"Warning: {error_msg}")
|
||||
print("Continuing with fallback mode...")
|
||||
return
|
||||
else:
|
||||
raise RuntimeError(error_msg)
|
||||
else:
|
||||
# Not installed at all
|
||||
error_msg = (
|
||||
f"Dependencies for '{group}' not found!\n"
|
||||
f" Expected at: {deps_path}\n"
|
||||
f" Description: {description}\n"
|
||||
f" To install, run: python setup.py {group}\n"
|
||||
f" If you installed elsewhere, set TENSORRT_DIFFUSION_DEPS_ROOT or pass deps_root."
|
||||
)
|
||||
if fallback:
|
||||
if verbose:
|
||||
print(f"Warning: {error_msg}")
|
||||
print("Continuing with fallback mode (assuming requirements.txt setup)...")
|
||||
return
|
||||
else:
|
||||
raise RuntimeError(error_msg)
|
||||
|
||||
|
||||
def get_configured_groups(deps_root: str | None = None) -> list[str]:
|
||||
"""
|
||||
Get list of dependency groups that are currently installed.
|
||||
|
||||
A group is considered installed only if both:
|
||||
1. The group directory exists
|
||||
2. The .install_complete marker file exists in that directory
|
||||
|
||||
Args:
|
||||
deps_root: Root directory where dependencies are installed.
|
||||
If None, uses TENSORRT_DIFFUSION_DEPS_ROOT environment variable,
|
||||
or defaults to "/workspace/deps"
|
||||
|
||||
Returns:
|
||||
List of installed group names (e.g., ["sd", "flux"])
|
||||
"""
|
||||
# Resolve deps_root
|
||||
deps_root = _resolve_deps_root(deps_root)
|
||||
|
||||
installed = []
|
||||
for group in VALID_GROUPS:
|
||||
# Check if the group directory exists and has the completion marker
|
||||
group_dir = os.path.join(deps_root, group)
|
||||
marker_file = os.path.join(group_dir, INSTALL_COMPLETE_MARKER)
|
||||
|
||||
# Only consider it installed if marker file exists
|
||||
if os.path.exists(group_dir) and os.path.isdir(group_dir) and os.path.exists(marker_file):
|
||||
installed.append(group)
|
||||
return sorted(installed)
|
||||
|
||||
|
||||
def print_status(deps_root: str | None = None):
|
||||
"""
|
||||
Print the status of all dependency groups.
|
||||
|
||||
Args:
|
||||
deps_root: Root directory where dependencies are installed.
|
||||
If None, uses TENSORRT_DIFFUSION_DEPS_ROOT environment variable,
|
||||
or defaults to "/workspace/deps"
|
||||
"""
|
||||
# Resolve deps_root
|
||||
deps_root = _resolve_deps_root(deps_root)
|
||||
|
||||
print("Dependency Groups Status:")
|
||||
print("-" * 60)
|
||||
print(f"Location: {deps_root}")
|
||||
print("-" * 60)
|
||||
|
||||
installed = get_configured_groups(deps_root)
|
||||
|
||||
for group in sorted(VALID_GROUPS):
|
||||
description = GROUP_DESCRIPTIONS.get(group, group)
|
||||
status = "Installed" if group in installed else "Not installed"
|
||||
print(f" {group:8} - {status:15} - {description}")
|
||||
|
||||
print("-" * 60)
|
||||
|
||||
if not installed:
|
||||
print("No dependency groups installed.")
|
||||
print("Run: python setup.py all")
|
||||
else:
|
||||
print(f"{len(installed)} group(s) installed: {', '.join(installed)}")
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import warnings
|
||||
from importlib import import_module
|
||||
|
||||
|
||||
def import_from_diffusers(model_name, module_name):
|
||||
try:
|
||||
module = import_module(module_name)
|
||||
return getattr(module, model_name)
|
||||
except ImportError:
|
||||
warnings.warn(f"Failed to import {module_name}. The {model_name} model will not be available.", ImportWarning)
|
||||
except AttributeError:
|
||||
warnings.warn(f"The {model_name} model is not available in the installed version of diffusers.", ImportWarning)
|
||||
return None
|
||||
@@ -0,0 +1,326 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import gc
|
||||
import os
|
||||
import subprocess
|
||||
import warnings
|
||||
from collections import OrderedDict, defaultdict
|
||||
|
||||
import numpy as np
|
||||
import onnx
|
||||
import tensorrt as trt
|
||||
import torch
|
||||
from cuda.bindings import runtime as cudart
|
||||
from onnx import numpy_helper
|
||||
from polygraphy.backend.common import bytes_from_path
|
||||
from polygraphy.backend.trt import (
|
||||
engine_from_bytes,
|
||||
)
|
||||
|
||||
TRT_LOGGER = trt.Logger(trt.Logger.ERROR)
|
||||
|
||||
|
||||
# Map of TensorRT dtype -> torch dtype
|
||||
trt_to_torch_dtype_dict = {
|
||||
trt.DataType.BOOL: torch.bool,
|
||||
trt.DataType.UINT8: torch.uint8,
|
||||
trt.DataType.INT8: torch.int8,
|
||||
trt.DataType.INT32: torch.int32,
|
||||
trt.DataType.INT64: torch.int64,
|
||||
trt.DataType.HALF: torch.float16,
|
||||
trt.DataType.FLOAT: torch.float32,
|
||||
trt.DataType.BF16: torch.bfloat16,
|
||||
}
|
||||
|
||||
|
||||
def _CUASSERT(cuda_ret):
|
||||
err = cuda_ret[0]
|
||||
if err != cudart.cudaError_t.cudaSuccess:
|
||||
raise RuntimeError(
|
||||
f"CUDA ERROR: {err}, error code reference: https://nvidia.github.io/cuda-python/module/cudart.html#cuda.cudart.cudaError_t"
|
||||
)
|
||||
if len(cuda_ret) > 1:
|
||||
return cuda_ret[1]
|
||||
return None
|
||||
|
||||
|
||||
def get_refit_weights(state_dict, onnx_opt_path, weight_name_mapping, weight_shape_mapping):
|
||||
onnx_opt_dir = os.path.dirname(onnx_opt_path)
|
||||
onnx_opt_model = onnx.load(onnx_opt_path)
|
||||
# Create initializer data hashes
|
||||
initializer_hash_mapping = {}
|
||||
for initializer in onnx_opt_model.graph.initializer:
|
||||
initializer_data = numpy_helper.to_array(initializer, base_dir=onnx_opt_dir).astype(np.float16)
|
||||
initializer_hash = hash(initializer_data.data.tobytes())
|
||||
initializer_hash_mapping[initializer.name] = initializer_hash
|
||||
|
||||
refit_weights = OrderedDict()
|
||||
updated_weight_names = set() # save names of updated weights to refit only the required weights
|
||||
for wt_name, wt in state_dict.items():
|
||||
# query initializer to compare
|
||||
initializer_name = weight_name_mapping[wt_name]
|
||||
initializer_hash = initializer_hash_mapping[initializer_name]
|
||||
|
||||
# get shape transform info
|
||||
initializer_shape, is_transpose = weight_shape_mapping[wt_name]
|
||||
if is_transpose:
|
||||
wt = torch.transpose(wt, 0, 1)
|
||||
else:
|
||||
wt = torch.reshape(wt, initializer_shape)
|
||||
|
||||
# include weight if hashes differ
|
||||
wt_hash = hash(wt.cpu().detach().numpy().astype(np.float16).data.tobytes())
|
||||
if initializer_hash != wt_hash:
|
||||
updated_weight_names.add(initializer_name)
|
||||
# Store all weights as the refitter may require unchanged weights too
|
||||
# docs: https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#refitting-engine-c
|
||||
refit_weights[initializer_name] = wt.contiguous()
|
||||
return refit_weights, updated_weight_names
|
||||
|
||||
|
||||
class Engine:
|
||||
def __init__(
|
||||
self,
|
||||
engine_path,
|
||||
):
|
||||
self.engine_path = engine_path
|
||||
self.engine = None
|
||||
self.context = None
|
||||
self.buffers = OrderedDict()
|
||||
self.tensors = OrderedDict()
|
||||
self.cuda_graph_instance = None # cuda graph
|
||||
|
||||
def __del__(self):
|
||||
del self.engine
|
||||
del self.context
|
||||
del self.buffers
|
||||
del self.tensors
|
||||
|
||||
def refit(self, refit_weights, updated_weight_names):
|
||||
# Initialize refitter
|
||||
refitter = trt.Refitter(self.engine, TRT_LOGGER)
|
||||
refitted_weights = set()
|
||||
|
||||
def refit_single_weight(trt_weight_name):
|
||||
# get weight from state dict
|
||||
trt_datatype = refitter.get_weights_prototype(trt_weight_name).dtype
|
||||
refit_weights[trt_weight_name] = refit_weights[trt_weight_name].to(trt_to_torch_dtype_dict[trt_datatype])
|
||||
|
||||
# trt.Weight and trt.TensorLocation
|
||||
trt_wt_tensor = trt.Weights(
|
||||
trt_datatype, refit_weights[trt_weight_name].data_ptr(), torch.numel(refit_weights[trt_weight_name])
|
||||
)
|
||||
trt_wt_location = (
|
||||
trt.TensorLocation.DEVICE if refit_weights[trt_weight_name].is_cuda else trt.TensorLocation.HOST
|
||||
)
|
||||
|
||||
# apply refit
|
||||
refitter.set_named_weights(trt_weight_name, trt_wt_tensor, trt_wt_location)
|
||||
refitted_weights.add(trt_weight_name)
|
||||
|
||||
# iterate through all tensorrt refittable weights
|
||||
for trt_weight_name in refitter.get_all_weights():
|
||||
if trt_weight_name not in updated_weight_names:
|
||||
continue
|
||||
|
||||
refit_single_weight(trt_weight_name)
|
||||
|
||||
# iterate through missing weights required by tensorrt - addresses the case where lora_scale=0
|
||||
for trt_weight_name in refitter.get_missing_weights():
|
||||
refit_single_weight(trt_weight_name)
|
||||
|
||||
if not refitter.refit_cuda_engine():
|
||||
print("Error: failed to refit new weights.")
|
||||
exit(0)
|
||||
|
||||
print(f"[I] Total refitted weights {len(refitted_weights)}.")
|
||||
|
||||
def build(
|
||||
self,
|
||||
onnx_path,
|
||||
tf32=False,
|
||||
input_profile=None,
|
||||
enable_refit=False,
|
||||
enable_all_tactics=False,
|
||||
timing_cache=None,
|
||||
update_output_names=None,
|
||||
native_instancenorm=True,
|
||||
verbose=False,
|
||||
weight_streaming=False,
|
||||
builder_optimization_level=3,
|
||||
precision_constraints='none',
|
||||
):
|
||||
print(f"Building TensorRT engine for {onnx_path}: {self.engine_path}")
|
||||
|
||||
# Base command
|
||||
build_command = [f"polygraphy convert {onnx_path} --convert-to trt --output {self.engine_path}"]
|
||||
|
||||
# Build arguments
|
||||
build_args = [
|
||||
"--strongly-typed",
|
||||
"--tf32" if tf32 else "",
|
||||
"--weight-streaming" if weight_streaming else "",
|
||||
"--refittable" if enable_refit else "",
|
||||
"--tactic-sources" if not enable_all_tactics else "",
|
||||
"--onnx-flags native_instancenorm" if native_instancenorm else "",
|
||||
f"--builder-optimization-level {builder_optimization_level}",
|
||||
f"--precision-constraints {precision_constraints}",
|
||||
]
|
||||
|
||||
# Timing cache
|
||||
if timing_cache:
|
||||
build_args.extend([
|
||||
f"--load-timing-cache {timing_cache}",
|
||||
f"--save-timing-cache {timing_cache}"
|
||||
])
|
||||
|
||||
# Verbosity setting
|
||||
verbosity = "extra_verbose" if verbose else "error"
|
||||
build_args.append(f"--verbosity {verbosity}")
|
||||
|
||||
# Output names
|
||||
if update_output_names:
|
||||
print(f"Updating network outputs to {update_output_names}")
|
||||
build_args.append(f"--trt-outputs {' '.join(update_output_names)}")
|
||||
|
||||
# Input profiles
|
||||
if input_profile:
|
||||
profile_args = defaultdict(str)
|
||||
for name, dims in input_profile.items():
|
||||
assert len(dims) == 3
|
||||
profile_args["--trt-min-shapes"] += f"{name}:{str(list(dims[0])).replace(' ', '')} "
|
||||
profile_args["--trt-opt-shapes"] += f"{name}:{str(list(dims[1])).replace(' ', '')} "
|
||||
profile_args["--trt-max-shapes"] += f"{name}:{str(list(dims[2])).replace(' ', '')} "
|
||||
|
||||
build_args.extend(f"{k} {v}" for k, v in profile_args.items())
|
||||
|
||||
# Filter out empty strings and join command
|
||||
build_args = [arg for arg in build_args if arg]
|
||||
final_command = ' '.join(build_command + build_args)
|
||||
|
||||
# Execute command with improved error handling
|
||||
try:
|
||||
print(f"Engine build command: {final_command}")
|
||||
subprocess.run(final_command, check=True, shell=True)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
error_msg = (
|
||||
f"Failed to build TensorRT engine. Error details:\n"
|
||||
f"Command: {exc.cmd}\n"
|
||||
)
|
||||
raise RuntimeError(error_msg) from exc
|
||||
|
||||
def load(self, weight_streaming=False, weight_streaming_budget_percentage=None):
|
||||
if self.engine is not None:
|
||||
print(f"[W]: Engine {self.engine_path} already loaded, skip reloading")
|
||||
return
|
||||
if not hasattr(self, "engine_bytes_cpu") or self.engine_bytes_cpu is None:
|
||||
# keep a cpu copy of the engine to reduce reloading time.
|
||||
print(f"Loading TensorRT engine to cpu bytes: {self.engine_path}")
|
||||
self.engine_bytes_cpu = bytes_from_path(self.engine_path)
|
||||
print(f"Loading TensorRT engine from bytes: {self.engine_path}")
|
||||
self.engine = engine_from_bytes(self.engine_bytes_cpu)
|
||||
if weight_streaming:
|
||||
if weight_streaming_budget_percentage is None:
|
||||
warnings.warn(
|
||||
f"Weight streaming budget is not set for {self.engine_path}. Weights will not be streamed."
|
||||
)
|
||||
else:
|
||||
self.engine.weight_streaming_budget_v2 = int(
|
||||
weight_streaming_budget_percentage / 100 * self.engine.streamable_weights_size
|
||||
)
|
||||
|
||||
def unload(self, verbose=True):
|
||||
if self.engine is not None:
|
||||
if verbose:
|
||||
print(f"Unloading TensorRT engine: {self.engine_path}")
|
||||
del self.engine
|
||||
self.engine = None
|
||||
gc.collect()
|
||||
else:
|
||||
if verbose:
|
||||
print(f"[W]: Unload an unloaded engine {self.engine_path}, skip unloading")
|
||||
|
||||
def activate(self, device_memory=None):
|
||||
if device_memory is not None:
|
||||
self.context = self.engine.create_execution_context(
|
||||
trt.ExecutionContextAllocationStrategy.USER_MANAGED
|
||||
)
|
||||
self.context.device_memory = device_memory
|
||||
else:
|
||||
self.context = self.engine.create_execution_context()
|
||||
|
||||
def reactivate(self, device_memory):
|
||||
assert self.context
|
||||
self.context.device_memory = device_memory
|
||||
|
||||
def deactivate(self):
|
||||
del self.context
|
||||
self.context = None
|
||||
|
||||
def allocate_buffers(self, shape_dict=None, device="cuda"):
|
||||
for binding in range(self.engine.num_io_tensors):
|
||||
name = self.engine.get_tensor_name(binding)
|
||||
if shape_dict and name in shape_dict:
|
||||
shape = shape_dict[name]
|
||||
else:
|
||||
shape = self.engine.get_tensor_shape(name)
|
||||
print(
|
||||
f"[W]: {self.engine_path}: Could not find '{name}' in shape dict {shape_dict}. Using shape {shape} inferred from the engine."
|
||||
)
|
||||
if self.engine.get_tensor_mode(name) == trt.TensorIOMode.INPUT:
|
||||
self.context.set_input_shape(name, shape)
|
||||
dtype = trt_to_torch_dtype_dict[self.engine.get_tensor_dtype(name)]
|
||||
tensor = torch.empty(tuple(shape), dtype=dtype).to(device=device)
|
||||
self.tensors[name] = tensor
|
||||
|
||||
def deallocate_buffers(self):
|
||||
if not self.engine:
|
||||
return
|
||||
for idx in range(self.engine.num_io_tensors):
|
||||
binding = self.engine[idx]
|
||||
del self.tensors[binding]
|
||||
|
||||
def infer(self, feed_dict, stream, use_cuda_graph=False):
|
||||
for name, buf in feed_dict.items():
|
||||
self.tensors[name].copy_(buf)
|
||||
|
||||
for name, tensor in self.tensors.items():
|
||||
self.context.set_tensor_address(name, tensor.data_ptr())
|
||||
|
||||
if use_cuda_graph:
|
||||
if self.cuda_graph_instance is not None:
|
||||
_CUASSERT(cudart.cudaGraphLaunch(self.cuda_graph_instance, stream))
|
||||
_CUASSERT(cudart.cudaStreamSynchronize(stream))
|
||||
else:
|
||||
# do inference before CUDA graph capture
|
||||
noerror = self.context.execute_async_v3(stream)
|
||||
if not noerror:
|
||||
raise ValueError(f"ERROR: inference of {self.engine_path} failed.")
|
||||
# capture cuda graph
|
||||
_CUASSERT(
|
||||
cudart.cudaStreamBeginCapture(stream, cudart.cudaStreamCaptureMode.cudaStreamCaptureModeGlobal)
|
||||
)
|
||||
self.context.execute_async_v3(stream)
|
||||
self.graph = _CUASSERT(cudart.cudaStreamEndCapture(stream))
|
||||
self.cuda_graph_instance = _CUASSERT(cudart.cudaGraphInstantiate(self.graph, 0))
|
||||
else:
|
||||
noerror = self.context.execute_async_v3(stream)
|
||||
if not noerror:
|
||||
raise ValueError(f"ERROR: inference of {self.engine_path} failed.")
|
||||
|
||||
return self.tensors
|
||||
@@ -0,0 +1,34 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from demo_diffusion.image.load import (
|
||||
download_image,
|
||||
prepare_mask_and_masked_image,
|
||||
preprocess_image,
|
||||
save_image,
|
||||
)
|
||||
from demo_diffusion.image.resize import resize_with_antialiasing
|
||||
from demo_diffusion.image.video import tensor2vid
|
||||
|
||||
__all__ = [
|
||||
"preprocess_image",
|
||||
"prepare_mask_and_masked_image",
|
||||
"download_image",
|
||||
"save_image",
|
||||
"resize_with_antialiasing",
|
||||
"tensor2vid",
|
||||
]
|
||||
@@ -0,0 +1,78 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import os
|
||||
import random
|
||||
from io import BytesIO
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def preprocess_image(image):
|
||||
"""
|
||||
image: torch.Tensor
|
||||
"""
|
||||
w, h = image.size
|
||||
w, h = map(lambda x: x - x % 32, (w, h)) # resize to integer multiple of 32
|
||||
image = image.resize((w, h))
|
||||
image = np.array(image).astype(np.float32) / 255.0
|
||||
image = image[None].transpose(0, 3, 1, 2)
|
||||
image = torch.from_numpy(image).contiguous()
|
||||
return 2.0 * image - 1.0
|
||||
|
||||
|
||||
def prepare_mask_and_masked_image(image, mask):
|
||||
"""
|
||||
image: PIL.Image.Image
|
||||
mask: PIL.Image.Image
|
||||
"""
|
||||
if isinstance(image, Image.Image):
|
||||
image = np.array(image.convert("RGB"))
|
||||
image = image[None].transpose(0, 3, 1, 2)
|
||||
image = torch.from_numpy(image).to(dtype=torch.float32).contiguous() / 127.5 - 1.0
|
||||
if isinstance(mask, Image.Image):
|
||||
mask = np.array(mask.convert("L"))
|
||||
mask = mask.astype(np.float32) / 255.0
|
||||
mask = mask[None, None]
|
||||
mask[mask < 0.5] = 0
|
||||
mask[mask >= 0.5] = 1
|
||||
mask = torch.from_numpy(mask).to(dtype=torch.float32).contiguous()
|
||||
|
||||
masked_image = image * (mask < 0.5)
|
||||
|
||||
return mask, masked_image
|
||||
|
||||
|
||||
def download_image(url):
|
||||
response = requests.get(url)
|
||||
return Image.open(BytesIO(response.content)).convert("RGB")
|
||||
|
||||
|
||||
def save_image(images, image_path_dir, image_name_prefix, image_name_suffix):
|
||||
"""
|
||||
Save the generated images to png files.
|
||||
"""
|
||||
for i in range(images.shape[0]):
|
||||
image_path = os.path.join(
|
||||
image_path_dir,
|
||||
f"{image_name_prefix}{i + 1}-{random.randint(1000, 9999)}-{image_name_suffix}.png",
|
||||
)
|
||||
print(f"Saving image {i+1} / {images.shape[0]} to: {image_path}")
|
||||
Image.fromarray(images[i]).save(image_path)
|
||||
@@ -0,0 +1,123 @@
|
||||
# Copyright 2024 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
# Taken from https://github.com/huggingface/diffusers/blob/be62c85cd973f2001ab8c5d8919a9a6811fc7e43/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py#L633
|
||||
def resize_with_antialiasing(input, size, interpolation="bicubic", align_corners=True):
|
||||
h, w = input.shape[-2:]
|
||||
factors = (h / size[0], w / size[1])
|
||||
|
||||
# First, we have to determine sigma
|
||||
# Taken from skimage: https://github.com/scikit-image/scikit-image/blob/v0.19.2/skimage/transform/_warps.py#L171
|
||||
sigmas = (
|
||||
max((factors[0] - 1.0) / 2.0, 0.001),
|
||||
max((factors[1] - 1.0) / 2.0, 0.001),
|
||||
)
|
||||
|
||||
# Now kernel size. Good results are for 3 sigma, but that is kind of slow. Pillow uses 1 sigma
|
||||
# https://github.com/python-pillow/Pillow/blob/master/src/libImaging/Resample.c#L206
|
||||
# But they do it in the 2 passes, which gives better results. Let's try 2 sigmas for now
|
||||
ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3))
|
||||
|
||||
# Make sure it is odd
|
||||
if (ks[0] % 2) == 0:
|
||||
ks = ks[0] + 1, ks[1]
|
||||
|
||||
if (ks[1] % 2) == 0:
|
||||
ks = ks[0], ks[1] + 1
|
||||
|
||||
input = _gaussian_blur2d(input, ks, sigmas)
|
||||
|
||||
output = torch.nn.functional.interpolate(input, size=size, mode=interpolation, align_corners=align_corners)
|
||||
return output
|
||||
|
||||
|
||||
def _compute_padding(kernel_size):
|
||||
"""Compute padding tuple."""
|
||||
# 4 or 6 ints: (padding_left, padding_right,padding_top,padding_bottom)
|
||||
# https://pytorch.org/docs/stable/nn.html#torch.nn.functional.pad
|
||||
if len(kernel_size) < 2:
|
||||
raise AssertionError(kernel_size)
|
||||
computed = [k - 1 for k in kernel_size]
|
||||
|
||||
# for even kernels we need to do asymmetric padding :(
|
||||
out_padding = 2 * len(kernel_size) * [0]
|
||||
|
||||
for i in range(len(kernel_size)):
|
||||
computed_tmp = computed[-(i + 1)]
|
||||
|
||||
pad_front = computed_tmp // 2
|
||||
pad_rear = computed_tmp - pad_front
|
||||
|
||||
out_padding[2 * i + 0] = pad_front
|
||||
out_padding[2 * i + 1] = pad_rear
|
||||
|
||||
return out_padding
|
||||
|
||||
|
||||
def _filter2d(input, kernel):
|
||||
# prepare kernel
|
||||
b, c, h, w = input.shape
|
||||
tmp_kernel = kernel[:, None, ...].to(device=input.device, dtype=input.dtype)
|
||||
|
||||
tmp_kernel = tmp_kernel.expand(-1, c, -1, -1)
|
||||
|
||||
height, width = tmp_kernel.shape[-2:]
|
||||
|
||||
padding_shape: list[int] = _compute_padding([height, width])
|
||||
input = torch.nn.functional.pad(input, padding_shape, mode="reflect")
|
||||
|
||||
# kernel and input tensor reshape to align element-wise or batch-wise params
|
||||
tmp_kernel = tmp_kernel.reshape(-1, 1, height, width)
|
||||
input = input.view(-1, tmp_kernel.size(0), input.size(-2), input.size(-1))
|
||||
|
||||
# convolve the tensor with the kernel.
|
||||
output = torch.nn.functional.conv2d(input, tmp_kernel, groups=tmp_kernel.size(0), padding=0, stride=1)
|
||||
|
||||
out = output.view(b, c, h, w)
|
||||
return out
|
||||
|
||||
|
||||
def _gaussian(window_size: int, sigma):
|
||||
if isinstance(sigma, float):
|
||||
sigma = torch.tensor([[sigma]])
|
||||
|
||||
batch_size = sigma.shape[0]
|
||||
|
||||
x = (torch.arange(window_size, device=sigma.device, dtype=sigma.dtype) - window_size // 2).expand(batch_size, -1)
|
||||
|
||||
if window_size % 2 == 0:
|
||||
x = x + 0.5
|
||||
|
||||
gauss = torch.exp(-x.pow(2.0) / (2 * sigma.pow(2.0)))
|
||||
|
||||
return gauss / gauss.sum(-1, keepdim=True)
|
||||
|
||||
|
||||
def _gaussian_blur2d(input, kernel_size, sigma):
|
||||
if isinstance(sigma, tuple):
|
||||
sigma = torch.tensor([sigma], dtype=input.dtype)
|
||||
else:
|
||||
sigma = sigma.to(dtype=input.dtype)
|
||||
|
||||
ky, kx = int(kernel_size[0]), int(kernel_size[1])
|
||||
bs = sigma.shape[0]
|
||||
kernel_x = _gaussian(kx, sigma[:, 1].view(bs, 1))
|
||||
kernel_y = _gaussian(ky, sigma[:, 0].view(bs, 1))
|
||||
out_x = _filter2d(input, kernel_x[..., None, :])
|
||||
out = _filter2d(out_x, kernel_y[..., None])
|
||||
|
||||
return out
|
||||
@@ -0,0 +1,37 @@
|
||||
#
|
||||
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
# Not a contribution
|
||||
# Changes made by NVIDIA CORPORATION & AFFILIATES enabling tensor2vid or otherwise documented as
|
||||
# NVIDIA-proprietary are not a contribution and subject to the terms and conditions at the top of the file
|
||||
def tensor2vid(video: torch.Tensor, processor, output_type="np"):
|
||||
# Based on:
|
||||
# https://github.com/modelscope/modelscope/blob/1509fdb973e5871f37148a4b5e5964cafd43e64d/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py#L78
|
||||
|
||||
batch_size, channels, num_frames, height, width = video.shape
|
||||
outputs = []
|
||||
for batch_idx in range(batch_size):
|
||||
batch_vid = video[batch_idx].permute(1, 0, 2, 3)
|
||||
batch_output = processor.postprocess(batch_vid, output_type)
|
||||
|
||||
outputs.append(batch_output)
|
||||
|
||||
return outputs
|
||||
@@ -0,0 +1,111 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from demo_diffusion.model.base_model import BaseModel
|
||||
from demo_diffusion.model.clip import (
|
||||
CLIPImageProcessorModel,
|
||||
CLIPModel,
|
||||
CLIPVisionWithProjModel,
|
||||
CLIPWithProjModel,
|
||||
SD3_CLIPGModel,
|
||||
SD3_CLIPLModel,
|
||||
SD3_T5XXLModel,
|
||||
get_clip_embedding_dim,
|
||||
)
|
||||
from demo_diffusion.model.controlnet import SD3ControlNet
|
||||
from demo_diffusion.model.diffusion_transformer import (
|
||||
CosmosTransformerModel,
|
||||
FluxTransformerModel,
|
||||
SD3_MMDiTModel,
|
||||
SD3TransformerModel,
|
||||
WanTransformerModel,
|
||||
)
|
||||
from demo_diffusion.model.gan import VQGANModel
|
||||
from demo_diffusion.model.load import unload_torch_model
|
||||
from demo_diffusion.model.lora import FLUXLoraLoader, SDLoraLoader, merge_loras
|
||||
from demo_diffusion.model.scheduler import make_scheduler
|
||||
from demo_diffusion.model.t5 import T5Model
|
||||
from demo_diffusion.model.tokenizer import make_tokenizer
|
||||
from demo_diffusion.model.unet import (
|
||||
UNet2DConditionControlNetModel,
|
||||
UNetCascadeModel,
|
||||
UNetModel,
|
||||
UNetTemporalModel,
|
||||
UNetXLModel,
|
||||
UNetXLModelControlNet,
|
||||
)
|
||||
from demo_diffusion.model.vae import (
|
||||
AutoencoderKLWanEncoderModel,
|
||||
AutoencoderKLWanModel,
|
||||
SD3_VAEDecoderModel,
|
||||
SD3_VAEEncoderModel,
|
||||
TorchVAEEncoder,
|
||||
VAEDecTemporalModel,
|
||||
VAEEncoderModel,
|
||||
VAEModel,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# base_model
|
||||
"BaseModel",
|
||||
# clip
|
||||
"get_clip_embedding_dim",
|
||||
"CLIPModel",
|
||||
"CLIPWithProjModel",
|
||||
"SD3_CLIPGModel",
|
||||
"SD3_CLIPLModel",
|
||||
"SD3_T5XXLModel",
|
||||
"CLIPVisionWithProjModel",
|
||||
"CLIPImageProcessorModel",
|
||||
"CosmosTransformerModel",
|
||||
# diffusion_transformer
|
||||
"SD3_MMDiTModel",
|
||||
"FluxTransformerModel",
|
||||
"SD3TransformerModel",
|
||||
"SD3ControlNet",
|
||||
"WanTransformerModel",
|
||||
# gan
|
||||
"VQGANModel",
|
||||
# lora
|
||||
"SDLoraLoader",
|
||||
"FLUXLoraLoader",
|
||||
"merge_loras",
|
||||
# scheduler
|
||||
"make_scheduler",
|
||||
# t5
|
||||
"T5Model",
|
||||
# tokenizer
|
||||
"make_tokenizer",
|
||||
# unet
|
||||
"UNetModel",
|
||||
"UNetXLModel",
|
||||
"UNetXLModelControlNet",
|
||||
"UNet2DConditionControlNetModel",
|
||||
"UNetTemporalModel",
|
||||
"UNetCascadeModel",
|
||||
# vae
|
||||
"VAEModel",
|
||||
"SD3_VAEDecoderModel",
|
||||
"VAEDecTemporalModel",
|
||||
"TorchVAEEncoder",
|
||||
"VAEEncoderModel",
|
||||
"SD3_VAEEncoderModel",
|
||||
"AutoencoderKLWanModel",
|
||||
"AutoencoderKLWanEncoderModel",
|
||||
# load
|
||||
"unload_torch_model",
|
||||
]
|
||||
@@ -0,0 +1,320 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import onnx
|
||||
import torch
|
||||
from diffusers import DiffusionPipeline
|
||||
from onnx import numpy_helper
|
||||
|
||||
from demo_diffusion.model import load, optimizer
|
||||
from demo_diffusion.model.lora import merge_loras
|
||||
|
||||
|
||||
class BaseModel:
|
||||
def __init__(
|
||||
self,
|
||||
version="1.4",
|
||||
pipeline=None,
|
||||
device="cuda",
|
||||
hf_token="",
|
||||
verbose=True,
|
||||
framework_model_dir="pytorch_model",
|
||||
fp16=False,
|
||||
tf32=False,
|
||||
bf16=False,
|
||||
int8=False,
|
||||
fp8=False,
|
||||
fp4=False,
|
||||
max_batch_size=16,
|
||||
text_maxlen=77,
|
||||
embedding_dim=768,
|
||||
compression_factor=8,
|
||||
):
|
||||
|
||||
self.name = self.__class__.__name__
|
||||
self.pipeline_type = pipeline
|
||||
self.pipeline = pipeline.name
|
||||
self.version = version
|
||||
self.path = load.get_path(version, pipeline)
|
||||
self.device = device
|
||||
self.hf_token = hf_token
|
||||
self.hf_safetensor = True
|
||||
self.verbose = verbose
|
||||
self.framework_model_dir = framework_model_dir
|
||||
|
||||
self.fp16 = fp16
|
||||
self.tf32 = tf32
|
||||
self.bf16 = bf16
|
||||
self.int8 = int8
|
||||
self.fp8 = fp8
|
||||
self.fp4 = fp4
|
||||
|
||||
self.compression_factor = compression_factor
|
||||
self.min_batch = 1
|
||||
self.max_batch = max_batch_size
|
||||
self.min_image_shape = 256 # min image resolution: 256x256
|
||||
self.max_image_shape = 1360 # max image resolution: 1360x1360
|
||||
self.min_latent_shape = self.min_image_shape // self.compression_factor
|
||||
self.max_latent_shape = self.max_image_shape // self.compression_factor
|
||||
|
||||
self.text_maxlen = text_maxlen
|
||||
self.embedding_dim = embedding_dim
|
||||
self.extra_output_names = []
|
||||
|
||||
self.do_constant_folding = True
|
||||
|
||||
def get_pipeline(self):
|
||||
model_opts = {"variant": "fp16", "torch_dtype": torch.float16} if self.fp16 else {}
|
||||
model_opts = {"torch_dtype": torch.bfloat16} if self.bf16 else model_opts
|
||||
return DiffusionPipeline.from_pretrained(
|
||||
self.path,
|
||||
use_safetensors=self.hf_safetensor,
|
||||
token=self.hf_token,
|
||||
**model_opts,
|
||||
).to(self.device)
|
||||
|
||||
def get_model(self, torch_inference=""):
|
||||
pass
|
||||
|
||||
def get_input_names(self):
|
||||
pass
|
||||
|
||||
def get_output_names(self):
|
||||
pass
|
||||
|
||||
def get_dynamic_axes(self):
|
||||
return None
|
||||
|
||||
def get_sample_input(self, batch_size, image_height, image_width, static_shape):
|
||||
pass
|
||||
|
||||
def get_input_profile(self, batch_size, image_height, image_width, static_batch, static_shape):
|
||||
return None
|
||||
|
||||
def get_shape_dict(self, batch_size, image_height, image_width):
|
||||
return None
|
||||
|
||||
# Helper utility for ONNX export
|
||||
def export_onnx(
|
||||
self,
|
||||
onnx_path,
|
||||
onnx_opt_path,
|
||||
onnx_opset,
|
||||
opt_image_height,
|
||||
opt_image_width,
|
||||
opt_num_frames=None,
|
||||
custom_model=None,
|
||||
enable_lora_merge=False,
|
||||
static_shape=False,
|
||||
lora_loader=None,
|
||||
dynamo=False,
|
||||
):
|
||||
onnx_opt_graph = None
|
||||
# Export optimized ONNX model (if missing)
|
||||
if not os.path.exists(onnx_opt_path):
|
||||
if not os.path.exists(onnx_path):
|
||||
print(f"[I] Exporting ONNX model: {onnx_path}")
|
||||
|
||||
def export_onnx(model):
|
||||
if enable_lora_merge:
|
||||
assert lora_loader is not None
|
||||
model = merge_loras(model, lora_loader)
|
||||
|
||||
export_kwargs = {}
|
||||
if dynamo:
|
||||
export_kwargs["dynamic_shapes"] = self.get_dynamic_axes()
|
||||
else:
|
||||
export_kwargs["dynamic_axes"] = self.get_dynamic_axes()
|
||||
|
||||
inputs = self.get_sample_input(
|
||||
1, opt_image_height, opt_image_width, static_shape,
|
||||
**({'num_frames': opt_num_frames} if opt_num_frames else {})
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
torch.onnx.export(
|
||||
model,
|
||||
inputs,
|
||||
onnx_path,
|
||||
export_params=True,
|
||||
do_constant_folding=self.do_constant_folding,
|
||||
input_names=self.get_input_names(),
|
||||
output_names=self.get_output_names(),
|
||||
verbose=False,
|
||||
dynamo=dynamo,
|
||||
opset_version=onnx_opset,
|
||||
**export_kwargs,
|
||||
)
|
||||
|
||||
if custom_model:
|
||||
with torch.inference_mode():
|
||||
export_onnx(custom_model)
|
||||
else:
|
||||
# WAR: Enable autocast for BF16 Stable Cascade pipeline
|
||||
do_autocast = True if self.version == "cascade" and self.bf16 else False
|
||||
model = self.get_model()
|
||||
with torch.inference_mode(), torch.autocast("cuda", enabled=do_autocast):
|
||||
export_onnx(model)
|
||||
del model
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
else:
|
||||
print(f"[I] Found cached ONNX model: {onnx_path}")
|
||||
|
||||
print(f"[I] Optimizing ONNX model: {onnx_opt_path}")
|
||||
onnx_opt_graph = self.optimize(onnx.load(onnx_path))
|
||||
if load.onnx_graph_needs_external_data(onnx_opt_graph):
|
||||
onnx.save_model(
|
||||
onnx_opt_graph,
|
||||
onnx_opt_path,
|
||||
save_as_external_data=True,
|
||||
all_tensors_to_one_file=True,
|
||||
convert_attribute=False,
|
||||
)
|
||||
else:
|
||||
onnx.save(onnx_opt_graph, onnx_opt_path)
|
||||
else:
|
||||
print(f"[I] Found cached optimized ONNX model: {onnx_opt_path} ")
|
||||
|
||||
# Helper utility for weights map
|
||||
def export_weights_map(self, onnx_opt_path, weights_map_path):
|
||||
if not os.path.exists(weights_map_path):
|
||||
onnx_opt_dir = os.path.dirname(onnx_opt_path)
|
||||
onnx_opt_model = onnx.load(onnx_opt_path)
|
||||
state_dict = self.get_model().state_dict()
|
||||
# Create initializer data hashes
|
||||
initializer_hash_mapping = {}
|
||||
for initializer in onnx_opt_model.graph.initializer:
|
||||
initializer_data = numpy_helper.to_array(initializer, base_dir=onnx_opt_dir).astype(np.float16)
|
||||
initializer_hash = hash(initializer_data.data.tobytes())
|
||||
initializer_hash_mapping[initializer.name] = (initializer_hash, initializer_data.shape)
|
||||
|
||||
weights_name_mapping = {}
|
||||
weights_shape_mapping = {}
|
||||
# set to keep track of initializers already added to the name_mapping dict
|
||||
initializers_mapped = set()
|
||||
for wt_name, wt in state_dict.items():
|
||||
# get weight hash
|
||||
wt = wt.cpu().detach().numpy().astype(np.float16)
|
||||
wt_hash = hash(wt.data.tobytes())
|
||||
wt_t_hash = hash(np.transpose(wt).data.tobytes())
|
||||
|
||||
for initializer_name, (initializer_hash, initializer_shape) in initializer_hash_mapping.items():
|
||||
# Due to constant folding, some weights are transposed during export
|
||||
# To account for the transpose op, we compare the initializer hash to the
|
||||
# hash for the weight and its transpose
|
||||
if wt_hash == initializer_hash or wt_t_hash == initializer_hash:
|
||||
# The assert below ensures there is a 1:1 mapping between
|
||||
# PyTorch and ONNX weight names. It can be removed in cases where 1:many
|
||||
# mapping is found and name_mapping[wt_name] = list()
|
||||
assert initializer_name not in initializers_mapped
|
||||
weights_name_mapping[wt_name] = initializer_name
|
||||
initializers_mapped.add(initializer_name)
|
||||
is_transpose = False if wt_hash == initializer_hash else True
|
||||
weights_shape_mapping[wt_name] = (initializer_shape, is_transpose)
|
||||
|
||||
# Sanity check: Were any weights not matched
|
||||
if wt_name not in weights_name_mapping:
|
||||
print(f"[I] PyTorch weight {wt_name} not matched with any ONNX initializer")
|
||||
print(f"[I] {len(weights_name_mapping.keys())} PyTorch weights were matched with ONNX initializers")
|
||||
assert weights_name_mapping.keys() == weights_shape_mapping.keys()
|
||||
with open(weights_map_path, "w") as fp:
|
||||
json.dump([weights_name_mapping, weights_shape_mapping], fp)
|
||||
else:
|
||||
print(f"[I] Found cached weights map: {weights_map_path} ")
|
||||
|
||||
def optimize(self, onnx_graph, return_onnx=True, **kwargs):
|
||||
opt = optimizer.Optimizer(onnx_graph, verbose=self.verbose, version=self.version)
|
||||
opt.info(self.name + ": original")
|
||||
opt.cleanup()
|
||||
opt.info(self.name + ": cleanup")
|
||||
if kwargs.get("modify_fp8_graph", False):
|
||||
is_fp16_io = kwargs.get("is_fp16_io", True)
|
||||
opt.modify_fp8_graph(is_fp16_io=is_fp16_io)
|
||||
opt.info(self.name + ": modify fp8 graph")
|
||||
elif self.bf16:
|
||||
# Cast Resize I/O for strongly-typed TRT builds: BF16 -> FP32 inputs, FP32 -> BF16 outputs.
|
||||
# TRT does not support BF16 for the Resize operator.
|
||||
opt.infer_shapes()
|
||||
opt.cast_resize_io(output_dtype=onnx.TensorProto.BFLOAT16)
|
||||
opt.info(self.name + ": cast resize I/O for bf16")
|
||||
if self.version.startswith("flux.1") and self.fp8:
|
||||
opt.flux_convert_rope_weight_type()
|
||||
opt.info(self.name + ": convert rope weight type for fp8 flux")
|
||||
opt.fold_constants()
|
||||
opt.info(self.name + ": fold constants")
|
||||
opt.infer_shapes()
|
||||
opt.info(self.name + ": shape inference")
|
||||
if kwargs.get("modify_int8_graph", False):
|
||||
opt.modify_int8_graph()
|
||||
opt.info(self.name + ": modify int8 graph")
|
||||
onnx_opt_graph = opt.cleanup(return_onnx=return_onnx)
|
||||
opt.info(self.name + ": finished")
|
||||
return onnx_opt_graph
|
||||
|
||||
def check_dims(self, batch_size, image_height, image_width, num_frames=None):
|
||||
assert batch_size >= self.min_batch and batch_size <= self.max_batch
|
||||
latent_height = image_height // self.compression_factor
|
||||
latent_width = image_width // self.compression_factor
|
||||
assert latent_height >= self.min_latent_shape and latent_height <= self.max_latent_shape
|
||||
assert latent_width >= self.min_latent_shape and latent_width <= self.max_latent_shape
|
||||
|
||||
if num_frames:
|
||||
latent_frames = (self.num_frames - 1) // self.temporal_compression_factor + 1
|
||||
return (latent_height, latent_width, latent_frames)
|
||||
|
||||
return (latent_height, latent_width)
|
||||
|
||||
def get_minmax_dims(self, batch_size, image_height, image_width, static_batch, static_shape, num_frames=None):
|
||||
min_batch = batch_size if static_batch else self.min_batch
|
||||
max_batch = batch_size if static_batch else self.max_batch
|
||||
latent_height = image_height // self.compression_factor
|
||||
latent_width = image_width // self.compression_factor
|
||||
min_image_height = image_height if static_shape else self.min_image_shape
|
||||
max_image_height = image_height if static_shape else self.max_image_shape
|
||||
min_image_width = image_width if static_shape else self.min_image_shape
|
||||
max_image_width = image_width if static_shape else self.max_image_shape
|
||||
min_latent_height = latent_height if static_shape else self.min_latent_shape
|
||||
max_latent_height = latent_height if static_shape else self.max_latent_shape
|
||||
min_latent_width = latent_width if static_shape else self.min_latent_shape
|
||||
max_latent_width = latent_width if static_shape else self.max_latent_shape
|
||||
|
||||
frame_dims = ()
|
||||
if num_frames:
|
||||
latent_frames = (num_frames - 1) // self.temporal_compression_factor + 1
|
||||
min_latent_frames = latent_frames if static_shape else self.min_latent_frames
|
||||
max_latent_frames = latent_frames if static_shape else self.max_latent_frames
|
||||
frame_dims = (min_latent_frames, max_latent_frames)
|
||||
|
||||
return (
|
||||
min_batch,
|
||||
max_batch,
|
||||
min_image_height,
|
||||
max_image_height,
|
||||
min_image_width,
|
||||
max_image_width,
|
||||
min_latent_height,
|
||||
max_latent_height,
|
||||
min_latent_width,
|
||||
max_latent_width,
|
||||
*frame_dims
|
||||
)
|
||||
@@ -0,0 +1,555 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
from huggingface_hub import hf_hub_download
|
||||
from safetensors import safe_open
|
||||
from transformers import (
|
||||
CLIPImageProcessor,
|
||||
CLIPTextModel,
|
||||
CLIPTextModelWithProjection,
|
||||
CLIPVisionModelWithProjection,
|
||||
)
|
||||
|
||||
from demo_diffusion.model import base_model, load, optimizer
|
||||
from demo_diffusion.utils_sd3.other_impls import (
|
||||
SDClipModel,
|
||||
SDXLClipG,
|
||||
T5XXLModel,
|
||||
load_into,
|
||||
)
|
||||
|
||||
|
||||
def get_clipwithproj_embedding_dim(version: str, subfolder: str) -> int:
|
||||
"""Return the embedding dimension of a CLIP with projection model."""
|
||||
if version in ("xl-1.0", "xl-turbo", "cascade"):
|
||||
return 1280
|
||||
elif version in {"3.5-medium", "3.5-large"} and subfolder == "text_encoder":
|
||||
return 768
|
||||
elif version in {"3.5-medium", "3.5-large"} and subfolder == "text_encoder_2":
|
||||
return 1280
|
||||
else:
|
||||
raise ValueError(f"Invalid version {version} + subfolder {subfolder}")
|
||||
|
||||
|
||||
def get_clip_embedding_dim(version, pipeline):
|
||||
if version in (
|
||||
"1.4",
|
||||
"dreamshaper-7",
|
||||
"flux.1-dev",
|
||||
"flux.1-schnell",
|
||||
"flux.1-dev-canny",
|
||||
"flux.1-dev-depth",
|
||||
"flux.1-kontext-dev",
|
||||
):
|
||||
return 768
|
||||
elif version in ("xl-1.0", "xl-turbo") and pipeline.is_sd_xl_base():
|
||||
return 768
|
||||
elif version in ("sd3"):
|
||||
return 4096
|
||||
else:
|
||||
raise ValueError(f"Invalid version {version} + pipeline {pipeline}")
|
||||
|
||||
|
||||
class CLIPModel(base_model.BaseModel):
|
||||
def __init__(
|
||||
self,
|
||||
version,
|
||||
pipeline,
|
||||
device,
|
||||
hf_token,
|
||||
verbose,
|
||||
framework_model_dir,
|
||||
max_batch_size,
|
||||
embedding_dim,
|
||||
fp16=False,
|
||||
tf32=False,
|
||||
bf16=False,
|
||||
output_hidden_states=False,
|
||||
keep_pooled_output=False,
|
||||
subfolder="text_encoder",
|
||||
):
|
||||
super(CLIPModel, self).__init__(
|
||||
version,
|
||||
pipeline,
|
||||
device=device,
|
||||
hf_token=hf_token,
|
||||
verbose=verbose,
|
||||
framework_model_dir=framework_model_dir,
|
||||
fp16=fp16,
|
||||
tf32=tf32,
|
||||
bf16=bf16,
|
||||
max_batch_size=max_batch_size,
|
||||
embedding_dim=embedding_dim,
|
||||
)
|
||||
self.subfolder = subfolder
|
||||
self.hidden_layer_offset = 0 if pipeline.is_cascade() else -1
|
||||
self.keep_pooled_output = keep_pooled_output
|
||||
|
||||
# Output the final hidden state
|
||||
if output_hidden_states:
|
||||
self.extra_output_names = ["hidden_states"]
|
||||
|
||||
def get_model(self, torch_inference=""):
|
||||
model_opts = (
|
||||
{"torch_dtype": torch.float16} if self.fp16 else {"torch_dtype": torch.bfloat16} if self.bf16 else {}
|
||||
)
|
||||
clip_model_dir = load.get_checkpoint_dir(self.framework_model_dir, self.version, self.pipeline, self.subfolder)
|
||||
if not load.is_model_cached(clip_model_dir, model_opts, self.hf_safetensor, model_name="model"):
|
||||
model = CLIPTextModel.from_pretrained(
|
||||
self.path,
|
||||
subfolder=self.subfolder,
|
||||
use_safetensors=self.hf_safetensor,
|
||||
token=self.hf_token,
|
||||
attn_implementation="eager",
|
||||
**model_opts,
|
||||
).to(self.device)
|
||||
model.save_pretrained(clip_model_dir, **model_opts)
|
||||
else:
|
||||
print(f"[I] Load CLIPTextModel model from: {clip_model_dir}")
|
||||
model = CLIPTextModel.from_pretrained(clip_model_dir, **model_opts).to(self.device)
|
||||
model = optimizer.optimize_checkpoint(model, torch_inference)
|
||||
return model
|
||||
|
||||
def get_input_names(self):
|
||||
return ["input_ids"]
|
||||
|
||||
def get_output_names(self):
|
||||
output_names = ["text_embeddings"]
|
||||
if self.keep_pooled_output:
|
||||
output_names += ["pooled_embeddings"]
|
||||
return output_names
|
||||
|
||||
def get_dynamic_axes(self):
|
||||
dynamic_axes = {
|
||||
"input_ids": {0: "B"},
|
||||
"text_embeddings": {0: "B"},
|
||||
}
|
||||
if self.keep_pooled_output:
|
||||
dynamic_axes["pooled_embeddings"] = {0: "B"}
|
||||
return dynamic_axes
|
||||
|
||||
def get_input_profile(self, batch_size, image_height, image_width, static_batch, static_shape):
|
||||
self.check_dims(batch_size, image_height, image_width)
|
||||
min_batch, max_batch, _, _, _, _, _, _, _, _ = self.get_minmax_dims(
|
||||
batch_size, image_height, image_width, static_batch, static_shape
|
||||
)
|
||||
return {
|
||||
"input_ids": [(min_batch, self.text_maxlen), (batch_size, self.text_maxlen), (max_batch, self.text_maxlen)]
|
||||
}
|
||||
|
||||
def get_shape_dict(self, batch_size, image_height, image_width):
|
||||
self.check_dims(batch_size, image_height, image_width)
|
||||
output = {
|
||||
"input_ids": (batch_size, self.text_maxlen),
|
||||
"text_embeddings": (batch_size, self.text_maxlen, self.embedding_dim),
|
||||
}
|
||||
if self.keep_pooled_output:
|
||||
output["pooled_embeddings"] = (batch_size, self.embedding_dim)
|
||||
if "hidden_states" in self.extra_output_names:
|
||||
output["hidden_states"] = (batch_size, self.text_maxlen, self.embedding_dim)
|
||||
return output
|
||||
|
||||
def get_sample_input(self, batch_size, image_height, image_width, static_shape):
|
||||
self.check_dims(batch_size, image_height, image_width)
|
||||
return torch.zeros(batch_size, self.text_maxlen, dtype=torch.int32, device=self.device)
|
||||
|
||||
def optimize(self, onnx_graph):
|
||||
opt = optimizer.Optimizer(onnx_graph, verbose=self.verbose, version=self.version)
|
||||
opt.info(self.name + ": original")
|
||||
keep_outputs = [0, 1] if self.keep_pooled_output else [0]
|
||||
opt.select_outputs(keep_outputs)
|
||||
opt.cleanup()
|
||||
opt.fold_constants()
|
||||
opt.info(self.name + ": fold constants")
|
||||
opt.infer_shapes()
|
||||
opt.info(self.name + ": shape inference")
|
||||
opt.select_outputs(keep_outputs, names=self.get_output_names()) # rename network outputs
|
||||
opt.info(self.name + ": rename network output(s)")
|
||||
opt_onnx_graph = opt.cleanup(return_onnx=True)
|
||||
if "hidden_states" in self.extra_output_names:
|
||||
opt_onnx_graph = opt.clip_add_hidden_states(self.hidden_layer_offset, return_onnx=True)
|
||||
opt.info(self.name + ": added hidden_states")
|
||||
opt.info(self.name + ": finished")
|
||||
return opt_onnx_graph
|
||||
|
||||
class CLIPWithProjModel(CLIPModel):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
version,
|
||||
pipeline,
|
||||
device,
|
||||
hf_token,
|
||||
verbose,
|
||||
framework_model_dir,
|
||||
fp16=False,
|
||||
bf16=False,
|
||||
max_batch_size=16,
|
||||
output_hidden_states=False,
|
||||
subfolder="text_encoder_2",
|
||||
):
|
||||
|
||||
super(CLIPWithProjModel, self).__init__(
|
||||
version,
|
||||
pipeline,
|
||||
device=device,
|
||||
hf_token=hf_token,
|
||||
verbose=verbose,
|
||||
framework_model_dir=framework_model_dir,
|
||||
fp16=fp16,
|
||||
bf16=bf16,
|
||||
max_batch_size=max_batch_size,
|
||||
embedding_dim=get_clipwithproj_embedding_dim(version, subfolder),
|
||||
output_hidden_states=output_hidden_states,
|
||||
)
|
||||
self.subfolder = subfolder
|
||||
|
||||
def get_model(self, torch_inference=""):
|
||||
model_opts = {"variant": "fp16", "torch_dtype": torch.float16} if self.fp16 else {"torch_dtype": torch.bfloat16}
|
||||
clip_model_dir = load.get_checkpoint_dir(self.framework_model_dir, self.version, self.pipeline, self.subfolder)
|
||||
if not load.is_model_cached(clip_model_dir, model_opts, self.hf_safetensor, model_name="model"):
|
||||
model = CLIPTextModelWithProjection.from_pretrained(
|
||||
self.path,
|
||||
subfolder=self.subfolder,
|
||||
use_safetensors=self.hf_safetensor,
|
||||
token=self.hf_token,
|
||||
attn_implementation="eager",
|
||||
**model_opts,
|
||||
).to(self.device)
|
||||
model.save_pretrained(clip_model_dir, **model_opts)
|
||||
else:
|
||||
print(f"[I] Load CLIPTextModelWithProjection model from: {clip_model_dir}")
|
||||
model = CLIPTextModelWithProjection.from_pretrained(clip_model_dir, **model_opts).to(self.device)
|
||||
model = optimizer.optimize_checkpoint(model, torch_inference)
|
||||
return model
|
||||
|
||||
def get_input_names(self):
|
||||
return ["input_ids", "attention_mask"]
|
||||
|
||||
def get_output_names(self):
|
||||
return ["text_embeddings"]
|
||||
|
||||
def get_dynamic_axes(self):
|
||||
return {
|
||||
"input_ids": {0: "B"},
|
||||
"attention_mask": {0: "B"},
|
||||
"text_embeddings": {0: "B"},
|
||||
}
|
||||
|
||||
def get_input_profile(self, batch_size, image_height, image_width, static_batch, static_shape):
|
||||
self.check_dims(batch_size, image_height, image_width)
|
||||
min_batch, max_batch, _, _, _, _, _, _, _, _ = self.get_minmax_dims(
|
||||
batch_size, image_height, image_width, static_batch, static_shape
|
||||
)
|
||||
return {
|
||||
"input_ids": [(min_batch, self.text_maxlen), (batch_size, self.text_maxlen), (max_batch, self.text_maxlen)],
|
||||
"attention_mask": [
|
||||
(min_batch, self.text_maxlen),
|
||||
(batch_size, self.text_maxlen),
|
||||
(max_batch, self.text_maxlen),
|
||||
],
|
||||
}
|
||||
|
||||
def get_shape_dict(self, batch_size, image_height, image_width):
|
||||
self.check_dims(batch_size, image_height, image_width)
|
||||
output = {
|
||||
"input_ids": (batch_size, self.text_maxlen),
|
||||
"attention_mask": (batch_size, self.text_maxlen),
|
||||
"text_embeddings": (batch_size, self.embedding_dim),
|
||||
}
|
||||
if "hidden_states" in self.extra_output_names:
|
||||
output["hidden_states"] = (batch_size, self.text_maxlen, self.embedding_dim)
|
||||
return output
|
||||
|
||||
def get_sample_input(self, batch_size, image_height, image_width, static_shape):
|
||||
self.check_dims(batch_size, image_height, image_width)
|
||||
return (
|
||||
torch.zeros(batch_size, self.text_maxlen, dtype=torch.int32, device=self.device),
|
||||
torch.zeros(batch_size, self.text_maxlen, dtype=torch.int32, device=self.device),
|
||||
)
|
||||
|
||||
class SD3_CLIPGModel(CLIPModel):
|
||||
def __init__(
|
||||
self,
|
||||
version,
|
||||
pipeline,
|
||||
device,
|
||||
hf_token,
|
||||
verbose,
|
||||
framework_model_dir,
|
||||
max_batch_size,
|
||||
embedding_dim=None,
|
||||
fp16=False,
|
||||
pooled_output=False,
|
||||
):
|
||||
self.CLIPG_CONFIG = {
|
||||
"hidden_act": "gelu",
|
||||
"hidden_size": 1280,
|
||||
"intermediate_size": 5120,
|
||||
"num_attention_heads": 20,
|
||||
"num_hidden_layers": 32,
|
||||
}
|
||||
super(SD3_CLIPGModel, self).__init__(
|
||||
version,
|
||||
pipeline,
|
||||
device=device,
|
||||
hf_token=hf_token,
|
||||
verbose=verbose,
|
||||
framework_model_dir=framework_model_dir,
|
||||
fp16=fp16,
|
||||
max_batch_size=max_batch_size,
|
||||
embedding_dim=self.CLIPG_CONFIG["hidden_size"] if embedding_dim is None else embedding_dim,
|
||||
)
|
||||
self.subfolder = "text_encoders"
|
||||
if pooled_output:
|
||||
self.extra_output_names = ["pooled_output"]
|
||||
|
||||
def get_model(self, torch_inference=""):
|
||||
clip_g_model_dir = load.get_checkpoint_dir(
|
||||
self.framework_model_dir, self.version, self.pipeline, self.subfolder
|
||||
)
|
||||
clip_g_filename = "clip_g.safetensors"
|
||||
clip_g_model_path = f"{clip_g_model_dir}/{clip_g_filename}"
|
||||
if not os.path.exists(clip_g_model_path):
|
||||
hf_hub_download(
|
||||
repo_id=self.path,
|
||||
filename=clip_g_filename,
|
||||
local_dir=load.get_checkpoint_dir(self.framework_model_dir, self.version, self.pipeline, ""),
|
||||
subfolder=self.subfolder,
|
||||
)
|
||||
with safe_open(clip_g_model_path, framework="pt", device=self.device) as f:
|
||||
dtype = torch.float16 if self.fp16 else torch.float32
|
||||
model = SDXLClipG(self.CLIPG_CONFIG, device=self.device, dtype=dtype)
|
||||
load_into(f, model.transformer, "", self.device, dtype)
|
||||
model = optimizer.optimize_checkpoint(model, torch_inference)
|
||||
return model
|
||||
|
||||
def get_shape_dict(self, batch_size, image_height, image_width):
|
||||
self.check_dims(batch_size, image_height, image_width)
|
||||
output = {
|
||||
"input_ids": (batch_size, self.text_maxlen),
|
||||
"text_embeddings": (batch_size, self.text_maxlen, self.embedding_dim),
|
||||
}
|
||||
if "pooled_output" in self.extra_output_names:
|
||||
output["pooled_output"] = (batch_size, self.embedding_dim)
|
||||
|
||||
return output
|
||||
|
||||
def optimize(self, onnx_graph):
|
||||
opt = optimizer.Optimizer(onnx_graph, verbose=self.verbose, version=self.version)
|
||||
opt.info(self.name + ": original")
|
||||
opt.select_outputs([0, 1])
|
||||
opt.cleanup()
|
||||
opt.fold_constants()
|
||||
opt.info(self.name + ": fold constants")
|
||||
opt.infer_shapes()
|
||||
opt.info(self.name + ": shape inference")
|
||||
opt.select_outputs([0, 1], names=["text_embeddings", "pooled_output"]) # rename network output
|
||||
opt.info(self.name + ": rename output[0] and output[1]")
|
||||
opt_onnx_graph = opt.cleanup(return_onnx=True)
|
||||
opt.info(self.name + ": finished")
|
||||
return opt_onnx_graph
|
||||
|
||||
|
||||
class SD3_CLIPLModel(SD3_CLIPGModel):
|
||||
def __init__(
|
||||
self,
|
||||
version,
|
||||
pipeline,
|
||||
device,
|
||||
hf_token,
|
||||
verbose,
|
||||
framework_model_dir,
|
||||
max_batch_size,
|
||||
fp16=False,
|
||||
pooled_output=False,
|
||||
):
|
||||
self.CLIPL_CONFIG = {
|
||||
"hidden_act": "quick_gelu",
|
||||
"hidden_size": 768,
|
||||
"intermediate_size": 3072,
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 12,
|
||||
}
|
||||
super(SD3_CLIPLModel, self).__init__(
|
||||
version,
|
||||
pipeline,
|
||||
device=device,
|
||||
hf_token=hf_token,
|
||||
verbose=verbose,
|
||||
framework_model_dir=framework_model_dir,
|
||||
fp16=fp16,
|
||||
max_batch_size=max_batch_size,
|
||||
embedding_dim=self.CLIPL_CONFIG["hidden_size"],
|
||||
)
|
||||
self.subfolder = "text_encoders"
|
||||
if pooled_output:
|
||||
self.extra_output_names = ["pooled_output"]
|
||||
|
||||
def get_model(self, torch_inference=""):
|
||||
clip_l_model_dir = load.get_checkpoint_dir(
|
||||
self.framework_model_dir, self.version, self.pipeline, self.subfolder
|
||||
)
|
||||
clip_l_filename = "clip_l.safetensors"
|
||||
clip_l_model_path = f"{clip_l_model_dir}/{clip_l_filename}"
|
||||
if not os.path.exists(clip_l_model_path):
|
||||
hf_hub_download(
|
||||
repo_id=self.path,
|
||||
filename=clip_l_filename,
|
||||
local_dir=load.get_checkpoint_dir(self.framework_model_dir, self.version, self.pipeline, ""),
|
||||
subfolder=self.subfolder,
|
||||
)
|
||||
with safe_open(clip_l_model_path, framework="pt", device=self.device) as f:
|
||||
dtype = torch.float16 if self.fp16 else torch.float32
|
||||
model = SDClipModel(
|
||||
layer="hidden",
|
||||
layer_idx=-2,
|
||||
device=self.device,
|
||||
dtype=dtype,
|
||||
layer_norm_hidden_state=False,
|
||||
return_projected_pooled=False,
|
||||
textmodel_json_config=self.CLIPL_CONFIG,
|
||||
)
|
||||
load_into(f, model.transformer, "", self.device, dtype)
|
||||
model = optimizer.optimize_checkpoint(model, torch_inference)
|
||||
return model
|
||||
|
||||
|
||||
# NOTE: For legacy reasons, even though this is a T5 model, it inherits from CLIPModel.
|
||||
class SD3_T5XXLModel(CLIPModel):
|
||||
def __init__(
|
||||
self,
|
||||
version,
|
||||
pipeline,
|
||||
device,
|
||||
hf_token,
|
||||
verbose,
|
||||
framework_model_dir,
|
||||
max_batch_size,
|
||||
embedding_dim,
|
||||
fp16=False,
|
||||
):
|
||||
super(SD3_T5XXLModel, self).__init__(
|
||||
version,
|
||||
pipeline,
|
||||
device=device,
|
||||
hf_token=hf_token,
|
||||
verbose=verbose,
|
||||
framework_model_dir=framework_model_dir,
|
||||
fp16=fp16,
|
||||
max_batch_size=max_batch_size,
|
||||
embedding_dim=embedding_dim,
|
||||
)
|
||||
self.T5_CONFIG = {"d_ff": 10240, "d_model": 4096, "num_heads": 64, "num_layers": 24, "vocab_size": 32128}
|
||||
self.subfolder = "text_encoders"
|
||||
|
||||
def get_model(self, torch_inference=""):
|
||||
t5xxl_model_dir = load.get_checkpoint_dir(self.framework_model_dir, self.version, self.pipeline, self.subfolder)
|
||||
t5xxl_filename = "t5xxl_fp16.safetensors"
|
||||
t5xxl_model_path = f"{t5xxl_model_dir}/{t5xxl_filename}"
|
||||
if not os.path.exists(t5xxl_model_path):
|
||||
hf_hub_download(
|
||||
repo_id=self.path,
|
||||
filename=t5xxl_filename,
|
||||
local_dir=load.get_checkpoint_dir(self.framework_model_dir, self.version, self.pipeline, ""),
|
||||
subfolder=self.subfolder,
|
||||
)
|
||||
with safe_open(t5xxl_model_path, framework="pt", device=self.device) as f:
|
||||
dtype = torch.float16 if self.fp16 else torch.float32
|
||||
model = T5XXLModel(self.T5_CONFIG, device=self.device, dtype=dtype)
|
||||
load_into(f, model.transformer, "", self.device, dtype)
|
||||
model = optimizer.optimize_checkpoint(model, torch_inference)
|
||||
return model
|
||||
|
||||
|
||||
class CLIPVisionWithProjModel(base_model.BaseModel):
|
||||
def __init__(
|
||||
self,
|
||||
version,
|
||||
pipeline,
|
||||
device,
|
||||
hf_token,
|
||||
verbose,
|
||||
framework_model_dir,
|
||||
max_batch_size=1,
|
||||
subfolder="image_encoder",
|
||||
):
|
||||
|
||||
super(CLIPVisionWithProjModel, self).__init__(
|
||||
version,
|
||||
pipeline,
|
||||
device=device,
|
||||
hf_token=hf_token,
|
||||
verbose=verbose,
|
||||
framework_model_dir=framework_model_dir,
|
||||
max_batch_size=max_batch_size,
|
||||
)
|
||||
self.subfolder = subfolder
|
||||
|
||||
def get_model(self, torch_inference=""):
|
||||
clip_model_dir = load.get_checkpoint_dir(self.framework_model_dir, self.version, self.pipeline, self.subfolder)
|
||||
if not os.path.exists(clip_model_dir):
|
||||
model = CLIPVisionModelWithProjection.from_pretrained(
|
||||
self.path, subfolder=self.subfolder, use_safetensors=self.hf_safetensor, token=self.hf_token
|
||||
).to(self.device)
|
||||
model.save_pretrained(clip_model_dir)
|
||||
else:
|
||||
print(f"[I] Load CLIPVisionModelWithProjection model from: {clip_model_dir}")
|
||||
model = CLIPVisionModelWithProjection.from_pretrained(clip_model_dir).to(self.device)
|
||||
model = optimizer.optimize_checkpoint(model, torch_inference)
|
||||
return model
|
||||
|
||||
|
||||
class CLIPImageProcessorModel(base_model.BaseModel):
|
||||
def __init__(
|
||||
self,
|
||||
version,
|
||||
pipeline,
|
||||
device,
|
||||
hf_token,
|
||||
verbose,
|
||||
framework_model_dir,
|
||||
max_batch_size=1,
|
||||
subfolder="feature_extractor",
|
||||
):
|
||||
|
||||
super(CLIPImageProcessorModel, self).__init__(
|
||||
version,
|
||||
pipeline,
|
||||
device=device,
|
||||
hf_token=hf_token,
|
||||
verbose=verbose,
|
||||
framework_model_dir=framework_model_dir,
|
||||
max_batch_size=max_batch_size,
|
||||
)
|
||||
self.subfolder = subfolder
|
||||
|
||||
def get_model(self, torch_inference=""):
|
||||
clip_model_dir = load.get_checkpoint_dir(self.framework_model_dir, self.version, self.pipeline, self.subfolder)
|
||||
# NOTE to(device) not supported
|
||||
if not os.path.exists(clip_model_dir):
|
||||
model = CLIPImageProcessor.from_pretrained(
|
||||
self.path, subfolder=self.subfolder, use_safetensors=self.hf_safetensor, token=self.hf_token
|
||||
)
|
||||
model.save_pretrained(clip_model_dir)
|
||||
else:
|
||||
print(f"[I] Load CLIPImageProcessor model from: {clip_model_dir}")
|
||||
model = CLIPImageProcessor.from_pretrained(clip_model_dir)
|
||||
return model
|
||||
@@ -0,0 +1,223 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
from demo_diffusion.dynamic_import import import_from_diffusers
|
||||
from demo_diffusion.model import base_model, load, optimizer
|
||||
|
||||
# List of models to import from diffusers.models
|
||||
models_to_import = ["SD3Transformer2DModel", "SD3ControlNetModel"]
|
||||
for model in models_to_import:
|
||||
globals()[model] = import_from_diffusers(model, "diffusers.models")
|
||||
|
||||
|
||||
class SD3ControlNetWrapper(torch.nn.Module):
|
||||
def __init__(self, controlnet):
|
||||
super().__init__()
|
||||
self.controlnet = controlnet
|
||||
|
||||
def forward(self, hidden_states, controlnet_cond, conditioning_scale, pooled_projections, timestep):
|
||||
params = {
|
||||
"hidden_states": hidden_states,
|
||||
"pooled_projections": pooled_projections,
|
||||
"timestep": timestep,
|
||||
"controlnet_cond": controlnet_cond,
|
||||
"conditioning_scale": conditioning_scale,
|
||||
}
|
||||
out = self.controlnet(**params)["controlnet_block_samples"]
|
||||
return torch.stack(out, dim=0)
|
||||
|
||||
|
||||
class SD3ControlNet(base_model.BaseModel):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
version,
|
||||
controlnet,
|
||||
pipeline,
|
||||
device,
|
||||
hf_token,
|
||||
verbose,
|
||||
framework_model_dir,
|
||||
fp16=False,
|
||||
tf32=False,
|
||||
bf16=False,
|
||||
int8=False,
|
||||
fp8=False,
|
||||
max_batch_size=16,
|
||||
do_classifier_free_guidance=False,
|
||||
):
|
||||
super(SD3ControlNet, self).__init__(
|
||||
version,
|
||||
pipeline,
|
||||
device=device,
|
||||
hf_token=hf_token,
|
||||
verbose=verbose,
|
||||
framework_model_dir=framework_model_dir,
|
||||
fp16=fp16,
|
||||
tf32=tf32,
|
||||
bf16=bf16,
|
||||
int8=int8,
|
||||
fp8=fp8,
|
||||
max_batch_size=max_batch_size,
|
||||
)
|
||||
self.path = load.get_path(version, pipeline, controlnet)
|
||||
self.subfolder = "controlnet_{}".format(controlnet)
|
||||
self.controlnet_model_dir = load.get_checkpoint_dir(
|
||||
self.framework_model_dir, self.version, self.pipeline, self.subfolder
|
||||
)
|
||||
self.transformer_model_dir = load.get_checkpoint_dir(
|
||||
self.framework_model_dir, self.version, self.pipeline, "transformer"
|
||||
)
|
||||
if not os.path.exists(self.controlnet_model_dir):
|
||||
self.config = SD3ControlNetModel.load_config(self.path, token=self.hf_token)
|
||||
else:
|
||||
print(f"[I] Load SD3ControlNetModel config from: {self.controlnet_model_dir}")
|
||||
self.config = SD3ControlNetModel.load_config(self.controlnet_model_dir)
|
||||
self.xB = 2 if do_classifier_free_guidance else 1 # batch multiplier
|
||||
|
||||
def get_model(self, torch_inference=""):
|
||||
model_opts = (
|
||||
{"torch_dtype": torch.float16} if self.fp16 else {"torch_dtype": torch.bfloat16} if self.bf16 else {}
|
||||
)
|
||||
if not load.is_model_cached(self.controlnet_model_dir, model_opts, self.hf_safetensor):
|
||||
model = SD3ControlNetModel.from_pretrained(self.path, **model_opts, use_safetensors=self.hf_safetensor).to(
|
||||
self.device
|
||||
)
|
||||
model.save_pretrained(self.controlnet_model_dir, **model_opts)
|
||||
else:
|
||||
print(f"[I] Load SD3ControlNetModel model from: {self.controlnet_model_dir}")
|
||||
model = SD3ControlNetModel.from_pretrained(self.controlnet_model_dir, **model_opts).to(self.device)
|
||||
|
||||
# Load transformer model for pos_embed
|
||||
transformer = SD3Transformer2DModel.from_pretrained(self.transformer_model_dir, **model_opts).to(self.device)
|
||||
|
||||
if hasattr(model.config, "use_pos_embed") and model.config.use_pos_embed is False:
|
||||
pos_embed = model._get_pos_embed_from_transformer(transformer)
|
||||
model.pos_embed = pos_embed.to(model.dtype).to(model.device)
|
||||
# Free transformer model
|
||||
del transformer
|
||||
|
||||
model = optimizer.optimize_checkpoint(model, torch_inference)
|
||||
model = SD3ControlNetWrapper(model)
|
||||
return model
|
||||
|
||||
def get_input_names(self):
|
||||
return ["hidden_states", "controlnet_cond", "conditioning_scale", "pooled_projections", "timestep"]
|
||||
|
||||
def get_output_names(self):
|
||||
return ["controlnet_block_samples"]
|
||||
|
||||
def get_dynamic_axes(self):
|
||||
xB = "2B" if self.xB == 2 else "B"
|
||||
dynamic_axes = {
|
||||
"hidden_states": {0: xB, 2: "H", 3: "W"},
|
||||
"controlnet_cond": {0: xB, 2: "H", 3: "W"},
|
||||
"pooled_projections": {0: xB},
|
||||
"timestep": {0: xB},
|
||||
}
|
||||
return dynamic_axes
|
||||
|
||||
def get_input_profile(
|
||||
self,
|
||||
batch_size: int,
|
||||
image_height: int,
|
||||
image_width: int,
|
||||
static_batch: bool,
|
||||
static_shape: bool,
|
||||
):
|
||||
latent_height, latent_width = self.check_dims(batch_size, image_height, image_width)
|
||||
(
|
||||
min_batch,
|
||||
max_batch,
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
min_latent_height,
|
||||
max_latent_height,
|
||||
min_latent_width,
|
||||
max_latent_width,
|
||||
) = self.get_minmax_dims(batch_size, image_height, image_width, static_batch, static_shape)
|
||||
|
||||
input_profile = {
|
||||
"hidden_states": [
|
||||
(self.xB * min_batch, self.config["in_channels"], min_latent_height, min_latent_width),
|
||||
(self.xB * batch_size, self.config["in_channels"], latent_height, latent_width),
|
||||
(self.xB * max_batch, self.config["in_channels"], max_latent_height, max_latent_width),
|
||||
],
|
||||
"timestep": [(self.xB * min_batch,), (self.xB * batch_size,), (self.xB * max_batch,)],
|
||||
"pooled_projections": [
|
||||
(self.xB * min_batch, self.config["pooled_projection_dim"]),
|
||||
(self.xB * batch_size, self.config["pooled_projection_dim"]),
|
||||
(self.xB * max_batch, self.config["pooled_projection_dim"]),
|
||||
],
|
||||
"controlnet_cond": [
|
||||
(self.xB * min_batch, self.config["in_channels"], min_latent_height, min_latent_width),
|
||||
(self.xB * batch_size, self.config["in_channels"], latent_height, latent_width),
|
||||
(self.xB * max_batch, self.config["in_channels"], max_latent_height, max_latent_width),
|
||||
],
|
||||
}
|
||||
return input_profile
|
||||
|
||||
def get_shape_dict(self, batch_size, image_height, image_width):
|
||||
latent_height, latent_width = self.check_dims(batch_size, image_height, image_width)
|
||||
shape_dict = {
|
||||
"hidden_states": (self.xB * batch_size, self.config["in_channels"], latent_height, latent_width),
|
||||
"timestep": (self.xB * batch_size,),
|
||||
"pooled_projections": (self.xB * batch_size, self.config["pooled_projection_dim"]),
|
||||
"controlnet_cond": (self.xB * batch_size, self.config["in_channels"], latent_height, latent_width),
|
||||
"conditioning_scale": (),
|
||||
"controlnet_block_samples": (
|
||||
self.config["num_layers"],
|
||||
self.xB * batch_size,
|
||||
latent_height // 2 * latent_width // 2,
|
||||
self.config["num_attention_heads"] * self.config["attention_head_dim"],
|
||||
),
|
||||
}
|
||||
return shape_dict
|
||||
|
||||
def get_sample_input(self, batch_size, image_height, image_width, static_shape):
|
||||
dtype = torch.float16 if self.fp16 else torch.bfloat16 if self.bf16 else torch.float32
|
||||
latent_height, latent_width = self.check_dims(batch_size, image_height, image_width)
|
||||
sample_input = (
|
||||
torch.randn(
|
||||
self.xB * batch_size,
|
||||
self.config["in_channels"],
|
||||
latent_height,
|
||||
latent_width,
|
||||
dtype=dtype,
|
||||
device=self.device,
|
||||
),
|
||||
torch.randn(
|
||||
self.xB * batch_size,
|
||||
self.config["in_channels"],
|
||||
latent_height,
|
||||
latent_width,
|
||||
dtype=dtype,
|
||||
device=self.device,
|
||||
),
|
||||
torch.tensor(1.0, dtype=dtype, device=self.device),
|
||||
torch.randn(self.xB * batch_size, self.config["pooled_projection_dim"], dtype=dtype, device=self.device),
|
||||
torch.randn(self.xB * batch_size, dtype=torch.float32, device=self.device),
|
||||
)
|
||||
|
||||
return sample_input
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user