wehub/nvidia--tensorrt
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Docker Image CI / build-ubuntu2004 (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:36:55 +08:00
commit c8a779b1bb
1887 changed files with 3245738 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/Polygraphy/polygraphy/comparator/__init__.py
+5
View File
@@ -0,0 +1,5 @@
from polygraphy.comparator.comparator import *
from polygraphy.comparator.compare import *
from polygraphy.comparator.data_loader import *
from polygraphy.comparator.postprocess import *
from polygraphy.comparator.struct import *
tools/Polygraphy/polygraphy/comparator/comparator.py
+423
View File
@@ -0,0 +1,423 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import contextlib
import copy
import queue
from multiprocessing import Process, Queue
from polygraphy import mod, util
from polygraphy.common import TensorMetadata
from polygraphy.comparator import util as comp_util
from polygraphy.comparator.compare import CompareFunc
from polygraphy.comparator.data_loader import DataLoader, DataLoaderCache
from polygraphy.comparator.struct import AccuracyResult, IterationResult, RunResults
from polygraphy.logger import G_LOGGER, LogMode
@mod.export()
class Comparator:
"""
Compares inference outputs.
"""
@staticmethod
def run(
runners,
data_loader=None,
warm_up=None,
use_subprocess=None,
subprocess_timeout=None,
subprocess_polling_interval=None,
save_inputs_path=None,
):
"""
Runs the supplied runners sequentially.
Args:
runners (List[BaseRunner]):
A list of runners to run.
data_loader (Sequence[OrderedDict[str, numpy.ndarray]]):
A generator or iterable that yields a dictionary that maps input names to input numpy buffers.
In the simplest case, this can be a `List[Dict[str, numpy.ndarray]]` .
In case you don't know details about the inputs ahead of time, you can access the
`input_metadata` property in your data loader, which will be set to an `TensorMetadata`
instance by this function.
Note that this does not work for generators or lists.
The number of iterations run by this function is controlled by the number of items supplied
by the data loader.
Defaults to an instance of `DataLoader`.
warm_up (int):
The number of warm up runs to perform for each runner before timing.
Defaults to 0.
use_subprocess (bool):
Whether each runner should be run in a subprocess. This allows each runner to have exclusive
access to the GPU. When using a subprocess, runners and loaders will never be modified.
subprocess_timeout (int):
The timeout before a subprocess is killed automatically. This is useful for handling processes
that never terminate. A value of None disables the timeout. Defaults to None.
subprocess_polling_interval (int):
The polling interval, in seconds, for checking whether a subprocess has completed or crashed.
In rare cases, omitting this parameter when subprocesses are enabled may cause this function
to hang indefinitely if the subprocess crashes.
A value of 0 disables polling. Defaults to 30 seconds.
save_inputs_path (str):
Path at which to save inputs used during inference. This will include all inputs generated by
the provided data_loader, and will be saved as a JSON List[Dict[str, numpy.ndarray]].
Returns:
RunResults:
A mapping of runner names to the results of their inference.
The ordering of `runners` is preserved in this mapping.
"""
warm_up = util.default(warm_up, 0)
data_loader = util.default(data_loader, DataLoader())
use_subprocess = util.default(use_subprocess, False)
subprocess_polling_interval = util.default(subprocess_polling_interval, 30)
loader_cache = DataLoaderCache(data_loader, save_inputs_path=save_inputs_path)
def execute_runner(runner, loader_cache):
with runner as active_runner:
# DataLoaderCache will ensure that the feed_dict does not contain any extra entries
# based on the provided input_metadata.
loader_cache.set_input_metadata(
active_runner.get_input_metadata(use_numpy_dtypes=False)
)
if warm_up:
G_LOGGER.start(
f"{active_runner.name:35} | Running {warm_up} warm-up run(s)"
)
try:
feed_dict = loader_cache[0]
except IndexError:
G_LOGGER.warning(
f"{warm_up} warm-up run(s) were requested, but data loader did not supply any data. Skipping warm-up run(s)"
)
else:
G_LOGGER.ultra_verbose(
f"Warm-up Input Buffers:\n{util.indent_block(feed_dict)}"
)
# First do a few warm-up runs, and don't time them.
for _ in range(warm_up):
active_runner.infer(feed_dict=feed_dict)
G_LOGGER.finish(
f"{active_runner.name:35} | Finished {warm_up} warm-up run(s)"
)
# Then, actual iterations.
index = 0
iteration_results = []
iterations_num = len(loader_cache)
total_runtime = 0
for index, feed_dict in enumerate(loader_cache):
G_LOGGER.info(
f"{active_runner.name:35}\n---- Inference Input(s) ----\n{TensorMetadata().from_feed_dict(feed_dict)}",
mode=LogMode.ONCE,
)
G_LOGGER.extra_verbose(
lambda: f"{active_runner.name:35} | Feeding inputs:\n{util.indent_block(dict(feed_dict))}"
)
outputs = active_runner.infer(feed_dict=feed_dict)
runtime = active_runner.last_inference_time()
total_runtime += runtime
# Only make a deep copy if we have more than one iteration.
# For single iteration case, we can use the outputs directly since they won't be reused.
# This allows running with a large number of outputs (e.g. for accuracy debugging) without memory explosion.
iteration_results.append(
IterationResult(
outputs=copy.deepcopy(outputs) if iterations_num > 1 else outputs,
runtime=runtime,
runner_name=active_runner.name,
)
)
G_LOGGER.info(
f"{active_runner.name:35}\n---- Inference Output(s) ----\n{TensorMetadata().from_feed_dict(outputs)}",
mode=LogMode.ONCE,
)
G_LOGGER.extra_verbose(
lambda: f"{active_runner.name:35} | Inference Time: {runtime * 1000.0:.3f} ms | Received outputs:\n{util.indent_block(dict(outputs))}"
)
total_runtime_ms = total_runtime * 1000.0
G_LOGGER.finish(
f"{active_runner.name:35} | Completed {index + 1} iteration(s) in {total_runtime_ms:.4g} ms | Average inference time: {total_runtime_ms / float(index + 1):.4g} ms."
)
return iteration_results
# Wraps execute_runner to use a queue.
def execute_runner_with_queue(runner_queue, runner, loader_cache):
iteration_results = None
try:
iteration_results = execute_runner(runner, loader_cache)
except:
# Cannot necessarily send the exception back over the queue.
G_LOGGER.backrace()
util.try_send_on_queue(runner_queue, iteration_results)
# After finishing, send the updated loader_cache back.
util.try_send_on_queue(runner_queue, loader_cache)
# Do all inferences in one loop, then comparisons at a later stage.
# We run each runner in a separate process so that we can provide exclusive GPU access for each runner.
run_results = RunResults()
if not runners:
G_LOGGER.warning(
"No runners were provided to Comparator.run(). Inference will not be run, and run results will be empty."
)
for runner in runners:
G_LOGGER.start(f"{runner.name:35} | Activating and starting inference")
if use_subprocess:
runner_queue = Queue()
process = Process(
target=execute_runner_with_queue,
args=(runner_queue, runner, loader_cache),
)
process.start()
# If a subprocess hangs in a certain way, then process.join could block forever. Hence,
# we need to keep polling the process to make sure it really is alive.
iteration_results = None
while process.is_alive() and iteration_results is None:
try:
iteration_results = util.try_receive_on_queue(
runner_queue, timeout=subprocess_polling_interval / 2
)
# Receive updated loader cache, or fall back if it could not be sent.
loader_cache = util.try_receive_on_queue(
runner_queue, timeout=subprocess_polling_interval / 2
)
except queue.Empty:
G_LOGGER.extra_verbose("Polled subprocess - still running")
try:
assert iteration_results is not None
run_results.append((runner.name, iteration_results))
process.join(subprocess_timeout)
except:
G_LOGGER.critical(
f"{runner.name:35} | Terminated prematurely. Check the exception logged above. If there is no exception logged above, make sure not to use the --use-subprocess flag or set use_subprocess=False in Comparator.run()."
)
finally:
process.terminate()
if loader_cache is None:
G_LOGGER.critical(
"Could not send data loader cache to runner subprocess. Please try disabling subprocesses "
"by removing the --use-subprocess flag, or setting use_subprocess=False in Comparator.run()"
)
else:
run_results.append((runner.name, execute_runner(runner, loader_cache)))
G_LOGGER.verbose(f"Successfully ran: {[r.name for r in runners]}")
return run_results
@staticmethod
def postprocess(run_results, postprocess_func):
"""
Applies post processing to all the outputs in the provided run results.
This is a convenience function to avoid the need for manual iteration over the run_results dictionary.
Args:
run_results (RunResults): The result of Comparator.run().
postprocess_func (Callable(IterationResult) -> IterationResult):
The function to apply to each ``IterationResult``.
Returns:
RunResults: The updated run results.
"""
G_LOGGER.start(
f"Applying post-processing to outputs: {postprocess_func.__name__}"
)
for _, iteration_results in run_results:
for index, iter_res in enumerate(iteration_results):
iteration_results[index] = postprocess_func(iter_res)
G_LOGGER.finish("Finished applying post-processing")
return run_results
@staticmethod
def default_comparisons(run_results):
# Sets up default comparisons - which is to compare each runner to the subsequent one.
return [(i, i + 1) for i in range(len(run_results) - 1)]
@staticmethod
def compare_accuracy(
run_results, fail_fast=False, comparisons=None, compare_func=None
):
"""
Args:
run_results (RunResults): The result of Comparator.run()
fail_fast (bool): Whether to exit after the first failure
comparisons (List[Tuple[int, int]]):
Comparisons to perform, specified by runner indexes. For example, [(0, 1), (1, 2)]
would compare the first runner with the second, and the second with the third.
By default, this compares each result to the subsequent one.
compare_func (Callable(IterationResult, IterationResult) -> OrderedDict[str, bool]):
A function that takes in two IterationResults, and returns a dictionary that maps output
names to a boolean (or anything convertible to a boolean) indicating whether outputs matched.
The order of arguments to this function is guaranteed to be the same as the ordering of the
tuples contained in `comparisons`.
Returns:
AccuracyResult:
A summary of the results of the comparisons. The order of the keys (i.e. runner pairs) is
guaranteed to be the same as the order of `comparisons`. For more details, see the AccuracyResult
docstring (e.g. help(AccuracyResult)).
"""
def find_mismatched(match_dict):
return [name for name, matched in match_dict.items() if not bool(matched)]
compare_func = util.default(compare_func, CompareFunc.simple())
comparisons = util.default(
comparisons, Comparator.default_comparisons(run_results)
)
accuracy_result = AccuracyResult()
for runner0_index, runner1_index in comparisons:
(runner0_name, results0), (runner1_name, results1) = (
run_results[runner0_index],
run_results[runner1_index],
)
G_LOGGER.start(f"Accuracy Comparison | {runner0_name} vs. {runner1_name}")
with G_LOGGER.indent():
runner_pair = (runner0_name, runner1_name)
accuracy_result[runner_pair] = []
num_iters = min(len(results0), len(results1))
for iteration, (result0, result1) in enumerate(zip(results0, results1)):
if num_iters > 1:
G_LOGGER.info(f"Iteration: {iteration}")
with contextlib.ExitStack() as stack:
if num_iters > 1:
stack.enter_context(G_LOGGER.indent())
iteration_match_dict = compare_func(result0, result1)
accuracy_result[runner_pair].append(iteration_match_dict)
mismatched_outputs = find_mismatched(iteration_match_dict)
if fail_fast and mismatched_outputs:
return accuracy_result
G_LOGGER.extra_verbose(
f"Finished comparing {runner0_name} with {runner1_name}"
)
passed, _, total = accuracy_result.stats(runner_pair)
pass_rate = accuracy_result.percentage(runner_pair) * 100.0
msg = f"Accuracy Summary | {runner0_name} vs. {runner1_name} | Passed: {passed}/{total} iterations | Pass Rate: {pass_rate}%"
if passed == total:
G_LOGGER.finish(msg)
else:
G_LOGGER.error(msg)
return accuracy_result
@staticmethod
def validate(run_results, check_inf=None, check_nan=None, fail_fast=None):
"""
Checks output validity.
Args:
run_results (Dict[str, List[IterationResult]]): The result of Comparator.run().
check_inf (bool): Whether to fail on Infs. Defaults to False.
check_nan (bool): Whether to fail on NaNs. Defaults to True.
fail_fast (bool): Whether to fail after the first invalid value. Defaults to False.
Returns:
bool: True if all outputs were valid, False otherwise.
"""
check_inf = util.default(check_inf, False)
check_nan = util.default(check_nan, True)
fail_fast = util.default(fail_fast, False)
def is_finite(output):
non_finite = util.array.logical_not(util.array.isfinite(output))
if util.array.any(non_finite):
G_LOGGER.error(
"Inf Detected | One or more non-finite values were encountered in this output"
)
G_LOGGER.info(
"Note: Use -vv or set logging verbosity to EXTRA_VERBOSE to display non-finite values",
mode=LogMode.ONCE,
)
G_LOGGER.extra_verbose(f"Note: non-finite values at:\n{non_finite}")
G_LOGGER.extra_verbose(
f"Note: non-finite values:\n{output[non_finite]}"
)
return False
return True
def is_not_nan(output):
nans = util.array.isnan(output)
if util.array.any(nans):
G_LOGGER.error(
"NaN Detected | One or more NaNs were encountered in this output"
)
G_LOGGER.info(
"Note: Use -vv or set logging verbosity to EXTRA_VERBOSE to display locations of NaNs",
mode=LogMode.ONCE,
)
G_LOGGER.extra_verbose(f"Note: NaNs at:\n{nans}")
return False
return True
def validate_output(runner_name, output_name, output):
G_LOGGER.start(
f"{runner_name:35} | Validating output: {output_name} (check_inf={check_inf}, check_nan={check_nan})"
)
with G_LOGGER.indent():
comp_util.log_output_stats(output)
output_valid = True
if check_nan:
output_valid &= is_not_nan(output)
if check_inf:
output_valid &= is_finite(output)
if output_valid:
G_LOGGER.finish(f"PASSED | Output: {output_name} is valid")
else:
G_LOGGER.error(f"FAILED | Errors detected in output: {output_name}")
return output_valid
all_valid = True
G_LOGGER.start(f"Output Validation | Runners: {list(run_results.keys())}")
with G_LOGGER.indent():
for runner_name, results in run_results:
for result in results:
for output_name, output in result.items():
all_valid &= validate_output(runner_name, output_name, output)
if fail_fast and not all_valid:
return False
if all_valid:
G_LOGGER.finish("PASSED | Output Validation")
else:
G_LOGGER.error("FAILED | Output Validation")
return all_valid
tools/Polygraphy/polygraphy/comparator/compare.py
+1877
View File
File diff suppressed because it is too large Load Diff
tools/Polygraphy/polygraphy/comparator/data_loader.py
+569
View File
@@ -0,0 +1,569 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import contextlib
from collections import OrderedDict
from polygraphy import constants, func, mod, util
from polygraphy.comparator.struct import RunResults
from polygraphy.datatype import DataType
from polygraphy.exception import DataTypeConversionException, PolygraphyException
from polygraphy.json import save_json
from polygraphy.logger import G_LOGGER, LogMode
np = mod.lazy_import("numpy")
torch = mod.lazy_import("torch")
class ArraySampler:
def __init__(self, data_loader_backend_module, seed):
"""
Args:
data_loader_backend_module (str):
The module specifying the array type to use to generate arrays.
Can be either "numpy" or "torch".
seed (int):
The seed to use when generating random inputs.
"""
self.rng = None
VALID_ARRAY_MODULES = ["numpy", "torch"]
if data_loader_backend_module not in VALID_ARRAY_MODULES:
G_LOGGER.critical(
f"Invalid `data_loader_backend_module`. Note: got: {data_loader_backend_module} but valid modules are: {VALID_ARRAY_MODULES}"
)
self.data_loader_backend_module = data_loader_backend_module
if self.data_loader_backend_module == "numpy":
self.rng = np.random.RandomState(seed)
elif self.data_loader_backend_module == "torch":
self.rng = torch.Generator()
self.rng.manual_seed(seed)
def sample_integer(self, shape, dtype, low, high):
"""
Samples an array containing integral values in the range [low, high], inclusive
"""
dtype = (
DataType.to_dtype(
DataType.from_dtype(dtype), self.data_loader_backend_module
)
if dtype is not None
else dtype
)
if self.data_loader_backend_module == "numpy":
return np.array(
self.rng.randint(low=low, high=high + 1, size=shape, dtype=dtype)
)
elif self.data_loader_backend_module == "torch":
return torch.randint(low, high + 1, shape, generator=self.rng, dtype=dtype)
def sample_float(self, shape, dtype, fmin, fmax):
"""
Samples an array containing float values in the range [fmin, fmax], inclusive
"""
# Special handling for infinite lower/upper bounds
# Without this, two infinities will collapse into a NaN, resulting in no infinities
# in the final output.
scale = fmax - fmin
shift = fmin
if util.is_inf(fmin):
scale = fmin
shift = 0
if util.is_inf(fmax):
scale = fmax
dtype = (
DataType.to_dtype(
DataType.from_dtype(dtype), self.data_loader_backend_module
)
if dtype is not None
else dtype
)
if self.data_loader_backend_module == "numpy":
return np.array(
(self.rng.random_sample(size=shape) * scale + shift).astype(dtype)
)
elif self.data_loader_backend_module == "torch":
return torch.rand(shape, generator=self.rng, dtype=dtype)
def constant_array(self, shape, dtype):
dtype = (
DataType.to_dtype(
DataType.from_dtype(dtype), self.data_loader_backend_module
)
if dtype is not None
else dtype
)
if self.data_loader_backend_module == "numpy":
return np.array(shape, dtype=dtype)
elif self.data_loader_backend_module == "torch":
return torch.tensor(shape, dtype=dtype)
@mod.export()
class DataLoader:
"""
Generates synthetic input data.
"""
def __init__(
self,
seed=None,
iterations=None,
input_metadata=None,
int_range=None,
float_range=None,
val_range=None,
data_loader_backend_module=None,
):
"""
Args:
seed (int):
The seed to use when generating random inputs.
Defaults to ``util.constants.DEFAULT_SEED``.
iterations (int):
The number of iterations for which to supply data.
Defaults to 1.
input_metadata (TensorMetadata):
A mapping of input names to their corresponding shapes and data types.
This will be used to determine what shapes to supply for inputs with dynamic shape, as
well as to set the data type of the generated inputs.
If either dtype or shape are None, then the value will be automatically determined.
For input shape tensors, i.e. inputs whose *value* describes a shape in the model, the
provided shape will be used to populate the values of the inputs, rather than to determine
their shape.
val_range (Union[Tuple[number], Dict[str, Tuple[number]]]):
A tuple containing exactly 2 numbers, indicating the minimum and maximum values (inclusive)
the data loader should generate.
If either value in the tuple is None, the default will be used for that value.
If None is provided instead of a tuple, then the default values will be used for both the
minimum and maximum.
This can be specified on a per-input basis using a dictionary. In that case,
use an empty string ("") as the key to specify default range for inputs not explicitly listed.
Defaults to (0.0, 1.0).
data_loader_backend_module (str):
A string denoting what module to use to construct the input data arrays. Currently supports
"numpy" and "torch".
Defaults to "numpy".
int_range (Tuple[int]):
[DEPRECATED - Use val_range instead]
A tuple containing exactly 2 integers, indicating the minimum and maximum integer values (inclusive)
the data loader should generate. If either value in the tuple is None, the default will be used
for that value.
If None is provided instead of a tuple, then the default values will be used for both the
minimum and maximum.
float_range (Tuple[float]):
[DEPRECATED - Use val_range instead]
A tuple containing exactly 2 floats, indicating the minimum and maximum float values (inclusive)
the data loader should generate. If either value in the tuple is None, the default will be used
for that value.
If None is provided instead of a tuple, then the default values will be used for both the
minimum and maximum.
"""
def default_tuple(tup, default):
if tup is None or (
not isinstance(tup, tuple) and not isinstance(tup, list)
):
return default
new_tup = []
for elem, default_elem in zip(tup, default):
new_tup.append(util.default(elem, default_elem))
return tuple(new_tup)
self.seed = util.default(seed, constants.DEFAULT_SEED)
self.iterations = util.default(iterations, 1)
self.user_input_metadata = util.default(input_metadata, {})
self.data_loader_backend_module = util.default(
data_loader_backend_module, "numpy"
)
self._int_range_set = int_range is not None
if self._int_range_set:
mod.warn_deprecated(
"The int_range parameter in DataLoader", "val_range", remove_in="0.50.0"
)
self._int_range = default_tuple(int_range, (1, 25))
self._float_range_set = float_range is not None
if self._float_range_set:
mod.warn_deprecated(
"The float_range parameter in DataLoader",
"val_range",
remove_in="0.50.0",
)
self._float_range = default_tuple(float_range, (-1.0, 1.0))
self.input_metadata = None
self.default_val_range = default_tuple(val_range, (0.0, 1.0))
self.val_range = util.default(val_range, self.default_val_range)
if self.user_input_metadata:
G_LOGGER.info(
f"Will generate inference input data according to provided TensorMetadata: {self.user_input_metadata}"
)
def __repr__(self):
return util.make_repr(
"DataLoader",
seed=self.seed,
iterations=self.iterations,
input_metadata=self.user_input_metadata or None,
int_range=self._int_range,
float_range=self._float_range,
val_range=self.val_range,
data_loader_backend_module=self.data_loader_backend_module,
)[0]
def _get_range(self, name, cast_type):
if cast_type == int and self._int_range_set:
return self._int_range
elif cast_type == float and self._float_range_set:
return self._float_range
tup = util.value_or_from_dict(self.val_range, name, self.default_val_range)
return tuple(cast_type(val) for val in tup)
def __getitem__(self, index):
"""
Generates random input data.
May update the DataLoader's `input_metadata` attribute.
Args:
index (int):
Since this class behaves like an iterable, it takes an index parameter.
Generated data is guaranteed to be the same for the same index.
Returns:
OrderedDict[str, Union[numpy.ndarray, torch.Tensor]]: A mapping of input names to input numpy buffers.
"""
if index >= self.iterations:
raise IndexError()
G_LOGGER.verbose(f"Generating data using numpy seed: {self.seed + index}")
array_sampler = ArraySampler(self.data_loader_backend_module, self.seed + index)
def get_static_shape(name, shape):
static_shape = shape
if util.is_shape_dynamic(shape):
if shape.min is not None:
static_shape = shape.min
elif shape.max is not None:
static_shape = shape.max
else:
static_shape = util.override_dynamic_shape(shape)
if static_shape != shape:
if not util.is_valid_shape_override(static_shape, shape):
G_LOGGER.critical(
f"Input tensor: {name} | Cannot override original shape: {shape} to {static_shape}"
)
G_LOGGER.warning(
f"Input tensor: {name} [shape={shape}] | Will generate data of shape: {static_shape}.\n"
f"If this is incorrect, please provide a custom data loader.",
mode=LogMode.ONCE,
)
return static_shape
# Whether the user provided the values for a shape tensor input,
# rather than the shape of the input.
# If the shape is 1D, and has a value equal to the rank of the provided default shape, it is
# likely to be a shape tensor, and so its value, not shape, should be overriden.
# Note that this is a hack needed for older versions of TensorRT. Ideally, we wouldn't care
# whether the input is a shape tensor or not.
def is_shape_tensor(name, dtype):
if name not in self.input_metadata or name not in self.user_input_metadata:
return False
_, shape = self.input_metadata[name]
if (
(dtype is not None and not DataType.from_dtype(dtype).is_integral)
or util.is_shape_dynamic(shape)
or len(shape) != 1
):
return False
user_shape = self.user_input_metadata[name].shape
# Shape of shape cannot be dynamic.
return not util.is_shape_dynamic(user_shape) and len(user_shape) == shape[0]
def generate_buffer(name, dtype, shape):
if is_shape_tensor(name, dtype):
buffer = array_sampler.constant_array(shape, dtype)
G_LOGGER.info(
f"Assuming {name} is a shape tensor. Setting input values to: {buffer}. "
"If these values are not correct, please set it correctly in 'input_metadata' or by providing --input-shapes",
mode=LogMode.ONCE,
)
elif dtype is not None and (
DataType.from_dtype(dtype).is_integral
or DataType.from_dtype(dtype) == DataType.BOOL
):
imin, imax = self._get_range(
name,
cast_type=int if DataType.from_dtype(dtype).is_integral else bool,
)
G_LOGGER.verbose(
f"Input tensor: {name} | Generating input data in range: [{imin}, {imax}]",
mode=LogMode.ONCE,
)
# high is 1 greater than the max int drawn.
buffer = array_sampler.sample_integer(shape, dtype, imin, imax)
else:
fmin, fmax = self._get_range(name, cast_type=float)
G_LOGGER.verbose(
f"Input tensor: {name} | Generating input data in range: [{fmin}, {fmax}]",
mode=LogMode.ONCE,
)
buffer = array_sampler.sample_float(shape, dtype, fmin, fmax)
return buffer
if self.input_metadata is None and self.user_input_metadata is not None:
self.input_metadata = self.user_input_metadata
buffers = OrderedDict()
# Expand wildcards to inputs in user_input_metadata
inp_to_user_inp, unmatched_user_inps = util.match_keys(list(self.user_input_metadata), list(self.input_metadata))
# Warn about unused metadata
if unmatched_user_inps:
for name in unmatched_user_inps:
if util.contains_wildcard(name):
msg = f"Input tensor wildcard: {name} | Metadata was provided, but none of the matched inputs exist in one or more runners."
G_LOGGER.warning(msg)
else:
msg = f"Input tensor: {name} | Metadata was provided, but the input does not exist in one or more runners."
close_match = util.find_str_in_iterable(
name, self.input_metadata.keys()
)
if close_match:
msg += f"\nMaybe you meant to set: {close_match}?"
G_LOGGER.warning(msg)
for name, (dtype, shape) in self.input_metadata.items():
try:
dtype = (
DataType.to_dtype(
DataType.from_dtype(dtype), self.data_loader_backend_module
)
if dtype is not None
else None
)
except DataTypeConversionException:
G_LOGGER.critical(
f"Could not convert data type: {dtype} to {self.data_loader_backend_module}, so the default data loader cannot generate a {self.data_loader_backend_module} array for input: {name}. "
f"Please use a custom data loader to provide inputs. "
)
if name in inp_to_user_inp:
user_dtype, user_shape = self.user_input_metadata[inp_to_user_inp[name]]
dtype = util.default(user_dtype, dtype)
dtype = (
DataType.to_dtype(
DataType.from_dtype(dtype), self.data_loader_backend_module
)
if dtype is not None
else None
)
is_valid_shape_override = (
user_shape is not None
and util.is_valid_shape_override(user_shape, shape)
)
if util.is_shape_dynamic(user_shape):
G_LOGGER.warning(
f"Input tensor: {name} [shape={shape}] | Provided input shape: {user_shape} is dynamic.\nDynamic shapes cannot be used to generate inference data. Will use default shape instead.\nTo avoid this, please provide a fixed shape to the data loader. "
)
elif not is_valid_shape_override and not is_shape_tensor(name, dtype):
G_LOGGER.warning(
f"Input tensor: {name} [shape={shape}] | Cannot use provided custom shape: {user_shape} to override tensor shape. Will use default shape instead.",
mode=LogMode.ONCE,
)
else:
shape = util.default(user_shape, shape)
static_shape = get_static_shape(name, shape)
buffers[name] = generate_buffer(name, dtype, shape=static_shape)
# Warn about unused val_range
if not isinstance(self.val_range, tuple):
util.check_sequence_contains(
self.val_range.keys(),
list(self.input_metadata.keys()) + [""],
name="val_range",
log_func=G_LOGGER.warning,
check_missing=False,
)
return buffers
# Caches data loaded by a DataLoader for use across multiple runners.
class DataLoaderCache:
def __init__(self, data_loader, save_inputs_path=None):
self.data_loader = data_loader
self.cache = [] # List[OrderedDict[str, numpy.ndarray]]
self.save_inputs_path = save_inputs_path
@func.constantmethod
def __len__(self):
return len(self.cache)
@func.constantmethod
def __getitem__(self, iteration):
"""
Load the specified iteration from the cache if present, or load it from the data loader.
Args:
iteration (int): The iteration whose data to retrieve.
"""
if iteration >= len(self.cache):
raise IndexError()
# Attempts to match existing input buffers to the requested input_metadata
def coerce_cached_input(index, name, dtype, shape):
cached_feed_dict = self.cache[iteration]
cached_name = util.find_str_in_iterable(
name, cached_feed_dict.keys(), index
)
if cached_name is None:
G_LOGGER.critical(
f"Input tensor: {name} | Does not exist in the data loader cache."
)
if cached_name != name:
G_LOGGER.warning(
f"Input tensor: {name} | Buffer name ({cached_name}) does not match expected input name ({name})."
)
buffer = cached_feed_dict[cached_name]
if dtype != util.array.dtype(buffer):
G_LOGGER.warning(
f"Input tensor: {name} | Buffer dtype ({util.array.dtype(buffer)}) does not match expected input dtype ({dtype}), attempting to cast. "
)
try:
np_type = DataType.to_dtype(dtype, "numpy")
except:
pass
else:
type_info = None
if dtype.is_integral:
type_info = np.iinfo(np_type)
elif dtype.is_floating:
type_info = np.finfo(np_type)
if type_info is not None and util.array.any(
(buffer < type_info.min) | (buffer > type_info.max)
):
G_LOGGER.warning(
f"Some values in this input are out of range of {dtype}. Unexpected behavior may ensue!"
)
buffer = util.array.cast(buffer, dtype)
if not util.is_valid_shape_override(util.array.shape(buffer), shape):
G_LOGGER.warning(
f"Input tensor: {name} | Buffer shape ({util.array.shape(buffer)}) does not match expected input shape ({shape}). "
f"Attempting to transpose/reshape. "
)
buffer = util.try_match_shape(buffer, shape)
if util.array.dtype(buffer) != dtype or not util.is_valid_shape_override(
util.array.shape(buffer), shape
):
G_LOGGER.critical(
f"Input tensor: {name} | Cannot reuse input data due to mismatch in shape or data type.\n"
f"Note: Cached input: [dtype={util.array.dtype(buffer)}, shape={util.array.shape(buffer)}], "
f"Requested input: [dtype={dtype}, shape={shape}]"
)
return buffer
feed_dict = OrderedDict()
# Reload from data loader if needed
data_loader_feed_dict = None
for index, (name, (dtype, shape)) in enumerate(self.input_metadata.items()):
try:
buffer = coerce_cached_input(
index, name, DataType.from_dtype(dtype), shape
)
except PolygraphyException:
G_LOGGER.warning(
f"Could not use buffer previously cached from data loader for input: {name}. Attempting to reload inputs from the data loader.\nNote that this will only work if the data loader supports random access.\nPlease refer to warnings above for details on why the previously generated input buffer didn't work. "
)
try:
if data_loader_feed_dict is None:
data_loader_feed_dict = self.data_loader[iteration]
buffer = data_loader_feed_dict[name]
except:
G_LOGGER.critical(
"Could not reload inputs from data loader. Are the runners running the same model? "
"If not, please rewrite the data loader to support random access."
)
feed_dict[name] = buffer
return feed_dict
def set_input_metadata(self, input_metadata):
"""
Set the input metadata for the data loader.
Args:
input_metadata (TensorMetadata):
Input Metadata, including shape and type information. The cache may attempt to transform inputs to
match the specified input_metadata when data already in the cache does not exactly match.
"""
self.input_metadata = input_metadata
with contextlib.suppress(AttributeError):
self.data_loader.input_metadata = input_metadata
if not self.cache:
G_LOGGER.verbose("Loading inputs from data loader")
self.cache = list(self.data_loader)
def _is_feed_dict(inp):
if isinstance(inp, RunResults):
return False
try:
for name, _ in inp.items():
if not isinstance(name, str):
return False
except:
return False
else:
return True
if not self.cache:
G_LOGGER.warning("Data loader did not yield any input data.")
elif not _is_feed_dict(self.cache[0]):
G_LOGGER.critical(
f"Data loader returned an object that cannot be recognized as a feed_dict (Dict[str, Union[np.ndarray, torch.Tensor, DeviceView]]):"
f"\nNote: The object was:\n{self.cache[0]}.\n"
f"\nHint: If this is a `RunReults` object (e.g. generated with `--save-outputs`), try using the "
f"`data to-input` tool to convert it to a feed_dict compatible format. "
)
# Only save inputs the first time the cache is generated
if self.save_inputs_path is not None:
save_json(self.cache, self.save_inputs_path, "inference input data")
tools/Polygraphy/polygraphy/comparator/postprocess.py
+64
View File
@@ -0,0 +1,64 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from polygraphy import mod, util
np = mod.lazy_import("numpy")
@mod.export()
class PostprocessFunc:
"""
Provides functions that can apply post-processing to `IterationResult` s.
"""
@staticmethod
# This function returns a top_k function that can be used as a postprocess_func.
def top_k(k=None):
"""
Creates a function that applies a Top-K operation to a IterationResult.
Top-K will return the indices of the k largest values in the array.
Args:
k (Union[int, Tuple[int, int], Dict[str, int], Dict[str, Tuple[int, int]]]):
The number of indices to keep and optionally the axis on which to operate.
For example, a value of ``(5, 0)`` would keep the top 5 indices along axis 0.
If this exceeds the axis length, it will be clamped.
This can be specified on a per-output basis by providing a dictionary. In that case,
use an empty string ("") as the key to specify default top-k value for outputs not explicitly listed.
If no default is present, unspecified outputs will not be modified.
Defaults to 10.
Returns:
Callable(IterationResult) -> IterationResult: The top-k function.
"""
k = util.default(k, 10)
axis = -1
# Top-K implementation.
def top_k_impl(iter_result):
for name, output in iter_result.items():
k_val = util.value_or_from_dict(k, name)
if k_val:
nonlocal axis
if util.is_sequence(k_val):
k_val, axis = k_val
iter_result[name] = util.array.topk(output, k_val, axis)[1]
return iter_result
return top_k_impl
tools/Polygraphy/polygraphy/comparator/struct.py
+409
View File
@@ -0,0 +1,409 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from collections import OrderedDict
from polygraphy import config, mod, util
from polygraphy.common.interface import TypedDict, TypedList
from polygraphy.json import Decoder, Encoder, add_json_methods, load_json, save_json
from polygraphy.logger import G_LOGGER
class LazyArray:
"""
Represents a lazily loaded NumPy array or PyTorch Tensor.
For example, large arrays may be serialized to temporary files on the disk
to save memory.
"""
def __init__(self, arr):
"""
Args:
arr (Union[np.ndarray, torch.Tensor]): The array.
"""
self.arr = None
self.tmpfile = None
if config.ARRAY_SWAP_THRESHOLD_MB >= 0 and util.array.nbytes(arr) > (
config.ARRAY_SWAP_THRESHOLD_MB << 20
):
self.tmpfile = util.NamedTemporaryFile(suffix=".json")
G_LOGGER.extra_verbose(
f"Evicting large array ({util.array.nbytes(arr) / 1024.0 ** 2:.3f} MiB) from memory and saving to {self.tmpfile.name}"
)
save_json(arr, self.tmpfile.name)
else:
self.arr = arr
def load(self):
"""
Load the array, deserializing from the disk if it was stored earlier.
Returns:
Union[np.ndarray, torch.Tensor]: The array
"""
if self.arr is not None:
return self.arr
if self.tmpfile is None:
G_LOGGER.internal_error(
f"self.arr is None but self.tmpfile is also None; this should be impossible."
)
return load_json(self.tmpfile.name)
@Encoder.register(LazyArray, alias="LazyNumpyArray")
def encode(lazy_arr):
return {
"values": lazy_arr.load(),
}
@Decoder.register(LazyArray, alias="LazyNumpyArray")
def decode(dct):
return LazyArray(dct["values"])
@mod.export()
class IterationResult(TypedDict(lambda: str, lambda: LazyArray)):
"""
An ordered dictionary containing the result of a running a single iteration of a runner.
This maps output names to arrays, and preserves the output ordering from the runner.
NOTE: The ``POLYGRAPHY_ARRAY_SWAP_THRESHOLD_MB`` environment variable can be set to enable
the arrays to be swapped to the disk.
Also includes additional fields indicating the name of the runner which produced the
outputs, and the time required to do so.
"""
@staticmethod
def _to_lazy(nparray):
if isinstance(nparray, LazyArray):
return nparray
return LazyArray(nparray)
@staticmethod
def _to_lazy_dict(nparray_dict):
if nparray_dict is None:
return None
# Converts a Dict[str, np.ndarray] to a Dict[str, LazyArray]
lazy = OrderedDict()
for name, out in nparray_dict.items():
lazy[name] = IterationResult._to_lazy(out)
return lazy
def __init__(self, outputs=None, runtime=None, runner_name=None):
"""
Args:
outputs (Dict[str, Union[np.array, torch.Tensor]]): The outputs of this iteration, mapped to their names.
runtime (float):
The time required for this iteration, in seconds.
Only used for logging purposes.
runner_name (str):
The name of the runner that produced this output.
If this is omitted, a default name is generated.
"""
if outputs and config.ARRAY_SWAP_THRESHOLD_MB < 0:
total_size_gb = sum(
util.array.nbytes(arr)
for arr in outputs.values()
if util.array.is_torch(arr) or util.array.is_numpy(arr)
) / (1024.0**3)
if total_size_gb >= 1:
G_LOGGER.warning(
f"It looks like the outputs of this network are very large ({total_size_gb:.3f} GiB).\n"
"To reduce memory usage, you may want to allow Polygraphy to swap these arrays to the disk "
"using the POLYGRAPHY_ARRAY_SWAP_THRESHOLD_MB environment variable."
)
super().__init__(IterationResult._to_lazy_dict(outputs))
self.runtime = runtime
self.runner_name = util.default(runner_name, "custom_runner")
# Convenience methods to preserve np.ndarray in the interface.
def update(self, other):
return super().update(IterationResult._to_lazy_dict(other))
def __setitem__(self, name, arr):
return super().__setitem__(name, IterationResult._to_lazy(arr))
def values(self):
for arr in super().values():
yield arr.load()
def items(self):
for name, arr in super().items():
yield name, arr.load()
def __getitem__(self, name):
return super().__getitem__(name).load()
def __eq__(self, other):
if self.runtime != other.runtime or self.runner_name != other.runner_name:
return False
for key, val in self.items():
if key not in other:
return False
if not util.array.equal(val, other[key]):
return False
return True
@Encoder.register(IterationResult)
def encode(iter_result):
return {
"outputs": iter_result.dct,
"runtime": iter_result.runtime,
"runner_name": iter_result.runner_name,
}
@Decoder.register(IterationResult)
def decode(dct):
return IterationResult(
outputs=dct["outputs"], runtime=dct["runtime"], runner_name=dct["runner_name"]
)
@mod.export()
@add_json_methods("inference results")
class RunResults(TypedList(lambda: tuple)):
"""
Maps runners to per-iteration outputs (in the form of a ``List[IterationResult]``).
For example, if ``results`` is an instance of ``RunResults()``, then
to access the outputs of the first iteration from a specified runner, do:
::
iteration = 0
runner_name = "trt-runner"
outputs = results[runner_name][iteration]
# `outputs` is a `Dict[str, np.ndarray]`
Note: Technically, this is a ``List[Tuple[str, List[IterationResult]]]``, but includes
helpers that make it behave like an OrderedDict that can contain duplicates.
"""
def items(self):
"""
Creates a generator that yields ``Tuple[str, List[IterationResult]]`` - runner names
and corresponding outputs.
"""
for name, iteration_results in self.lst:
yield name, iteration_results
def keys(self):
"""
Creates a generator that yields runner names (str).
"""
for name, _ in self.lst:
yield name
def values(self):
"""
Creates a generator that yields runner outputs (List[IterationResult]).
"""
for _, iteration_results in self.lst:
yield iteration_results
def update(self, other):
"""
Updates the results stored in this instance.
Args:
other (Union[Dict[str, List[IterationResult]], RunResults]):
A dictionary or RunResults instance from which to update this one.
"""
for name, iteration_results in other.items():
self.lst[name] = iteration_results
return self
def add(self, out_list, runtime=None, runner_name=None):
"""
A helper to create a ``List[IterationResult]`` and map it to the specified runner_name.
This method cannot be used to modify an existing entry.
Calling this method is equivalent to:
::
results[runner_name] = []
for out in out_list:
results[runner_name].append(IterationResult(out, runtime, runner_name))
Args:
out_list (List[Dict[str, np.array]]):
One or more set of outputs where each output is a dictionary
of output names mapped to NumPy arrays.
runtime (float):
The time required for this iteration, in seconds.
Only used for logging purposes.
runner_name (str):
The name of the runner that produced this output.
If this is omitted, a default name is generated.
"""
runner_name = util.default(runner_name, "custom_runner")
iter_results = [IterationResult(out, runtime, runner_name) for out in out_list]
self[runner_name] = iter_results
def __getitem__(self, key):
if isinstance(key, int):
return self.lst[key]
for name, iteration_results in self.lst:
if name == key:
return iteration_results
G_LOGGER.critical(
f"{key:35} does not exist in this RunResults instance. Note: Available runners: {list(self.keys())}"
)
def __setitem__(self, key, value):
if isinstance(key, int):
self.lst[key] = value
return
for index, name in enumerate(self.keys()):
if name == key:
self.lst[index] = (key, value)
break
else:
self.append((key, value))
def __contains__(self, val):
if isinstance(val, str) or isinstance(val, bytes):
return val in list(self.keys())
return val in self.lst
def __eq__(self, other):
for (r0, its0), (r1, its1) in zip(self.lst, other.lst):
if r0 != r1:
return False
if its0 != its1:
return False
return True
@Encoder.register(RunResults)
def encode(results):
return {"lst": results.lst}
@Decoder.register(RunResults)
def decode(dct):
return RunResults(list(map(tuple, dct["lst"])))
@mod.export()
class AccuracyResult(TypedDict(lambda: tuple, lambda: list)):
"""
An ordered dictionary including details about the result of ``Comparator.compare_accuracy``.
More specifically, it is an ``OrderedDict[Tuple[str, str], List[OrderedDict[str, bool]]]`` which maps a runner
pair (a tuple containing both runner names) to a list of dictionaries of booleans (or anything that can be
converted into a boolean, such as an ``OutputCompareResult``), indicating whether there was a match in the outputs of
the corresponding iteration. The ``List[OrderedDict[str, bool]]`` is constructed from the dictionaries returned
by ``compare_func`` in ``compare_accuracy``.
For example, to see if there's a match between ``runner0`` and
``runner1`` during the 1st iteration for an output called ``output0``:
::
runner_pair = ("runner0", "runner1")
iteration = 0
output_name = "output0"
match = bool(accuracy_result[runner_pair][iteration][output_name])
If there's a mismatch, you can inspect the outputs from
the results of ``Comparator.run()``, assumed here to be called ``run_results``:
::
runner0_output = run_results["runner0"][iteration][output_name]
runner1_output = run_results["runner1"][iteration][output_name]
"""
def __bool__(self):
"""
Whether all outputs matched for every iteration.
You can use this function to avoid manually checking each output. For example:
::
if accuracy_result:
print("All matched!")
Returns:
bool
"""
return all(
[
bool(match)
for outs in self.values()
for out in outs
for match in out.values()
]
)
def _get_runner_pair(self, runner_pair):
return util.default(runner_pair, list(self.keys())[0])
def percentage(self, runner_pair=None):
"""
Returns the percentage of iterations that matched for the given pair of runners,
expressed as a decimal between 0.0 and 1.0.
Always returns 1.0 when the number of iterations is 0, or when there are no runner comparisons.
Args:
runner_pair (Tuple[str, str]):
A pair of runner names describing which runners to check.
Defaults to the first pair in the dictionary.
"""
if not list(self.keys()):
return 1.0 # No data in this result.
matched, _, total = self.stats(runner_pair)
if not total:
return 1.0 # No iterations
return float(matched) / float(total)
def stats(self, runner_pair=None):
"""
Returns the number of iterations that matched, mismatched, and the total number of iterations.
Args:
runner_pair (Tuple[str, str]):
A pair of runner names describing which runners to check.
Defaults to the first pair in the dictionary.
Returns:
Tuple[int, int, int]: Number of iterations that matched, mismatched, and total respectively.
"""
runner_pair = self._get_runner_pair(runner_pair)
outs = self[runner_pair]
matched = sum([all([match for match in out.values()]) for out in outs])
total = len(outs)
return matched, total - matched, total
tools/Polygraphy/polygraphy/comparator/util.py
+405
View File
@@ -0,0 +1,405 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import functools
import math
import os
from polygraphy import config, mod, util
from polygraphy.logger import G_LOGGER
from polygraphy.datatype import DataType
import math
import os
np = mod.lazy_import("numpy")
plt = mod.lazy_import("matplotlib.pyplot")
matplotlib = mod.lazy_import("matplotlib")
def cast_up(buffer):
dtype = util.array.dtype(buffer)
if dtype == DataType.FLOAT16:
buffer = util.array.cast(buffer, DataType.FLOAT32)
elif dtype in [DataType.INT8, DataType.UINT8, DataType.INT16, DataType.UINT16]:
buffer = util.array.cast(buffer, DataType.INT32)
elif dtype == DataType.UINT32:
buffer = util.array.cast(buffer, DataType.INT64)
return buffer
def use_higher_precision(func):
"""
Decorator that will cast the input numpy buffer(s) to a higher precision before computation.
"""
@functools.wraps(func)
def wrapped(*buffers):
if any(util.is_empty_shape(util.array.shape(buffer)) for buffer in buffers):
return 0
new_buffers = [cast_up(buffer) for buffer in buffers]
return func(*new_buffers)
return wrapped
@use_higher_precision
def compute_max(buffer):
return util.array.max(buffer)
# Returns index of max value
@use_higher_precision
def compute_argmax(buffer):
return util.array.unravel_index(util.array.argmax(buffer), util.array.shape(buffer))
@use_higher_precision
def compute_min(buffer):
return util.array.min(buffer)
# Returns index of min value
@use_higher_precision
def compute_argmin(buffer):
return util.array.unravel_index(util.array.argmin(buffer), util.array.shape(buffer))
def compute_mean(buffer):
return util.array.mean(buffer, dtype=DataType.FLOAT32)
@use_higher_precision
def compute_std(buffer):
return util.array.std(buffer)
@use_higher_precision
def compute_variance(buffer):
return util.array.var(buffer)
@use_higher_precision
def compute_median(buffer):
return util.array.median(buffer)
def compute_quantile(buffer, q):
return util.array.quantile(buffer, q)
def compute_average_magnitude(buffer):
return util.array.mean(util.array.abs(buffer), dtype=DataType.FLOAT32)
def str_histogram(output, hist_range=None):
if util.array.dtype(output) == DataType.BOOL:
return ""
try:
try:
hist, bin_edges = util.array.histogram(output, range=hist_range)
except (ValueError, RuntimeError) as err:
G_LOGGER.verbose(f"Could not generate histogram. Note: Error was: {err}")
return ""
max_num_elems = compute_max(hist)
if not max_num_elems: # Empty tensor
return
bin_edges = [f"{bin:.3g}" for bin in bin_edges]
max_start_bin_width = max(len(bin) for bin in bin_edges)
max_end_bin_width = max(len(bin) for bin in bin_edges[1:])
MAX_WIDTH = 40
ret = "---- Histogram ----\n"
ret += f"{'Bin Range':{max_start_bin_width + max_end_bin_width + 5}}| Num Elems | Visualization\n"
for num, bin_start, bin_end in zip(hist, bin_edges, bin_edges[1:]):
bar = "#" * int(MAX_WIDTH * float(num) / float(max_num_elems))
ret += f"({bin_start:<{max_start_bin_width}}, {bin_end:<{max_end_bin_width}}) | {num:10} | {bar}\n"
return ret
except Exception as err:
G_LOGGER.verbose(f"Could not generate histogram.\nNote: Error was: {err}")
if config.INTERNAL_CORRECTNESS_CHECKS:
raise
return ""
def str_output_stats(output, runner_name=None):
ret = ""
if runner_name:
ret += f"{runner_name} | Stats: "
try:
ret += f"mean={compute_mean(output):.5g}, std-dev={compute_std(output):.5g}, var={compute_variance(output):.5g}, median={compute_median(output):.5g}, min={compute_min(output):.5g} at {compute_argmin(output)}, max={compute_max(output):.5g} at {compute_argmax(output)}, avg-magnitude={compute_average_magnitude(output):.5g}"
# np.quantile doesn't work with boolean input, so we don't show quantile error if the output type is boolean
if output.dtype == bool:
ret += "\n"
else:
ret += f", p90={compute_quantile(output, 0.9):.5g}, p95={compute_quantile(output, 0.95):.5g}, p99={compute_quantile(output, 0.99):.5g}\n"
except Exception as err:
G_LOGGER.verbose(f"Could not generate statistics.\nNote: Error was: {err}")
ret += "<Error while computing statistics>"
if config.INTERNAL_CORRECTNESS_CHECKS:
raise
return ret
def log_output_stats(output, info_hist=False, runner_name=None, hist_range=None):
ret = str_output_stats(output, runner_name)
G_LOGGER.info(ret)
with G_LOGGER.indent():
# For small outputs, show the entire output instead of just a histogram.
SMALL_OUTPUT_THRESHOLD = 100
if util.array.size(output) <= SMALL_OUTPUT_THRESHOLD:
G_LOGGER.log(
lambda: f"---- Values ----\n{util.indent_block(output)}",
severity=G_LOGGER.INFO if info_hist else G_LOGGER.VERBOSE,
)
G_LOGGER.log(
lambda: str_histogram(output, hist_range),
severity=G_LOGGER.INFO if info_hist else G_LOGGER.VERBOSE,
)
def build_heatmaps(
arr, min_val, max_val, prefix, save_dir=None, show=None, use_lognorm=None
):
"""
Display an array as an image or set of images. The last two dimensions are interpreted as
the height and width and the leading dimensions are flattened and treated as the number
of images to display.
Args:
arr (Union[torch.Tensor, numpy.ndarray]): The input array or tensor.
min_val (float): The minimum value in the input array
max_val (float): The maximum value in the input array
prefix (str): The prefix to use when displaying titles for figures.
save_dir (Optional[str]): Path to a directory in which to save images of the heatmaps.
show (Optional[bool]): Whether to display the heatmap.
use_lognorm (bool): Whether to use LogNorm instead of Normalize when displaying values.
"""
G_LOGGER.start(f"Building heatmaps for {prefix}. This may take a while...")
with G_LOGGER.indent():
MAX_HEIGHT = 1080
MAX_WIDTH = 1920
MAX_NUM_ROWS = 14
MAX_NUM_COLS = 7
FONT_SIZE = "xx-small"
shape = util.array.shape(arr)
if len(shape) < 3:
arr = util.array.view(
arr,
dtype=util.array.dtype(arr),
shape=([1] * (3 - len(shape))) + list(shape),
)
original_shape = util.array.shape(arr)
arr = util.array.view(
arr,
dtype=util.array.dtype(arr),
shape=(-1, original_shape[-2], original_shape[-1]),
)
shape = util.array.shape(arr)
num_images = shape[0]
def coord_str_from_img_idx(img_idx):
coord = []
for dim in reversed(original_shape[:-2]):
coord.insert(0, img_idx % dim)
img_idx //= dim
return f"({','.join(map(str, coord))},0:{shape[-2]},0:{shape[-1]})"
# We treat each 2D slice of the array as a separate image.
# Multiple images may be displayed on a single figure (in a grid) and we may have multiple figures.
num_rows = min(MAX_HEIGHT // shape[-2], MAX_NUM_ROWS)
num_cols = min(MAX_WIDTH // shape[-1], MAX_NUM_COLS)
# Remove any excess images per figure
if num_images < num_rows * num_cols:
num_cols = min(num_images, num_cols)
num_rows = math.ceil(num_images / num_cols)
num_images_per_figure = num_rows * num_cols
num_figures = math.ceil(num_images / num_images_per_figure)
# Populate each image in each figure.
for fig_idx in range(num_figures):
fig, axs = plt.subplots(
num_rows, num_cols, squeeze=False, dpi=200, constrained_layout=True
)
base_img_idx = fig_idx * num_images_per_figure
try:
# When the error is all the same, we can't use LogNorm.
if use_lognorm and min_val != max_val:
norm = matplotlib.colors.LogNorm(vmin=min_val, vmax=max_val)
prefix += " (Log Scale)"
else:
norm = matplotlib.colors.Normalize(vmin=min_val, vmax=max_val)
fig_title = f"{prefix}: {coord_str_from_img_idx(base_img_idx)} to {coord_str_from_img_idx(min(base_img_idx + num_images_per_figure, num_images) - 1)}"
fig.suptitle(fig_title, fontsize=FONT_SIZE)
G_LOGGER.extra_verbose(f"Building heatmaps for {fig_title}")
images = []
for row in range(num_rows):
for col in range(num_cols):
img_idx = base_img_idx + (col + row * num_cols)
ax = axs[row, col]
ax.set_axis_off()
if img_idx < shape[0]:
img = arr[img_idx]
title = f"{coord_str_from_img_idx(img_idx)}"
else:
img = np.zeros(shape=(shape[-2:]))
title = "Out Of Bounds"
ax.set_title(title, fontsize=FONT_SIZE)
images.append(
ax.imshow(
img, cmap="plasma", filternorm=False, resample=False
)
)
for im in images:
im.set_norm(norm)
fig.colorbar(images[0], ax=axs, shrink=0.7)
if save_dir is not None:
path = os.path.join(
save_dir, f"{util.sanitize_filename(fig_title)}.svg"
)
util.makedirs(path)
G_LOGGER.info(f"Saving '{prefix}' heatmap to: '{path}'")
fig.savefig(path)
if show:
plt.show()
finally:
plt.close(fig)
def scatter_plot_error_magnitude(
absdiff,
reldiff,
reference_output,
min_reldiff,
max_reldiff,
runner0_name,
runner1_name,
out0_name,
out1_name,
save_dir=None,
show=None,
):
"""
Display a plot of absolute/relative difference against the magnitude of the output.
Args:
absdiff (Union[torch.Tensor, numpy.ndarray]): The absolute difference.
reldiff (Union[torch.Tensor, numpy.ndarray]): The relative difference.
reference_output (Union[torch.Tensor, numpy.ndarray]): The output to consider as the reference output.
min_reldiff (float): The minimum relative difference
max_reldiff (float): The maximum relative difference
runner0_name (str): The name of the first runner.
runner1_name (str): The name of the second runner.
out0_name (str): The name of the output of the first runner.
out1_name (str): The name of the output of the second runner.
save_dir (Optional[str]): Path to a directory in which to save images of the plots.
show (Optional[bool]): Whether to display the error metrics plot.
"""
G_LOGGER.start(
f"Building error metrics plot for {out0_name}. This may take a while..."
)
with G_LOGGER.indent():
title = f"Error metrics between output0 and output1\noutput0: {runner0_name:35} | {out0_name}\noutput1: {runner1_name:35} | {out1_name}"
fname = util.sanitize_filename(f"error_metrics_{out0_name}.png")
TICK_FONT_SIZE = 6
TITLE_FONT_SIZE = 7
NUM_X_TICKS = 20
NUM_Y_LINEAR_TICKS = 10
def set_ax_properties(ax):
ax.tick_params(axis="x", labelrotation=90)
ax.tick_params(axis="both", labelsize=TICK_FONT_SIZE)
ax.grid(linestyle="--")
ax.xaxis.label.set_fontsize(TITLE_FONT_SIZE)
ax.yaxis.label.set_fontsize(TITLE_FONT_SIZE)
def set_linear_ax(ax):
xticks = ax.get_xticks()
yticks = ax.get_yticks()
ax.set_xticks(np.linspace(0, xticks[-1], NUM_X_TICKS))
ax.set_yticks(np.linspace(0, yticks[-1], NUM_Y_LINEAR_TICKS))
set_ax_properties(ax)
def set_log_ax(ax, min_diff, max_diff):
ax.set_yscale("log")
xticks = ax.get_xticks()
# Add a very small epsilon to prevent division by 0:
eps = 1e-15
yrange = np.log10(np.array([min_diff + eps, max_diff + eps]))
yrange[0] = math.floor(yrange[0])
yrange[1] = math.ceil(yrange[1])
ax.set_xticks(np.linspace(0, xticks[-1], NUM_X_TICKS))
ax.set_yticks(np.power(10, np.arange(yrange[0], yrange[1], 1)))
set_ax_properties(ax)
magnitude = util.array.abs(reference_output)
fig, axs = plt.subplots(2, sharex=True, constrained_layout=True)
try:
fig.suptitle(title, fontsize=TITLE_FONT_SIZE)
axs[0].scatter(magnitude, absdiff, s=1)
axs[0].set(ylabel="Absolute error")
set_linear_ax(axs[0])
axs[1].scatter(magnitude, reldiff, s=1)
label_suffix = ""
# When the range of the data is 0, we can't use log scale.
if min_reldiff != max_reldiff:
set_log_ax(axs[1], min_reldiff, max_reldiff)
label_suffix = " (log scale)"
else:
set_linear_ax(axs[1])
axs[1].set(
xlabel="output1 magnitude", ylabel=f"Relative error{label_suffix}"
)
if save_dir is not None:
path = os.path.join(save_dir, fname)
util.makedirs(path)
G_LOGGER.info(f"Saving error metrics plot to: '{path}'")
fig.savefig(path, dpi=1200, bbox_inches="tight")
if show:
plt.show()
finally:
plt.close(fig)