wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/tools/benchmark/BUILD
+101
View File
@@ -0,0 +1,101 @@
# Description:
# Benchmark utility that can run on desktop and Android.
load("@rules_cc//cc:cc_library.bzl", "cc_library")
load(
"//tensorflow:tensorflow.bzl",
"tf_cc_binary",
"tf_cc_test",
"tf_copts",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//visibility:public"],
licenses = ["notice"],
)
cc_library(
name = "benchmark_model_lib",
testonly = 1,
srcs = [
"benchmark_model.cc",
],
hdrs = [
"benchmark_model.h",
],
copts = tf_copts(),
visibility = ["//visibility:public"],
deps = select({
"//tensorflow:android": [
"//tensorflow/core:portable_tensorflow_lib",
"//tensorflow/core:portable_tensorflow_test_lib",
],
"//conditions:default": [
"//tensorflow/core:core_cpu",
"//tensorflow/core:framework",
"//tensorflow/core:framework_internal",
"//tensorflow/core:framework_lite",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core:tensorflow",
"//tensorflow/core:test",
],
}) + [
"//tensorflow/core/platform:numbers",
"//tensorflow/core/util:stats_calculator_portable",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@xla//xla/tsl/platform:status",
],
)
tf_cc_test(
name = "benchmark_model_test",
size = "medium",
srcs = ["benchmark_model_test.cc"],
deps = [
":benchmark_model_lib",
"//tensorflow/cc:cc_ops",
"//tensorflow/cc:scope",
"//tensorflow/core:core_cpu",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
],
)
# This binary may be built for either desktop or Android.
# A typical Android build command will look like the following:
# bazel build tensorflow/core:portable_tensorflow_lib \
# --crosstool_top=//external:android/crosstool \
# --cpu=armeabi-v7a \
# --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
# --config monolithic
tf_cc_binary(
name = "benchmark_model",
testonly = 1,
srcs = ["benchmark_model_main.cc"],
copts = tf_copts(),
linkopts = select({
"//tensorflow:android": [
"-pie",
"-s",
"-landroid",
"-latomic",
"-ljnigraphics",
"-llog",
"-lm",
"-z defs",
"-s",
"-Wl,--exclude-libs,ALL", # Exclude syms in all libs from auto export
],
"//conditions:default": [],
}),
linkstatic = 1,
visibility = ["//visibility:public"],
deps = [":benchmark_model_lib"],
)
tensorflow/tools/benchmark/README.md
+109
View File
@@ -0,0 +1,109 @@
# TensorFlow Model Benchmark Tool
## Description
A simple C++ binary to benchmark a compute graph and its individual operators,
both on desktop machines and on Android.
## To build/install/run
### On Android:
(0) Refer to https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android
to edit the `WORKSPACE` to configure the android NDK/SDK.
(1) build for your specific platform, e.g.:
```
bazel build -c opt \
--crosstool_top=//external:android/crosstool \
--cpu=armeabi-v7a \
--host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
--config monolithic \
tensorflow/tools/benchmark:benchmark_model
```
(2) Connect your phone. Push the binary to your phone with adb push
(make the directory if required):
```
adb push bazel-bin/tensorflow/tools/benchmark/benchmark_model /data/local/tmp
```
(3) Push the compute graph that you need to test. For example:
```
adb push tensorflow_inception_graph.pb /data/local/tmp
```
(4) Run the benchmark. For example:
```
adb shell /data/local/tmp/benchmark_model \
--graph=/data/local/tmp/tensorflow_inception_graph.pb \
--input_layer="input:0" \
--input_layer_shape="1,224,224,3" \
--input_layer_type="float" \
--output_layer="output:0"
```
### On desktop:
(1) build the binary
```
bazel build -c opt tensorflow/tools/benchmark:benchmark_model
```
(2) Run on your compute graph, similar to the Android case but without the need
of adb shell. For example:
```
bazel-bin/tensorflow/tools/benchmark/benchmark_model \
--graph=tensorflow_inception_graph.pb \
--input_layer="input:0" \
--input_layer_shape="1,224,224,3" \
--input_layer_type="float" \
--output_layer="output:0"
```
The Inception graph used as an example here may be downloaded from
https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip
## Model downloader
To download TF .pb graphs of several popular models, run:
```sh
bash download_models.sh
```
## Comparing performance with vanilla TF
We provide example scripts comparing TF-oneDNN performance with vanilla TF's
that users can modify for their own benchmarks. The scripts assume that models
are already downloaded by `download_models.sh`. To run end-to-end model
performance comparison between TF-oneDNN and vanilla TF, call
```sh
bash download_models.sh # Skip this step if models are already downloaded.
bash run_onednn_benchmarks.sh
```
The output is a summary table in a CSV file: results.csv. Example output:
```sh
Showing runtimes in microseconds. `?` means not available.
Model, Batch, Vanilla, oneDNN, Speedup
bert-large, 1, x, y, x/y
bert-large, 16, ..., ..., ...
bert-large, 64, ..., ..., ...
inception, 1, ..., ..., ...
inception, 16, ..., ..., ...
inception, 64, ..., ..., ...
⋮
ssd-resnet34, 1, ?, ..., ?
ssd-resnet34, 16, ?, ..., ?
ssd-resnet34, 64, ?, ..., ?
```
Vanilla TF can't run `ssd-resnet34` on CPU because it doesn't support NCHW
format.
tensorflow/tools/benchmark/benchmark_model.cc
+725
View File
@@ -0,0 +1,725 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// A C++ binary to benchmark a compute graph and its individual operators,
// both on desktop machines and on Android.
//
// See README.md for usage instructions.
#include "tensorflow/tools/benchmark/benchmark_model.h"
#include <cstdlib>
#include <memory>
#include <string>
#include <unordered_set>
#include <vector>
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "xla/tsl/platform/errors.h"
#include "xla/tsl/platform/status.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/node_def.pb.h"
#include "tensorflow/core/framework/numeric_types.h"
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/numbers.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/tstring.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/protobuf/config.pb.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/public/session_options.h"
#include "tensorflow/core/util/command_line_flags.h"
#include "tensorflow/core/util/reporter.h"
#include "tensorflow/core/util/stat_summarizer.h"
#include "tensorflow/core/util/stat_summarizer_options.h"
#include "tensorflow/core/util/stats_calculator.h"
namespace tensorflow {
namespace benchmark_model {
namespace {
absl::Status InitializeVariables(Session* session,
const std::vector<std::string>& init_ops) {
LOG(INFO) << "Initializing graph variables";
for (const std::string& init_op : init_ops) {
TF_RETURN_IF_ERROR(session->Run({}, {}, {init_op}, nullptr));
}
return absl::OkStatus();
}
template <class T>
void InitializeTensor(const std::vector<float>& initialization_values,
Tensor* input_tensor) {
auto type_tensor = input_tensor->flat<T>();
type_tensor = type_tensor.constant(0);
if (!initialization_values.empty()) {
for (int i = 0; i < initialization_values.size(); ++i) {
type_tensor(i) = static_cast<T>(initialization_values[i]);
}
}
}
void CreateTensorsFromInputInfo(
const std::vector<InputLayerInfo>& inputs,
std::vector<std::pair<std::string, tensorflow::Tensor> >* input_tensors) {
for (const InputLayerInfo& input : inputs) {
Tensor input_tensor(input.data_type, input.shape);
switch (input.data_type) {
case DT_INT32: {
InitializeTensor<int32_t>(input.initialization_values, &input_tensor);
break;
}
case DT_INT64: {
InitializeTensor<int64_t>(input.initialization_values, &input_tensor);
break;
}
case DT_FLOAT: {
InitializeTensor<float>(input.initialization_values, &input_tensor);
break;
}
case DT_QUINT8: {
InitializeTensor<quint8>(input.initialization_values, &input_tensor);
break;
}
case DT_UINT8: {
InitializeTensor<uint8_t>(input.initialization_values, &input_tensor);
break;
}
case DT_BOOL: {
InitializeTensor<bool>(input.initialization_values, &input_tensor);
break;
}
case DT_STRING: {
if (!input.initialization_values.empty()) {
LOG(FATAL) << "Initialization values are not supported for strings";
}
auto type_tensor = input_tensor.flat<tstring>();
type_tensor = type_tensor.constant("");
break;
}
default:
LOG(FATAL) << "Unsupported input type: "
<< DataTypeString(input.data_type);
}
input_tensors->push_back({input.name, input_tensor});
}
}
absl::Status GetOutputShapes(
const std::vector<InputLayerInfo>& inputs,
const std::set<std::string>& wanted_shapes, Session* session,
std::unordered_map<std::string, TensorShape>* node_shapes) {
std::vector<std::pair<std::string, tensorflow::Tensor> > input_tensors;
CreateTensorsFromInputInfo(inputs, &input_tensors);
std::vector<tensorflow::Tensor> output_tensors;
std::vector<std::string> output_tensor_names;
for (const std::string& wanted_shape : wanted_shapes) {
bool is_input = false;
for (const std::pair<std::string, tensorflow::Tensor>& input_tensor :
input_tensors) {
if (input_tensor.first == wanted_shape) {
(*node_shapes)[wanted_shape] = input_tensor.second.shape();
is_input = true;
break;
}
}
if (!is_input) {
output_tensor_names.push_back(wanted_shape);
}
}
TF_RETURN_IF_ERROR(
session->Run(input_tensors, output_tensor_names, {}, &output_tensors));
CHECK_EQ(output_tensors.size(), output_tensor_names.size());
for (int i = 0; i < output_tensor_names.size(); ++i) {
const std::string& wanted_shape_name = output_tensor_names[i];
const TensorShape& found_shape = output_tensors[i].shape();
(*node_shapes)[wanted_shape_name] = found_shape;
}
return absl::OkStatus();
}
absl::Status CalculateFlops(
const GraphDef& graph, const std::vector<InputLayerInfo>& inputs,
Session* session, int64_t* total_flops,
std::unordered_map<std::string, int64_t>* flops_by_op) {
std::unordered_set<std::string> floppable_ops = {
"Conv2D", "MatMul", "QuantizedConv2D", "QuantizedMatMul",
"DepthwiseConv2dNative"};
std::set<std::string> wanted_shapes;
for (const NodeDef& node : graph.node()) {
if (floppable_ops.count(node.op())) {
for (const std::string& input : node.input()) {
wanted_shapes.insert(input);
}
wanted_shapes.insert(node.name());
}
}
std::unordered_map<std::string, TensorShape> found_shapes;
TF_RETURN_IF_ERROR(
GetOutputShapes(inputs, wanted_shapes, session, &found_shapes));
*total_flops = 0;
for (const NodeDef& node : graph.node()) {
if (floppable_ops.count(node.op())) {
int64_t current_flops = 0;
// This is a very crude approximation to FLOPs that only looks at a few
// op types that commonly form the bulk of the computation for many
// models. It's included here because getting even an approximate value
// for FLOPs is still very useful for estimating utilization, versus a
// device's theoretical maximum FLOPs/second.
if ((node.op() == "Conv2D") || (node.op() == "QuantizedConv2D")) {
const TensorShape& filter_shape = found_shapes[node.input(1)];
const TensorShape& output_shape = found_shapes[node.name()];
int64_t filter_height = filter_shape.dim_size(0);
int64_t filter_width = filter_shape.dim_size(1);
int64_t filter_in_depth = filter_shape.dim_size(2);
int64_t output_count = output_shape.num_elements();
current_flops =
output_count * filter_in_depth * filter_height * filter_width * 2;
} else if ((node.op() == "MatMul") || (node.op() == "QuantizedMatMul")) {
const bool transpose_a = node.attr().at("transpose_a").b();
const TensorShape& a_shape = found_shapes[node.input(0)];
const TensorShape& output_shape = found_shapes[node.name()];
int64_t k;
if (transpose_a) {
k = a_shape.dim_size(0);
} else {
k = a_shape.dim_size(1);
}
int64_t output_count = output_shape.num_elements();
current_flops = k * output_count * 2;
} else if (node.op() == "DepthwiseConv2dNative") {
const TensorShape& filter_shape = found_shapes[node.input(1)];
const TensorShape& output_shape = found_shapes[node.name()];
int64_t filter_height = filter_shape.dim_size(0);
int64_t filter_width = filter_shape.dim_size(1);
int64_t output_count = output_shape.num_elements();
current_flops = output_count * filter_height * filter_width * 2;
}
(*flops_by_op)[node.op()] += current_flops;
*total_flops += current_flops;
}
}
return absl::OkStatus();
}
void RecordBenchmarkEntry(const std::string& output_prefix,
const std::string& benchmark_name,
const std::string& postfix, int num_runs,
double total_time_s, double throughput = -1.0) {
std::stringstream stream;
stream << benchmark_name;
if (!postfix.empty()) {
stream << "_" << postfix;
}
TestReporter node_reporter(output_prefix, stream.str());
TF_QCHECK_OK(node_reporter.Initialize());
TF_QCHECK_OK(
node_reporter.Benchmark(num_runs, -1.0, total_time_s, throughput));
TF_QCHECK_OK(node_reporter.Close());
}
void SleepSeconds(double sleep_seconds) {
if (sleep_seconds <= 0.0) {
return;
}
#ifdef PLATFORM_WINDOWS
Env::Default()->SleepForMicroseconds(sleep_seconds * 1000 * 1000);
#else
// Convert the inference_delay string into a timespec.
timespec req;
req.tv_sec = static_cast<time_t>(sleep_seconds);
req.tv_nsec = (sleep_seconds - req.tv_sec) * 1000000000;
nanosleep(&req, nullptr);
#endif
}
} // namespace
absl::Status InitializeSession(int num_threads, const std::string& graph,
std::unique_ptr<Session>* session,
std::unique_ptr<GraphDef>* graph_def) {
LOG(INFO) << "Loading TensorFlow.";
tensorflow::SessionOptions options;
tensorflow::ConfigProto& config = options.config;
if (num_threads > 0) {
config.set_intra_op_parallelism_threads(num_threads);
config.set_inter_op_parallelism_threads(num_threads);
}
LOG(INFO) << "Got config, " << config.device_count_size() << " devices";
session->reset(tensorflow::NewSession(options));
*graph_def = std::make_unique<GraphDef>();
tensorflow::GraphDef tensorflow_graph;
absl::Status s = ReadBinaryProto(Env::Default(), graph, graph_def->get());
if (!s.ok()) {
s = ReadTextProto(Env::Default(), graph, graph_def->get());
}
if (!s.ok()) {
LOG(ERROR) << "Could not create TensorFlow Graph: " << s;
return s;
}
s = (*session)->Create(*(graph_def->get()));
if (!s.ok()) {
LOG(ERROR) << "Could not create TensorFlow Session: " << s;
return s;
}
return absl::OkStatus();
}
absl::Status RunBenchmark(const std::vector<InputLayerInfo>& inputs,
const std::vector<std::string>& outputs,
const std::vector<std::string>& targets,
Session* session, StatSummarizer* stats,
int64_t* inference_time_us) {
std::vector<std::pair<std::string, tensorflow::Tensor> > input_tensors;
CreateTensorsFromInputInfo(inputs, &input_tensors);
std::vector<tensorflow::Tensor> output_tensors;
absl::Status s;
RunOptions run_options;
if (stats != nullptr) {
run_options.set_trace_level(RunOptions::FULL_TRACE);
}
RunMetadata run_metadata;
const int64_t start_time = Env::Default()->NowMicros();
s = session->Run(run_options, input_tensors, outputs, targets,
&output_tensors, &run_metadata);
const int64_t end_time = Env::Default()->NowMicros();
*inference_time_us = end_time - start_time;
if (!s.ok()) {
LOG(ERROR) << "Error during inference: " << s;
return s;
}
if (stats != nullptr) {
assert(run_metadata.has_step_stats());
const StepStats& step_stats = run_metadata.step_stats();
stats->ProcessStepStats(step_stats);
}
return s;
}
absl::Status TimeMultipleRuns(double sleep_seconds, int num_runs,
double max_time_s,
const std::vector<InputLayerInfo>& inputs,
const std::vector<std::string>& outputs,
const std::vector<std::string>& targets,
Session* session, StatSummarizer* stats,
int64_t* total_time_us,
int64_t* actual_num_runs) {
*total_time_us = 0;
LOG(INFO) << "Running benchmark for max " << num_runs << " iterations, max "
<< max_time_s << " seconds "
<< (stats != nullptr ? "with" : "without")
<< " detailed stat logging, with " << sleep_seconds
<< "s sleep between inferences";
Stat<int64_t> stat;
const bool until_max_time = num_runs <= 0;
for (int i = 0; until_max_time || i < num_runs; ++i) {
int64_t time;
absl::Status run_status =
RunBenchmark(inputs, outputs, targets, session, stats, &time);
stat.UpdateStat(time);
(*total_time_us) += time;
++(*actual_num_runs);
if (max_time_s > 0.0 && (*total_time_us / 1000000.0) > max_time_s) {
break;
}
if (!run_status.ok()) {
LOG(INFO) << "Failed on run " << i;
return run_status;
}
// If requested, sleep between runs for an arbitrary amount of time.
// This can be helpful to determine the effect of mobile processor
// scaling and thermal throttling.
if (sleep_seconds > 0.0) {
SleepSeconds(sleep_seconds);
}
}
std::stringstream stream;
stat.OutputToStream(&stream);
LOG(INFO) << stream.str() << std::endl;
return absl::OkStatus();
}
int Main(int argc, char** argv) {
std::string graph = "/data/local/tmp/tensorflow_inception_graph.pb";
std::string init_ops_string = "";
std::string input_layer_string = "input:0";
std::string input_layer_shape_string = "1,224,224,3";
std::string input_layer_type_string = "float";
std::string input_layer_values_string = "";
std::string output_layer_string = "output:0";
std::string target_layer_string = "";
int max_num_runs = 1000;
std::string max_time = "10.0";
std::string inference_delay = "-1.0";
std::string inter_benchmark_delay = "-1.0";
int num_threads = -1;
std::string benchmark_name = "";
std::string output_prefix = "";
bool show_sizes = false;
bool show_run_order = true;
int run_order_limit = 0;
bool show_time = true;
int time_limit = 10;
bool show_memory = true;
int memory_limit = 10;
bool show_type = true;
bool show_summary = true;
bool show_flops = false;
int warmup_runs = 1;
std::vector<Flag> flag_list = {
Flag("graph", &graph, "graph file name"),
Flag("init_ops", &init_ops_string, "init ops"),
Flag("input_layer", &input_layer_string, "input layer names"),
Flag("input_layer_shape", &input_layer_shape_string, "input layer shape"),
Flag("input_layer_type", &input_layer_type_string, "input layer type"),
Flag("input_layer_values", &input_layer_values_string,
"values to initialize the inputs with"),
Flag("output_layer", &output_layer_string, "output layer name"),
Flag("target_layer", &target_layer_string, "target layer name"),
Flag("max_num_runs", &max_num_runs, "number of runs max"),
Flag("max_time", &max_time, "length to run max"),
Flag("inference_delay", &inference_delay,
"delay between runs in seconds"),
Flag("inter_benchmark_delay", &inter_benchmark_delay,
"delay between benchmarks in seconds"),
Flag("num_threads", &num_threads, "number of threads"),
Flag("benchmark_name", &benchmark_name, "benchmark name"),
Flag("output_prefix", &output_prefix, "benchmark output prefix"),
Flag("show_sizes", &show_sizes, "whether to show sizes"),
Flag("show_run_order", &show_run_order,
"whether to list stats by run order"),
Flag("run_order_limit", &run_order_limit,
"how many items to show by run order"),
Flag("show_time", &show_time, "whether to list stats by time taken"),
Flag("time_limit", &time_limit, "how many items to show by time taken"),
Flag("show_memory", &show_memory, "whether to list stats by memory used"),
Flag("memory_limit", &memory_limit,
"how many items to show by memory used"),
Flag("show_type", &show_type, "whether to list stats by op type"),
Flag("show_summary", &show_summary,
"whether to show a summary of the stats"),
Flag("show_flops", &show_flops, "whether to estimate the model's FLOPs"),
Flag("warmup_runs", &warmup_runs, "how many runs to initialize model"),
};
std::string usage = Flags::Usage(argv[0], flag_list);
const bool parse_result = Flags::Parse(&argc, argv, flag_list);
if (!parse_result) {
LOG(ERROR) << usage;
return -1;
}
std::vector<std::string> init_ops = str_util::Split(init_ops_string, ',');
std::vector<std::string> input_layers =
str_util::Split(input_layer_string, ',');
std::vector<std::string> input_layer_shapes =
str_util::Split(input_layer_shape_string, ':');
std::vector<std::string> input_layer_types =
str_util::Split(input_layer_type_string, ',');
std::vector<std::string> input_layer_values =
str_util::Split(input_layer_values_string, ':');
std::vector<std::string> output_layers =
str_util::Split(output_layer_string, ',');
std::vector<std::string> target_layers =
str_util::Split(target_layer_string, ',');
if ((input_layers.size() != input_layer_shapes.size()) ||
(input_layers.size() != input_layer_types.size())) {
LOG(ERROR) << "There must be the same number of items in --input_layer,"
<< " --input_layer_shape, and --input_layer_type, for example"
<< " --input_layer=input1,input2 --input_layer_type=float,float "
<< " --input_layer_shape=1,224,224,4:1,20";
LOG(ERROR) << "--input_layer=" << input_layer_string << " ("
<< input_layers.size() << " items)";
LOG(ERROR) << "--input_layer_type=" << input_layer_type_string << " ("
<< input_layer_types.size() << " items)";
LOG(ERROR) << "--input_layer_shape=" << input_layer_shape_string << " ("
<< input_layer_shapes.size() << " items)";
return -1;
}
const size_t inputs_count = input_layers.size();
::tensorflow::port::InitMain(argv[0], &argc, &argv);
if (argc > 1) {
LOG(ERROR) << "Unknown argument " << argv[1] << "\n" << usage;
return -1;
}
LOG(INFO) << "Graph: [" << graph << "]";
LOG(INFO) << "Init ops:" << init_ops_string;
LOG(INFO) << "Input layers: [" << input_layer_string << "]";
LOG(INFO) << "Input shapes: [" << input_layer_shape_string << "]";
LOG(INFO) << "Input types: [" << input_layer_type_string << "]";
LOG(INFO) << "Output layers: [" << output_layer_string << "]";
LOG(INFO) << "Target layers: [" << target_layer_string << "]";
LOG(INFO) << "Num runs: [" << max_num_runs << "]";
LOG(INFO) << "Inter-inference delay (seconds): [" << inference_delay << "]";
LOG(INFO) << "Inter-benchmark delay (seconds): [" << inter_benchmark_delay
<< "]";
LOG(INFO) << "Num threads: [" << num_threads << "]";
LOG(INFO) << "Benchmark name: [" << benchmark_name << "]";
LOG(INFO) << "Output prefix: [" << output_prefix << "]";
LOG(INFO) << "Show sizes: [" << show_sizes << "]";
LOG(INFO) << "Warmup runs: [" << warmup_runs << "]";
std::unique_ptr<Session> session;
std::unique_ptr<StatSummarizer> stats;
std::unique_ptr<GraphDef> graph_def;
int64_t initialization_start_us = Env::Default()->NowMicros();
absl::Status initialize_status =
InitializeSession(num_threads, graph, &session, &graph_def);
int64_t initialization_end_us = Env::Default()->NowMicros();
double initialization_time_s =
(initialization_end_us - initialization_start_us) / 1000000.0;
LOG(INFO) << "Initialized session in " << initialization_time_s << "s";
if (!initialize_status.ok()) {
return -1;
}
if (!init_ops.empty()) {
absl::Status initialize_variables_status =
InitializeVariables(session.get(), init_ops);
if (!initialize_variables_status.ok()) {
LOG(ERROR) << "Graph variables initialization failed with "
<< initialize_variables_status;
return -1;
}
}
StatSummarizerOptions stats_options;
stats_options.show_run_order = show_run_order;
stats_options.run_order_limit = run_order_limit;
stats_options.show_time = show_time;
stats_options.time_limit = time_limit;
stats_options.show_memory = show_memory;
stats_options.memory_limit = memory_limit;
stats_options.show_type = show_type;
stats_options.show_summary = show_summary;
stats = std::make_unique<tensorflow::StatSummarizer>(stats_options);
const double inter_inference_sleep_seconds =
std::strtod(inference_delay.c_str(), nullptr);
const double inter_benchmark_sleep_seconds =
std::strtod(inter_benchmark_delay.c_str(), nullptr);
const double max_benchmark_time_seconds =
std::strtod(max_time.c_str(), nullptr);
std::vector<InputLayerInfo> inputs;
for (int n = 0; n < inputs_count; ++n) {
InputLayerInfo input;
CHECK(DataTypeFromString(input_layer_types[n], &input.data_type))
<< input_layer_types[n] << " was an invalid type";
std::vector<std::string> split_layer_shapes =
str_util::Split(input_layer_shapes[n], ',');
for (const std::string& layer_shape : split_layer_shapes) {
int32_t tmp;
CHECK(absl::SimpleAtoi(layer_shape, &tmp))
<< "Incorrect size string specified: " << input_layer_shapes[n];
if (tmp == -1) {
LOG(ERROR) << "Any unknown sizes in the shapes (-1's) must be replaced"
<< " with the size you want to benchmark with.";
return -1;
} else {
input.shape.AddDim(tmp);
}
}
input.name = input_layers[n];
if (n < input_layer_values.size()) {
std::vector<std::string> string_tokens =
str_util::Split(input_layer_values[n], ',');
input.initialization_values.clear();
input.initialization_values.reserve(string_tokens.size());
for (const std::string& str_val : string_tokens) {
float val;
CHECK(absl::SimpleAtof(str_val, &val))
<< "Incorrect initialization values string specified: "
<< input_layer_values[n];
input.initialization_values.push_back(val);
}
}
inputs.push_back(input);
}
// If requested, run through the graph first to preinitialize everything
// before the benchmarking runs.
int64_t warmup_time_us = 0;
int64_t num_warmup_runs = 0;
if (warmup_runs > 0) {
absl::Status warmup_time_status =
TimeMultipleRuns(inter_inference_sleep_seconds, warmup_runs, -1.0,
inputs, output_layers, target_layers, session.get(),
nullptr, &warmup_time_us, &num_warmup_runs);
if (!warmup_time_status.ok()) {
LOG(ERROR) << "Timing failed with " << warmup_time_status;
return -1;
}
}
// Capture overall inference time without stat logging overhead. This is the
// timing data that can be compared to other libraries.
SleepSeconds(inter_benchmark_sleep_seconds);
int64_t no_stat_time_us = 0;
int64_t no_stat_num_runs = 0;
absl::Status no_stat_time_status = TimeMultipleRuns(
inter_inference_sleep_seconds, max_num_runs, max_benchmark_time_seconds,
inputs, output_layers, target_layers, session.get(), nullptr,
&no_stat_time_us, &no_stat_num_runs);
const double no_stat_wall_time = no_stat_time_us / 1000000.0;
if (!no_stat_time_status.ok()) {
LOG(ERROR) << "Timing failed with " << no_stat_time_status;
return -1;
}
// Run again to gather detailed log stats to get a better idea of where
// relative time is going within the graph.
SleepSeconds(inter_benchmark_sleep_seconds);
int64_t stat_time_us = 0;
int64_t stat_num_runs = 0;
absl::Status stat_time_status = TimeMultipleRuns(
inter_inference_sleep_seconds, max_num_runs, max_benchmark_time_seconds,
inputs, output_layers, target_layers, session.get(), stats.get(),
&stat_time_us, &stat_num_runs);
if (!stat_time_status.ok()) {
LOG(ERROR) << "Timing failed with " << stat_time_status;
return -1;
}
LOG(INFO) << "Average inference timings in us: "
<< "Warmup: "
<< (warmup_runs > 0 ? warmup_time_us / warmup_runs : 0) << ", "
<< "no stats: " << no_stat_time_us / no_stat_num_runs << ", "
<< "with stats: " << stat_time_us / stat_num_runs;
stats->PrintStepStats();
if (show_sizes) {
stats->PrintOutputs();
}
if (show_flops) {
int64_t total_flops;
std::unordered_map<std::string, int64_t> flops_by_op;
absl::Status flop_status = CalculateFlops(*graph_def, inputs, session.get(),
&total_flops, &flops_by_op);
if (!flop_status.ok()) {
LOG(ERROR) << "FLOPs calculation failed with " << flop_status;
return -1;
}
std::string pretty_flops;
if (total_flops < 1000) {
pretty_flops = absl::StrCat(total_flops, " FLOPs");
} else if (total_flops < (1000 * 1000)) {
const float rounded_flops = (total_flops / 1000.0f);
pretty_flops = absl::StrCat(rounded_flops, "k FLOPs");
} else if (total_flops < (1000 * 1000 * 1000)) {
const float rounded_flops = round(total_flops / 1000.0f) / 1000.0f;
pretty_flops = absl::StrCat(rounded_flops, " million FLOPs");
} else {
const float rounded_flops =
round(total_flops / (1000.0f * 1000.0f)) / 1000.0f;
pretty_flops = absl::StrCat(rounded_flops, " billion FLOPs");
}
LOG(INFO) << "FLOPs estimate: " << strings::HumanReadableNum(total_flops);
const double mean_run_time = no_stat_wall_time / no_stat_num_runs;
LOG(INFO) << "FLOPs/second: "
<< strings::HumanReadableNum(
static_cast<int64_t>(total_flops / mean_run_time));
}
if (!benchmark_name.empty() && !output_prefix.empty()) {
// Compute the total number of values per input.
int64_t total_size = inputs[0].shape.num_elements();
// Throughput in MB/s
const double throughput =
DataTypeSize(inputs[0].data_type) * total_size * no_stat_num_runs /
static_cast<double>(no_stat_wall_time) / (1024 * 1024);
// Report the stats.
RecordBenchmarkEntry(output_prefix, benchmark_name, "", no_stat_num_runs,
no_stat_wall_time, throughput);
// Session initialization time.
RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-init", 1,
initialization_time_s);
// First inference time. Note: if warmup_runs is > 1 this will actually be
// an average of all the warmup runs.
RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-first-inference",
warmup_runs, warmup_time_us / 1000000.0);
// Time from starting to initialize TF to getting the first result back.
// This also assumes that only one warmup run is performed.
RecordBenchmarkEntry(
output_prefix, benchmark_name, "meta-init-plus-first-inference", 1,
initialization_time_s + (warmup_time_us / 1000000.0) / warmup_runs);
std::map<std::string, int64_t> node_type_map_count;
std::map<std::string, int64_t> node_type_map_time;
std::map<std::string, int64_t> node_type_map_memory;
std::map<std::string, int64_t> node_type_map_times_called;
int64_t accumulated_us;
stats->ComputeStatsByType(&node_type_map_count, &node_type_map_time,
&node_type_map_memory,
&node_type_map_times_called, &accumulated_us);
for (const auto& time : node_type_map_time) {
LOG(INFO) << "Outputting: [" << time.first << "]";
RecordBenchmarkEntry(output_prefix, benchmark_name, time.first,
stat_num_runs,
(time.second * stat_num_runs) / 1000000.0f);
}
}
return 0;
}
} // namespace benchmark_model
} // namespace tensorflow
tensorflow/tools/benchmark/benchmark_model.h
+69
View File
@@ -0,0 +1,69 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_TOOLS_BENCHMARK_BENCHMARK_MODEL_H_
#define TENSORFLOW_TOOLS_BENCHMARK_BENCHMARK_MODEL_H_
#include <cstdint>
#include <memory>
#include <vector>
#include "absl/status/status.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/util/stat_summarizer.h"
namespace tensorflow {
namespace benchmark_model {
// Used to help construct dummy inputs for the benchmarking.
struct InputLayerInfo {
std::string name;
DataType data_type;
TensorShape shape;
std::vector<float> initialization_values;
};
// Loads a model from disk into a new session.
absl::Status InitializeSession(int num_threads, const std::string& graph,
std::unique_ptr<Session>* session,
std::unique_ptr<GraphDef>* graph_def);
// Does a single run of the model that's been loaded into the given session.
absl::Status RunBenchmark(const std::vector<InputLayerInfo>& inputs,
const std::vector<std::string>& outputs,
const std::vector<std::string>& targets,
Session* session, StatSummarizer* stats,
int64_t* inference_time_us);
// Runs the model multiple time, keeping track of timing information.
absl::Status TimeMultipleRuns(double sleep_seconds, int num_runs,
double max_time_s,
const std::vector<InputLayerInfo>& inputs,
const std::vector<std::string>& outputs,
const std::vector<std::string>& targets,
Session* session, StatSummarizer* stats,
int64_t* total_time_us, int64_t* actual_num_runs);
// Handles all setup and argument parsing.
int Main(int argc, char** argv);
} // namespace benchmark_model
} // namespace tensorflow
#endif // TENSORFLOW_TOOLS_BENCHMARK_BENCHMARK_MODEL_H_
tensorflow/tools/benchmark/benchmark_model_main.cc
+20
View File
@@ -0,0 +1,20 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/tools/benchmark/benchmark_model.h"
int main(int argc, char** argv) {
return tensorflow::benchmark_model::Main(argc, argv);
}
tensorflow/tools/benchmark/benchmark_model_test.cc
+118
View File
@@ -0,0 +1,118 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/tools/benchmark/benchmark_model.h"
#include <cstdint>
#include <memory>
#include <string>
#include "tensorflow/cc/framework/scope.h"
#include "tensorflow/cc/ops/array_ops.h"
#include "tensorflow/cc/ops/math_ops.h"
#include "xla/tsl/lib/core/status_test_util.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_testutil.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/path.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/util/stat_summarizer.h"
namespace tensorflow {
namespace {
void CreateTestGraph(const ::tensorflow::Scope& root,
benchmark_model::InputLayerInfo* input,
std::string* output_name, GraphDef* graph_def) {
// Create a simple graph and write it to filename_pb.
const int input_width = 400;
const int input_height = 10;
input->shape = TensorShape({input_width, input_height});
input->data_type = DT_FLOAT;
const TensorShape constant_shape({input_height, input_width});
Tensor constant_tensor(DT_FLOAT, constant_shape);
test::FillFn<float>(&constant_tensor, [](int) -> float { return 3.0; });
auto placeholder =
ops::Placeholder(root, DT_FLOAT, ops::Placeholder::Shape(input->shape));
input->name = placeholder.node()->name();
auto m = ops::MatMul(root, placeholder, constant_tensor);
*output_name = m.node()->name();
TF_ASSERT_OK(root.ToGraphDef(graph_def));
}
TEST(BenchmarkModelTest, InitializeAndRun) {
const std::string dir = testing::TmpDir();
const std::string filename_pb = io::JoinPath(dir, "graphdef.pb");
auto root = Scope::NewRootScope().ExitOnError();
benchmark_model::InputLayerInfo input;
std::string output_name;
GraphDef graph_def;
CreateTestGraph(root, &input, &output_name, &graph_def);
std::string graph_def_serialized;
graph_def.SerializeToString(&graph_def_serialized);
TF_ASSERT_OK(
WriteStringToFile(Env::Default(), filename_pb, graph_def_serialized));
std::unique_ptr<Session> session;
std::unique_ptr<GraphDef> loaded_graph_def;
TF_ASSERT_OK(benchmark_model::InitializeSession(1, filename_pb, &session,
&loaded_graph_def));
std::unique_ptr<StatSummarizer> stats;
stats =
std::make_unique<tensorflow::StatSummarizer>(*(loaded_graph_def.get()));
int64_t time;
int64_t num_runs = 0;
TF_ASSERT_OK(benchmark_model::TimeMultipleRuns(
0.0, 10, 0.0, {input}, {output_name}, {}, session.get(), stats.get(),
&time, &num_runs));
ASSERT_EQ(num_runs, 10);
}
TEST(BenchmarkModeTest, TextProto) {
const std::string dir = testing::TmpDir();
const std::string filename_txt = io::JoinPath(dir, "graphdef.pb.txt");
auto root = Scope::NewRootScope().ExitOnError();
benchmark_model::InputLayerInfo input;
std::string output_name;
GraphDef graph_def;
CreateTestGraph(root, &input, &output_name, &graph_def);
TF_ASSERT_OK(WriteTextProto(Env::Default(), filename_txt, graph_def));
std::unique_ptr<Session> session;
std::unique_ptr<GraphDef> loaded_graph_def;
TF_ASSERT_OK(benchmark_model::InitializeSession(1, filename_txt, &session,
&loaded_graph_def));
std::unique_ptr<StatSummarizer> stats;
stats =
std::make_unique<tensorflow::StatSummarizer>(*(loaded_graph_def.get()));
int64_t time;
int64_t num_runs = 0;
TF_ASSERT_OK(benchmark_model::TimeMultipleRuns(
0.0, 10, 0.0, {input}, {output_name}, {}, session.get(), stats.get(),
&time, &num_runs));
ASSERT_EQ(num_runs, 10);
}
} // namespace
} // namespace tensorflow
tensorflow/tools/benchmark/download_models.sh
+52
View File
@@ -0,0 +1,52 @@
#!/bin/bash
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
source onednn_benchmark_config.sh
# Store models in home directory
mkdir -p ${TF_GRAPHS}
cd ${TF_GRAPHS}
# Download TF graphs linked from MLPerf Inference v2.0
# https://github.com/mlcommons/inference#mlperf-inference-v20-submission-02252022
wget -c https://zenodo.org/record/2535873/files/resnet50_v1.pb -O resnet50_v1-5.pb
wget -c https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip
wget -c https://zenodo.org/record/2269307/files/mobilenet_v1_1.0_224.tgz -O mobilenet-v1.tgz # 90MB
wget -c http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_coco_2018_01_28.tar.gz -O ssd-mobilenet-v1.tar.gz
wget -c https://zenodo.org/record/3345892/files/tf_ssd_resnet34_22.1.zip -O ssd-resnet34.zip
wget -c https://zenodo.org/record/3939747/files/model.pb -O bert-large.pb # 1.2GB. Could take 10+ minutes to download.
# Extract graphs from tar/zip files.
unzip ./inception5h.zip -d inception
mv inception/tensorflow_inception_graph.pb inception.pb
rm -rf inception
tar -xzf mobilenet-v1.tgz ./mobilenet_v1_1.0_224_frozen.pb
mv mobilenet_v1_1.0_224_frozen.pb mobilenet-v1.pb
tar -xzf ssd-mobilenet-v1.tar.gz ssd_mobilenet_v1_coco_2018_01_28/frozen_inference_graph.pb
mv ssd_mobilenet_v1_coco_2018_01_28/frozen_inference_graph.pb ssd-mobilenet-v1.pb
rm -rf ssd_mobilenet_v1_coco_2018_01_28
unzip ssd-resnet34.zip
mv tf_ssd_resnet34_22.1/resnet34_tf.22.1.pb ssd-resnet34.pb
rm -rf tf_ssd_resnet34_22.1
# Now we should have the following model files in ~/tf-graphs:
# - bert-large.pb
# - inception.pb
# - mobilenet-v1.pb
# - resnet50_v1-5.pb
# - ssd-mobilenet-v1.pb
# - ssd-resnet34.pb
tensorflow/tools/benchmark/onednn_benchmark_config.sh
+30
View File
@@ -0,0 +1,30 @@
#!/bin/bash
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Stores shared and platform-specific benchmark configurations
# Path to store downloaded TensorFlow models.
export TF_GRAPHS=~/tf-graphs
export BUILDER=bazel
configure_build() {
cd ../../..
yes "" | ./configure
}
benchmark_command() {
echo "--config=opt --dynamic_mode=off //tensorflow/tools/benchmark:benchmark_model -- "
}
tensorflow/tools/benchmark/parse_onednn_benchmarks.py
+109
View File
@@ -0,0 +1,109 @@
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Parses results from run_onednn_benchmarks.sh.
Example results:
Showing runtimes in microseconds. `?` means not available.
Model, Batch, Vanilla, oneDNN, Speedup
bert-large, 1, x, y, x/y
bert-large, 16, ..., ..., ...
inception, 1, ..., ..., ...
inception, 16, ..., ..., ...
⋮
ssd-resnet34, 1, ?, ..., ?
ssd-resnet34, 16, ?, ..., ?
Vanilla TF can't run ssd-resnet34 on CPU because it doesn't support NCHW format.
"""
import enum
import re
import sys
db = dict()
models = set()
batch_sizes = set()
State = enum.Enum("State", "FIND_CONFIG_OR_MODEL FIND_RUNNING_TIME")
def parse_results(lines):
"""Parses benchmark results from run_onednn_benchmarks.sh.
Stores results in a global dict.
Args:
lines: Array of strings corresponding to each line of the output from
run_onednn_benchmarks.sh
Raises:
RuntimeError: If the program reaches an unknown state.
"""
idx = 0
batch, onednn, model = None, None, None
state = State.FIND_CONFIG_OR_MODEL
while idx < len(lines):
if state is State.FIND_CONFIG_OR_MODEL:
config = re.match(
r"\+ echo 'BATCH=(?P<batch>[\d]+), ONEDNN=(?P<onednn>[\d]+)",
lines[idx])
if config:
batch = int(config.group("batch"))
onednn = int(config.group("onednn"))
batch_sizes.add(batch)
else:
model_re = re.search(r"tf-graphs\/(?P<model>[\w\d_-]+).pb", lines[idx])
assert model_re
model = model_re.group("model")
models.add(model)
state = State.FIND_RUNNING_TIME
elif state is State.FIND_RUNNING_TIME:
match = re.search(r"no stats: (?P<avg>[\d.]+)", lines[idx])
state = State.FIND_CONFIG_OR_MODEL
if match:
avg = float(match.group("avg"))
key = (model, batch, onednn)
assert None not in key
db[key] = avg
else:
# Some models such as ssd-resnet34 can't run on CPU with vanilla TF and
# won't have results. This line contains either a config or model name.
continue
else:
raise RuntimeError("Reached the unreachable code.")
idx = idx + 1
def main():
filename = sys.argv[1]
with open(filename, "r") as f:
lines = f.readlines()
parse_results(lines)
print("Showing runtimes in microseconds. `?` means not available.")
print("%20s, %6s, %14s, %14s, %10s" %
("Model", "Batch", "Vanilla", "oneDNN", "Speedup"))
for model in sorted(models):
for batch in sorted(batch_sizes):
key = (model, batch, 0)
eigen = db[key] if key in db else "?"
key = (model, batch, 1)
onednn = db[key] if key in db else "?"
speedup = "%10.2f" % (eigen / onednn) if "?" not in (eigen,
onednn) else "?"
print("%20s, %6d, %14s, %14s, %10s" %
(model, batch, str(eigen), str(onednn), speedup))
if __name__ == "__main__":
main()
tensorflow/tools/benchmark/run_models.sh
+85
View File
@@ -0,0 +1,85 @@
#!/bin/bash
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Run models downloaded with download_models.sh with the TF benchmark tool.
# This script must be called from its current directory.
set -x
source onednn_benchmark_config.sh
date
# Navigate to the workspace root directory and configure build configurations.
configure_build
pwd
date
for ONEDNN in 0 1; do
date
export CONFIG="$(benchmark_command ${ONEDNN})"
export TF_ENABLE_ONEDNN_OPTS=${ONEDNN}
export BENCH="${BUILDER} run ${CONFIG}"
for BATCH in 1 16 64; do
# Print information for parse_onednn_benchmarks.py
echo "BATCH=${BATCH}, ONEDNN=${ONEDNN}"
# Run each graph.
date
${BENCH} \
--graph=${TF_GRAPHS}/resnet50_v1-5.pb \
--input_layer="input_tensor:0" \
--input_layer_shape="${BATCH},224,224,3" \
--input_layer_type="float" \
--output_layer="softmax_tensor:0"
date
${BENCH} \
--graph=${TF_GRAPHS}/inception.pb \
--input_layer="input:0" \
--input_layer_shape="${BATCH},224,224,3" \
--input_layer_type="float" \
--output_layer="output:0"
date
${BENCH} \
--graph=${TF_GRAPHS}/mobilenet-v1.pb \
--input_layer="input:0" \
--input_layer_shape="${BATCH},224,224,3" \
--input_layer_type="float" \
--output_layer="MobilenetV1/Predictions/Reshape_1:0"
date
${BENCH} \
--graph=${TF_GRAPHS}/ssd-mobilenet-v1.pb \
--input_layer="image_tensor:0" \
--input_layer_shape="${BATCH},300,300,3" \
--input_layer_type="uint8" \
--output_layer="detection_classes:0"
date
${BENCH} \
--graph=${TF_GRAPHS}/ssd-resnet34.pb \
--input_layer="image:0" \
--input_layer_shape="${BATCH},3,1200,1200" \
--input_layer_type="float" \
--output_layer="detection_classes:0"
date
# Only run BERT with batch size 1 for now.
if [[ $BATCH == 1 ]]; then
${BENCH} \
--graph=${TF_GRAPHS}/bert-large.pb \
--input_layer="input_ids:0,input_mask:0,segment_ids:0" \
--input_layer_shape="1,384:1,384:1,384" \
--input_layer_type="int32,int32,int32" \
--output_layer="logits:0"
fi
done
done
tensorflow/tools/benchmark/run_onednn_benchmarks.sh
+27
View File
@@ -0,0 +1,27 @@
#!/bin/bash
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Benchmarks all downloaded models, parses, and summarizes results.
set -x
source onednn_benchmark_config.sh
export OUTDIR=~/onednn_benchmarks
mkdir -p ${OUTDIR}
bash run_models.sh 2>&1 | tee ${OUTDIR}/verbose.log
grep -v 'profiler_session\|xplane' ${OUTDIR}/verbose.log > ${OUTDIR}/run.log
grep "\+ ${BUILDER} run\|no stats:\|'BATCH=" ${OUTDIR}/run.log > ${OUTDIR}/to_parse.log
python3 parse_onednn_benchmarks.py ${OUTDIR}/to_parse.log | tee ${OUTDIR}/results.csv