wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/core/profiler/BUILD
+199
View File
@@ -0,0 +1,199 @@
# copybara:uncomment(oss-unused) load("//net/grpc/go/build_defs:go_grpc_library.bzl", "go_grpc_library")
load("@rules_python//python:proto.bzl", "py_proto_library")
load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
load("//tensorflow:tensorflow.default.bzl", "filegroup")
load("//tensorflow/core/platform:build_config.bzl", "tf_additional_all_protos", "tf_proto_library")
load("//tensorflow/core/platform:rules_cc.bzl", "cc_library")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:internal"],
licenses = ["notice"],
)
package_group(
name = "friends",
includes = ["//tensorflow:internal"],
)
package_group(
name = "internal",
packages = [
"//tensorflow/core/profiler/...",
"//tensorflow/python/eager/...",
"//tensorflow/python/profiler/...",
"//tensorflow/python/tpu/profiler/...",
],
)
tf_proto_library(
name = "profiler_service_monitor_result_proto",
srcs = ["profiler_service_monitor_result.proto"],
make_default_target_header_only = True,
protodeps = [
"@tsl//tsl/profiler/protobuf:profiler_service_monitor_result_proto",
],
visibility = ["//visibility:public"],
exports = [
"@tsl//tsl/profiler/protobuf:profiler_service_monitor_result_proto",
],
)
tf_proto_library(
name = "profiler_options_proto",
srcs = ["profiler_options.proto"],
make_default_target_header_only = True,
protodeps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"],
visibility = ["//visibility:public"],
exports = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"],
)
# This is needed because of how tf_android_core_proto_sources parses proto paths.
exports_files(
srcs = ["profiler_options.proto"],
visibility = ["//tensorflow/core:__pkg__"],
)
tf_proto_library(
name = "profiler_service_proto",
srcs = ["profiler_service.proto"],
make_default_target_header_only = True,
protodeps = [
"@tsl//tsl/profiler/protobuf:profiler_service_proto",
],
use_grpc_namespace = True,
visibility = ["//visibility:public"],
exports = ["@tsl//tsl/profiler/protobuf:profiler_service_proto"],
)
tf_proto_library(
name = "profiler_analysis_proto",
srcs = ["profiler_analysis.proto"],
make_default_target_header_only = True,
protodeps = [
"@tsl//tsl/profiler/protobuf:profiler_analysis_proto",
],
use_grpc_namespace = True,
visibility = ["//visibility:public"],
exports = ["@tsl//tsl/profiler/protobuf:profiler_analysis_proto"],
)
tf_proto_library(
name = "protos_all",
srcs = [
"profile.proto",
"tfprof_log.proto",
"tfprof_options.proto",
"tfprof_output.proto",
],
protodeps = [
"@tsl//tsl/profiler/protobuf:profile_proto",
] + tf_additional_all_protos(),
visibility = [":friends"],
exports = [
"@tsl//tsl/profiler/protobuf:profile_proto",
],
)
cc_library(
name = "profiler_impl",
visibility = [
"//tensorflow:__pkg__",
"//tensorflow/python:__pkg__",
"//tensorflow/security/fuzzing/cc/ops:__pkg__",
],
deps = [
"//tensorflow/core/profiler/lib:profiler_factory_impl",
"//tensorflow/core/profiler/lib:profiler_session_impl",
"@xla//xla/tsl/profiler/backends/cpu:annotation_stack_impl",
"@xla//xla/tsl/profiler/backends/cpu:threadpool_listener",
"@xla//xla/tsl/profiler/backends/cpu:traceme_recorder_impl",
"@xla//xla/tsl/profiler/utils:time_utils_impl",
],
alwayslink = True,
)
# Libraries and binaries of TensorFlow profiler 1.x.
tf_cc_binary(
name = "profiler",
srcs = ["profiler.cc"],
visibility = ["//visibility:public"],
deps = [
":protos_all_cc",
"//tensorflow/c:checkpoint_reader",
"//tensorflow/c:tf_status",
"//tensorflow/core:framework_internal",
"//tensorflow/core:lib",
"//tensorflow/core:lib_proto_parsing",
"//tensorflow/core/framework:graph_proto_cc",
"//tensorflow/core/platform:protobuf",
"//tensorflow/core/profiler:tfprof_options",
"//tensorflow/core/profiler/internal:tfprof_stats",
"//tensorflow/core/profiler/internal:tfprof_utils",
"//tensorflow/core/profiler/internal/advisor:tfprof_advisor",
"//tensorflow/core/protobuf:for_core_protos_cc",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@linenoise",
],
)
cc_library(
name = "tfprof_options",
srcs = ["tfprof_options.cc"],
hdrs = ["tfprof_options.h"],
visibility = ["//visibility:public"],
deps = [
":protos_all_cc",
"//tensorflow/core:framework_headers_lib",
"//tensorflow/core:framework_types_hdr",
"//tensorflow/core:portable_gif_internal",
"//tensorflow/core/lib/core:status",
"//tensorflow/core/platform:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
filegroup(
name = "mobile_srcs_no_runtime",
srcs = [
"@tsl//tsl/profiler/lib:mobile_srcs_no_runtime",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "mobile_hdrs_no_runtime",
srcs = [
"//tensorflow/core/profiler/lib:mobile_hdrs_no_runtime",
"@tsl//tsl/profiler/lib:mobile_hdrs_no_runtime",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "mobile_hdrs_only_runtime",
srcs = [
"//tensorflow/core/profiler/lib:mobile_hdrs_only_runtime",
"@tsl//tsl/profiler/lib:mobile_hdrs_only_runtime",
],
visibility = ["//tensorflow/core:__pkg__"],
)
# copybara:uncomment_begin(google-only)
# py_proto_library(
# name = "profiler_analysis_proto_py_pb2",
# has_services = 1,
# visibility = ["//visibility:public"],
# deps = [":profiler_analysis_proto"],
# )
#
# py_proto_library(
# name = "protos_all_py_pb2",
# visibility = [":friends"],
# deps = [":protos_all"],
# )
# copybara:uncomment_end
tensorflow/core/profiler/backends/README.md
+6
View File
@@ -0,0 +1,6 @@
This directory contains subclasses of `tensorflow::profiler::ProfilerInterface`
and related code.
`tensorflow::ProfilerSession` manages a collection of profile sources. Each
profile source implements a subclass of
`tensorflow::profiler::ProfilerInterface`.
tensorflow/core/profiler/backends/gpu/BUILD
+53
View File
@@ -0,0 +1,53 @@
load(
"@xla//xla/tsl/platform/default:cuda_build_defs.bzl",
"if_cuda_is_configured",
)
load("//tensorflow:tensorflow.default.bzl", "tf_cuda_cc_test")
load("//tensorflow/core/platform:build_config_root.bzl", "tf_cuda_tests_tags")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
tf_cuda_cc_test(
name = "device_tracer_test",
size = "small",
srcs = ["device_tracer_test.cc"],
args = ["--heap_check="],
tags = tf_cuda_tests_tags() + [
"gpu_cupti",
"nomac",
],
deps = [
"//tensorflow/cc:cc_ops",
"//tensorflow/core:all_kernels",
"//tensorflow/core:core_cpu",
"//tensorflow/core:core_cpu_internal",
"//tensorflow/core:direct_session",
"//tensorflow/core:framework",
"//tensorflow/core:framework_internal",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
"//tensorflow/core/common_runtime:direct_session_internal",
"//tensorflow/core/framework:graph_proto_cc",
"//tensorflow/core/framework:types_proto_cc",
"//tensorflow/core/kernels:ops_util",
"//tensorflow/core/profiler/lib:profiler_interface",
"//tensorflow/core/profiler/lib:profiler_session",
"//tensorflow/core/profiler/utils:xplane_schema",
"//tensorflow/core/profiler/utils:xplane_utils",
"//tensorflow/core/profiler/utils:xplane_visitor",
"@com_google_absl//absl/strings",
"@xla//xla/backends/profiler/gpu:cuda_test",
"@xla//xla/backends/profiler/gpu:cupti_collector",
"@xla//xla/backends/profiler/gpu:device_tracer",
"@xla//xla/tsl/profiler/utils:tf_xplane_visitor",
] + if_cuda_is_configured([
"@local_config_cuda//cuda:cuda_headers",
"@local_config_cuda//cuda:cupti_headers",
]),
)
tensorflow/core/profiler/backends/gpu/device_tracer_test.cc
+533
View File
@@ -0,0 +1,533 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <array>
#include <map>
#include <memory>
#include <sstream>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#if GOOGLE_CUDA
#include "third_party/gpus/cuda/extras/CUPTI/include/cupti_activity.h"
#include "third_party/gpus/cuda/include/cuda.h"
#include "third_party/gpus/cuda/include/cuda_runtime.h"
#include "xla/backends/profiler/gpu/cupti_collector.h"
#endif // GOOGLE_CUDA
#include "xla/tsl/profiler/utils/tf_xplane_visitor.h"
#include "tensorflow/core/common_runtime/direct_session.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_testutil.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/graph/testlib.h"
#include "tensorflow/core/kernels/ops_util.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/core/status_test_util.h"
#include "tensorflow/core/lib/core/threadpool.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/test_benchmark.h"
#include "tensorflow/core/profiler/lib/profiler_interface.h"
#include "tensorflow/core/profiler/lib/profiler_session.h"
#include "tensorflow/core/profiler/utils/xplane_schema.h"
#include "tensorflow/core/profiler/utils/xplane_utils.h"
#include "tensorflow/core/profiler/utils/xplane_visitor.h"
#include "tensorflow/core/public/session_options.h"
#include "tensorflow/core/util/device_name_utils.h"
// TODO(b/186367334)
#define CUPTI_NVBUG_3299481_WAR (10000 <= CUDA_VERSION && CUDA_VERSION < 11000)
#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM
namespace xla {
namespace profiler {
extern std::unique_ptr<tsl::profiler::ProfilerInterface> CreateGpuTracer(
const tensorflow::ProfileOptions& options);
} // namespace profiler
} // namespace xla
#endif
namespace tensorflow {
namespace profiler {
#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM
std::unique_ptr<tsl::profiler::ProfilerInterface> CreateGpuTracer() {
ProfileOptions options = tsl::ProfilerSession::DefaultOptions();
return xla::profiler::CreateGpuTracer(options);
}
#else
// We don't have device tracer for non-cuda case.
std::unique_ptr<tsl::profiler::ProfilerInterface> CreateGpuTracer() {
return nullptr;
}
#endif
namespace {
std::unique_ptr<Session> CreateSession() {
SessionOptions options;
(*options.config.mutable_device_count())["CPU"] = 1;
(*options.config.mutable_device_count())["GPU"] = 1;
options.config.set_allow_soft_placement(true);
return std::unique_ptr<Session>(NewSession(options));
}
class DeviceTracerTest : public ::testing::Test {
public:
void Initialize(std::initializer_list<float> a_values) {
Graph graph(OpRegistry::Global());
Tensor a_tensor(DT_FLOAT, TensorShape({2, 2}));
test::FillValues<float>(&a_tensor, a_values);
Node* a = test::graph::HostConstant(&graph, a_tensor);
Tensor x_tensor(DT_FLOAT, TensorShape({2, 1}));
test::FillValues<float>(&x_tensor, {1, 1});
Node* x = test::graph::HostConstant(&graph, x_tensor);
x_ = x->name();
// y = A * x
Node* y = test::graph::Matmul(&graph, a, x, false, false);
y->set_assigned_device_name("/device:GPU:0");
y_ = y->name();
// Use an Identity op to force a memcpy to CPU and back to GPU.
Node* i = test::graph::Identity(&graph, y);
i->set_assigned_device_name("/cpu:0");
Node* y_neg = test::graph::Unary(&graph, "Neg", i);
y_neg_ = y_neg->name();
y_neg->set_assigned_device_name("/device:GPU:0");
test::graph::ToGraphDef(&graph, &def_);
}
protected:
void ExpectFailure(const absl::Status& status, error::Code code) {
EXPECT_FALSE(status.ok()) << status;
if (!status.ok()) {
LOG(INFO) << "Status message: " << status.message();
EXPECT_EQ(code, status.code()) << status;
}
}
std::string x_;
std::string y_;
std::string y_neg_;
GraphDef def_;
};
TEST_F(DeviceTracerTest, StartStop) {
auto tracer = CreateGpuTracer();
if (!tracer) return;
TF_EXPECT_OK(tracer->Start());
TF_EXPECT_OK(tracer->Stop());
}
TEST_F(DeviceTracerTest, StopBeforeStart) {
auto tracer = CreateGpuTracer();
if (!tracer) return;
TF_EXPECT_OK(tracer->Stop());
TF_EXPECT_OK(tracer->Stop());
}
TEST_F(DeviceTracerTest, CollectBeforeStart) {
auto tracer = CreateGpuTracer();
if (!tracer) return;
XSpace space;
TF_EXPECT_OK(tracer->CollectData(&space));
EXPECT_EQ(space.planes_size(), 0);
}
TEST_F(DeviceTracerTest, CollectBeforeStop) {
auto tracer = CreateGpuTracer();
if (!tracer) return;
TF_EXPECT_OK(tracer->Start());
XSpace space;
absl::Status status = tracer->CollectData(&space);
ExpectFailure(status, tensorflow::error::FAILED_PRECONDITION);
TF_EXPECT_OK(tracer->Stop());
}
TEST_F(DeviceTracerTest, StartTwoTracers) {
auto tracer1 = CreateGpuTracer();
auto tracer2 = CreateGpuTracer();
if (!tracer1 || !tracer2) return;
TF_EXPECT_OK(tracer1->Start());
absl::Status status = tracer2->Start();
ExpectFailure(status, tensorflow::error::UNAVAILABLE);
TF_EXPECT_OK(tracer1->Stop());
TF_EXPECT_OK(tracer2->Start());
TF_EXPECT_OK(tracer2->Stop());
}
TEST_F(DeviceTracerTest, RunWithTracer) {
// On non-GPU platforms, we may not support DeviceTracer.
auto tracer = CreateGpuTracer();
if (!tracer) return;
Initialize({3, 2, -1, 0});
auto session = CreateSession();
ASSERT_TRUE(session != nullptr);
TF_ASSERT_OK(session->Create(def_));
std::vector<std::pair<std::string, Tensor>> inputs;
// Request two targets: one fetch output and one non-fetched output.
std::vector<std::string> output_names = {y_ + ":0"};
std::vector<std::string> target_nodes = {y_neg_};
std::vector<Tensor> outputs;
TF_ASSERT_OK(tracer->Start());
absl::Status s = session->Run(inputs, output_names, target_nodes, &outputs);
TF_ASSERT_OK(s);
TF_ASSERT_OK(tracer->Stop());
ASSERT_EQ(1, outputs.size());
// The first output should be initialized and have the correct
// output.
auto mat = outputs[0].matrix<float>();
ASSERT_TRUE(outputs[0].IsInitialized());
EXPECT_FLOAT_EQ(5.0, mat(0, 0));
}
TEST_F(DeviceTracerTest, RunWithTraceOption) {
Initialize({3, 2, -1, 0});
auto session = CreateSession();
ASSERT_TRUE(session != nullptr);
TF_ASSERT_OK(session->Create(def_));
std::vector<std::pair<std::string, Tensor>> inputs;
// Request two targets: one fetch output and one non-fetched output.
std::vector<std::string> output_names = {y_ + ":0"};
std::vector<std::string> target_nodes = {y_neg_};
std::vector<Tensor> outputs;
// Prepares RunOptions and RunOutputs
RunOptions run_options;
run_options.set_trace_level(RunOptions::FULL_TRACE);
RunMetadata run_metadata;
absl::Status s = session->Run(run_options, inputs, output_names, target_nodes,
&outputs, &run_metadata);
TF_ASSERT_OK(s);
ASSERT_TRUE(run_metadata.has_step_stats());
// Depending on whether this runs on CPU or GPU, we will have a
// different number of devices.
EXPECT_GE(run_metadata.step_stats().dev_stats_size(), 1);
}
#if TENSORFLOW_USE_ROCM
TEST_F(DeviceTracerTest, TraceToXSpace) {
auto tracer = CreateGpuTracer();
if (!tracer) return;
Initialize({3, 2, -1, 0});
auto session = CreateSession();
ASSERT_TRUE(session != nullptr);
TF_ASSERT_OK(session->Create(def_));
std::vector<std::pair<string, Tensor>> inputs;
// Request two targets: one fetch output and one non-fetched output.
std::vector<string> output_names = {y_ + ":0"};
std::vector<string> target_nodes = {y_neg_};
std::vector<Tensor> outputs;
TF_ASSERT_OK(tracer->Start());
Status s = session->Run(inputs, output_names, target_nodes, &outputs);
TF_ASSERT_OK(s);
TF_ASSERT_OK(tracer->Stop());
XSpace space;
TF_ASSERT_OK(tracer->CollectData(&space));
// At least one gpu plane and one host plane for launching events.
const XPlane* host_plane = FindPlaneWithName(space, kRoctracerApiPlaneName);
ASSERT_NE(host_plane, nullptr);
const XPlane* device_plane = FindPlaneWithName(space, GpuPlaneName(0));
ASSERT_NE(device_plane, nullptr); // Check if device plane is serialized.
// The device plane should have at least five events: one for MemcpyH2D, one
// for MemcpyD2H, two for Matmul (one from Eigen, one from cudnn), one for
// memset.
EXPECT_GE(device_plane->event_metadata_size(), 5);
// Check if device capacity is serialized.
XPlaneVisitor plane = tsl::profiler::CreateTfXPlaneVisitor(device_plane);
// Check if the device events timestamps are set.
int total_events = 0;
plane.ForEachLine([&](const tensorflow::profiler::XLineVisitor& line) {
line.ForEachEvent([&](const tensorflow::profiler::XEventVisitor& event) {
EXPECT_GT(event.TimestampNs(), 0);
EXPECT_GT(event.DurationNs(), 0);
++total_events;
});
});
EXPECT_GE(total_events, 5);
}
#endif // TENSORFLOW_USE_ROCM
#if GOOGLE_CUDA
TEST_F(DeviceTracerTest, TraceToXSpace) {
auto tracer = CreateGpuTracer();
if (!tracer) return;
Initialize({3, 2, -1, 0});
auto session = CreateSession();
ASSERT_TRUE(session != nullptr);
TF_ASSERT_OK(session->Create(def_));
std::vector<std::pair<std::string, Tensor>> inputs;
// Request two targets: one fetch output and one non-fetched output.
std::vector<std::string> output_names = {y_ + ":0"};
std::vector<std::string> target_nodes = {y_neg_};
std::vector<Tensor> outputs;
TF_ASSERT_OK(tracer->Start());
absl::Status s = session->Run(inputs, output_names, target_nodes, &outputs);
TF_ASSERT_OK(s);
TF_ASSERT_OK(tracer->Stop());
XSpace space;
TF_ASSERT_OK(tracer->CollectData(&space));
// At least one gpu plane and one host plane for launching events.
const XPlane* host_plane = FindPlaneWithName(space, kCuptiDriverApiPlaneName);
ASSERT_NE(host_plane, nullptr);
const XPlane* device_plane = FindPlaneWithName(space, GpuPlaneName(0));
ASSERT_NE(device_plane, nullptr); // Check if device plane is serialized.
// The device plane should have at least five events: one for MemcpyH2D, one
// for MemcpyD2H, two for Matmul (one from Eigen, one from cudnn), one for
// memset.
EXPECT_GE(device_plane->event_metadata_size(), 5);
// Check if device capacity is serialized.
XPlaneVisitor plane = tsl::profiler::CreateTfXPlaneVisitor(device_plane);
EXPECT_TRUE(plane.GetStat(StatType::kDevCapClockRateKHz).has_value());
EXPECT_TRUE(plane.GetStat(StatType::kDevCapCoreCount).has_value());
EXPECT_TRUE(plane.GetStat(StatType::kDevCapMemoryBandwidth).has_value());
EXPECT_TRUE(plane.GetStat(StatType::kDevCapMemorySize).has_value());
EXPECT_TRUE(plane.GetStat(StatType::kDevCapComputeCapMajor).has_value());
EXPECT_TRUE(plane.GetStat(StatType::kDevCapComputeCapMinor).has_value());
// Check if the device events timestamps are set.
int total_events = 0;
plane.ForEachLine([&](const tensorflow::profiler::XLineVisitor& line) {
line.ForEachEvent([&](const tensorflow::profiler::XEventVisitor& event) {
EXPECT_GT(event.TimestampNs(), 0);
EXPECT_GT(event.DurationNs(), 0);
++total_events;
});
});
EXPECT_GE(total_events, 5);
}
TEST_F(DeviceTracerTest, CudaRuntimeResource) {
auto tracer = CreateGpuTracer();
if (!tracer) return;
const size_t size_in_bytes = 8;
const int8_t test_value = 7;
TF_EXPECT_OK(tracer->Start());
void* hostptr = nullptr;
void* devptr = nullptr;
// These six CUDA API calls will create 4 CUDA API and 4 device XEvents.
ASSERT_EQ(cudaSuccess,
cudaHostAlloc(&hostptr, size_in_bytes, cudaHostAllocPortable));
ASSERT_EQ(cudaSuccess, cudaMalloc(&devptr, size_in_bytes));
VLOG(3) << "Allocated device memory, addr: " << devptr;
ASSERT_EQ(cudaSuccess, cudaMemset(devptr, test_value, size_in_bytes));
ASSERT_EQ(cudaSuccess,
cudaMemcpy(hostptr, devptr, size_in_bytes, cudaMemcpyDeviceToHost));
VLOG(3) << "Free device memory, addr: " << devptr;
auto* value_from_device = static_cast<uint8_t*>(hostptr);
for (size_t idx = 0; idx < size_in_bytes; ++idx) {
ASSERT_EQ(value_from_device[idx], test_value);
}
ASSERT_EQ(cudaSuccess, cudaFree(devptr));
ASSERT_EQ(cudaSuccess, cudaFreeHost(hostptr));
TF_EXPECT_OK(tracer->Stop());
XSpace space;
TF_EXPECT_OK(tracer->CollectData(&space));
const XPlane* cupti_host_plane =
FindPlaneWithName(space, kCuptiDriverApiPlaneName);
ASSERT_NE(cupti_host_plane, nullptr);
XPlaneVisitor host_plane =
tsl::profiler::CreateTfXPlaneVisitor(cupti_host_plane);
// Expect at least one XLine for CUPTI activity events. There may be an
// additional line for CuptiTracerEventType Overhead.
EXPECT_GE(host_plane.NumLines(), 1);
// These follow the order in which they were invoked above.
const std::array<StatType, 4> expected_event_stat_type = {
StatType::kMemallocDetails,
StatType::kMemsetDetails,
StatType::kMemcpyDetails,
StatType::kMemFreeDetails,
};
int event_idx = 0;
host_plane.ForEachLine([&](const tensorflow::profiler::XLineVisitor& line) {
VLOG(3) << "Line " << line.Id() << "\n";
line.ForEachEvent([&](const tensorflow::profiler::XEventVisitor& event) {
if (event_idx < expected_event_stat_type.size()) {
VLOG(3) << " Event "
<< (event.Type().has_value() ? std::to_string(*event.Type())
: "UNKNOWN_TYPE")
<< "\n";
absl::optional<XStatVisitor> stat =
event.GetStat(expected_event_stat_type[event_idx]);
// The stat may not exist if we're looking at the wrong line.
if (stat.has_value()) {
event_idx += 1;
VLOG(3) << " Stat name=" << stat->Name() << " type=" << *stat->Type()
<< " " << stat->ToString() << ", event_idx:" << event_idx
<< "\n";
}
}
});
});
// Expect that all stats show up. One host side event for each API call.
EXPECT_EQ(event_idx, expected_event_stat_type.size());
const XPlane* cupti_device_plane = FindPlaneWithName(space, GpuPlaneName(0));
ASSERT_NE(cupti_device_plane, nullptr);
XPlaneVisitor device_plane =
tsl::profiler::CreateTfXPlaneVisitor(cupti_device_plane);
bool found_activity_memory_host = false;
bool found_activity_memory_device = false;
bool found_activity_memset = false;
bool found_activity_memcpy = false;
device_plane.ForEachLine([&](const tensorflow::profiler::XLineVisitor& line) {
line.ForEachEvent([&](const tensorflow::profiler::XEventVisitor& event) {
event.ForEachStat([&](XStatVisitor stat) {
VLOG(3) << " Stat name=" << stat.Name() << " type=" << *stat.Type()
<< " " << stat.ToString() << "\n";
// These are the attributes set in cupti_collector::CreateXEvent.
auto space_delimited_stats = absl::StrSplit(stat.StrOrRefValue(), " ");
if (stat.Type() == StatType::kMemoryResidencyDetails) {
size_t num_bytes = 0;
size_t addr = 0;
absl::string_view kind;
for (const auto& detail : space_delimited_stats) {
std::vector<absl::string_view> name_value =
absl::StrSplit(detail, ":");
// We allocated 8 bytes of device memory with cudaMalloc/Free.
if (absl::StartsWith(detail, "num_bytes:")) {
(void)absl::SimpleAtoi(name_value[1], &num_bytes);
} else if (absl::StartsWith(detail, "addr:")) {
std::stringstream hex_string;
hex_string << std::hex << name_value[1];
hex_string >> addr;
} else if (absl::StartsWith(detail, "kind:")) {
kind = name_value[1];
}
}
if (addr == reinterpret_cast<size_t>(devptr) &&
num_bytes == size_in_bytes) {
found_activity_memory_device = true;
EXPECT_EQ(kind, xla::profiler::GetMemoryKindName(
CUPTI_ACTIVITY_MEMORY_KIND_DEVICE));
} else if (addr == reinterpret_cast<size_t>(hostptr) &&
num_bytes == size_in_bytes) {
found_activity_memory_host = true;
EXPECT_EQ(kind, xla::profiler::GetMemoryKindName(
CUPTI_ACTIVITY_MEMORY_KIND_PINNED));
}
} else if (stat.Type() == StatType::kMemsetDetails) {
CHECK(!found_activity_memset);
found_activity_memset = true;
for (const auto& detail : space_delimited_stats) {
std::vector<absl::string_view> name_value =
absl::StrSplit(detail, ":");
// We set 8 bytes of device memory with cudaMemset.
if (absl::StartsWith(detail, "num_bytes:")) {
size_t num_bytes = 0;
(void)absl::SimpleAtoi(name_value[1], &num_bytes);
EXPECT_EQ(num_bytes, 8);
} else if (absl::StartsWith(detail, "kind:")) {
EXPECT_EQ(name_value[1], xla::profiler::GetMemoryKindName(
CUPTI_ACTIVITY_MEMORY_KIND_DEVICE));
}
}
} else if (stat.Type() == StatType::kMemcpyDetails) {
CHECK(!found_activity_memcpy);
found_activity_memcpy = true;
for (const auto& detail : space_delimited_stats) {
std::vector<absl::string_view> name_value =
absl::StrSplit(detail, ":");
// We copied 8 bytes from device memory to pinned host memory.
if (absl::StartsWith(detail, "num_bytes:")) {
size_t num_bytes = 0;
(void)absl::SimpleAtoi(name_value[1], &num_bytes);
EXPECT_EQ(num_bytes, 8);
} else if (absl::StartsWith(detail, "kind_src:")) {
EXPECT_EQ(name_value[1], xla::profiler::GetMemoryKindName(
CUPTI_ACTIVITY_MEMORY_KIND_DEVICE));
} else if (absl::StartsWith(detail, "kind_dst:")) {
EXPECT_EQ(name_value[1], xla::profiler::GetMemoryKindName(
CUPTI_ACTIVITY_MEMORY_KIND_PINNED));
}
}
}
});
});
});
// Expect these CUDA API activities to be found.
EXPECT_TRUE(found_activity_memset);
EXPECT_TRUE(found_activity_memcpy);
EXPECT_FALSE(found_activity_memory_device);
#if CUPTI_NVBUG_3299481_WAR
EXPECT_FALSE(found_activity_memory_host);
#endif
}
void BM_CuptiTracer_ActivityBufferOverhead(::testing::benchmark::State& state) {
auto tracer = CreateGpuTracer();
CHECK_NOTNULL(tracer);
TF_EXPECT_OK(tracer->Start());
for (auto s : state) {
cudaFree(nullptr);
}
TF_EXPECT_OK(tracer->Stop());
}
BENCHMARK(BM_CuptiTracer_ActivityBufferOverhead);
void BM_CudaFree(::testing::benchmark::State& state) {
for (auto s : state) {
cudaFree(nullptr);
}
}
BENCHMARK(BM_CudaFree);
#endif // GOOGLE_CUDA
} // namespace
} // namespace profiler
} // namespace tensorflow
tensorflow/core/profiler/builds/BUILD
+5
View File
@@ -0,0 +1,5 @@
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow/core/profiler:internal"],
licenses = ["notice"],
)
tensorflow/core/profiler/builds/build_config.bzl
+14
View File
@@ -0,0 +1,14 @@
"""Provides a redirection point for platform specific implementations of Starlark utilities."""
load(
"@xla//xla/tsl/profiler/builds:build_config.bzl",
_if_profiler_oss = "if_profiler_oss",
_tf_profiler_alias = "tf_profiler_alias",
_tf_profiler_copts = "tf_profiler_copts",
_tf_profiler_pybind_cc_library_wrapper = "tf_profiler_pybind_cc_library_wrapper",
)
tf_profiler_alias = _tf_profiler_alias
tf_profiler_pybind_cc_library_wrapper = _tf_profiler_pybind_cc_library_wrapper
tf_profiler_copts = _tf_profiler_copts
if_profiler_oss = _if_profiler_oss
tensorflow/core/profiler/convert/BUILD
+73
View File
@@ -0,0 +1,73 @@
load("//tensorflow/core/platform:rules_cc.bzl", "cc_library")
load("//tensorflow/core/profiler/builds:build_config.bzl", "tf_profiler_copts")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow/core/profiler:internal"],
licenses = ["notice"],
)
cc_library(
name = "xplane_to_step_stats",
srcs = ["xplane_to_step_stats.cc"],
hdrs = ["xplane_to_step_stats.h"],
copts = tf_profiler_copts(),
deps = [
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/profiler/utils:gpu_event_stats",
"//tensorflow/core/profiler/utils:xplane_schema",
"//tensorflow/core/profiler/utils:xplane_utils",
"//tensorflow/core/profiler/utils:xplane_visitor",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/log",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/strings",
"@org_xprof//xprof/utils:gpu_event_stats",
"@tsl//tsl/profiler/protobuf:xplane_proto_cc",
"@xla//xla/tsl/profiler/utils:math_utils",
"@xla//xla/tsl/profiler/utils:tf_xplane_visitor",
],
)
# DO NOT ADD NEW DEPENDENCIES TO ANY TARGET IN THIS FILE.
# Instead, use //third_party/xprof/convert.
cc_library(
name = "hlo_proto_to_memory_visualization_utils",
hdrs = ["hlo_proto_to_memory_visualization_utils.h"],
copts = tf_profiler_copts(),
visibility = [
"//learning/deepmind/jax/statix:__subpackages__",
"//smartass/brain/tpu_worker:__subpackages__",
"//tensorflow/core/profiler:internal",
],
deps = [
"@org_xprof//xprof/convert:hlo_proto_to_memory_visualization_utils",
],
)
cc_library(
name = "profile_time_breakdown",
hdrs = ["profile_time_breakdown.h"],
copts = tf_profiler_copts(),
visibility = [
"//platforms/performance/autograppler/utils:__subpackages__",
"//tensorflow/core/profiler:internal",
],
deps = [
"@org_xprof//xprof/convert:profile_time_breakdown",
],
)
cc_library(
name = "xplane_to_op_stats",
hdrs = ["xplane_to_op_stats.h"],
copts = tf_profiler_copts(),
visibility = [
"//platforms/xla/tools/multihost_hlo_runner/hybrid_sim:__subpackages__",
"//tensorflow/core/profiler:internal",
],
deps = [
"@org_xprof//xprof/convert:xplane_to_op_stats",
],
)
tensorflow/core/profiler/convert/hlo_proto_to_memory_visualization_utils.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_HLO_PROTO_TO_MEMORY_VISUALIZATION_UTILS_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_HLO_PROTO_TO_MEMORY_VISUALIZATION_UTILS_H_
#include "xprof/convert/hlo_proto_to_memory_visualization_utils.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_HLO_PROTO_TO_MEMORY_VISUALIZATION_UTILS_H_
tensorflow/core/profiler/convert/profile_time_breakdown.h
+20
View File
@@ -0,0 +1,20 @@
/* Copyright 2024 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_PROFILE_TIME_BREAKDOWN_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_PROFILE_TIME_BREAKDOWN_H_
#include "xprof/convert/profile_time_breakdown.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_PROFILE_TIME_BREAKDOWN_H_
tensorflow/core/profiler/convert/trace_viewer/BUILD
+62
View File
@@ -0,0 +1,62 @@
load("//tensorflow/core/platform:rules_cc.bzl", "cc_library")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
cc_library(
name = "trace_events_filter_interface",
hdrs = ["trace_events_filter_interface.h"],
deps = [
"@org_xprof//xprof/convert/trace_viewer:trace_events_filter_interface",
],
)
cc_library(
name = "trace_viewer_visibility",
hdrs = ["trace_viewer_visibility.h"],
deps = [
"@org_xprof//xprof/convert/trace_viewer:trace_viewer_visibility",
],
)
cc_library(
name = "trace_viewer_color",
hdrs = ["trace_viewer_color.h"],
deps = [
"@org_xprof//xprof/convert/trace_viewer:trace_viewer_color",
],
)
cc_library(
name = "trace_events_to_json",
hdrs = ["trace_events_to_json.h"],
deps = [
"@org_xprof//xprof/convert/trace_viewer:trace_events_to_json",
],
)
cc_library(
name = "trace_event_arguments_builder",
hdrs = ["trace_event_arguments_builder.h"],
deps = [
"@org_xprof//xprof/convert/trace_viewer:trace_event_arguments_builder",
],
)
cc_library(
name = "trace_events_util",
hdrs = ["trace_events_util.h"],
deps = [
"@org_xprof//xprof/convert/trace_viewer:trace_events_util",
],
)
cc_library(
name = "trace_events",
hdrs = ["trace_events.h"],
deps = [
"@org_xprof//xprof/convert/trace_viewer:trace_events",
],
)
tensorflow/core/profiler/convert/trace_viewer/trace_event_arguments_builder.h
+20
View File
@@ -0,0 +1,20 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENT_ARGUMENTS_BUILDER_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENT_ARGUMENTS_BUILDER_H_
#include "xprof/convert/trace_viewer/trace_event_arguments_builder.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENT_ARGUMENTS_BUILDER_H_
tensorflow/core/profiler/convert/trace_viewer/trace_events.h
+20
View File
@@ -0,0 +1,20 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_H_
#include "xprof/convert/trace_viewer/trace_events.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_H_
tensorflow/core/profiler/convert/trace_viewer/trace_events_filter_interface.h
+20
View File
@@ -0,0 +1,20 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_FILTER_INTERFACE_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_FILTER_INTERFACE_H_
#include "xprof/convert/trace_viewer/trace_events_filter_interface.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_FILTER_INTERFACE_H_
tensorflow/core/profiler/convert/trace_viewer/trace_events_to_json.h
+20
View File
@@ -0,0 +1,20 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_TO_JSON_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_TO_JSON_H_
#include "xprof/convert/trace_viewer/trace_events_to_json.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_TO_JSON_H_
tensorflow/core/profiler/convert/trace_viewer/trace_events_util.h
+20
View File
@@ -0,0 +1,20 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_UTIL_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_UTIL_H_
#include "xprof/convert/trace_viewer/trace_events_util.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_EVENTS_UTIL_H_
tensorflow/core/profiler/convert/trace_viewer/trace_viewer_color.h
+20
View File
@@ -0,0 +1,20 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_VIEWER_COLOR_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_VIEWER_COLOR_H_
#include "xprof/convert/trace_viewer/trace_viewer_color.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_VIEWER_COLOR_H_
tensorflow/core/profiler/convert/trace_viewer/trace_viewer_visibility.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_VIEWER_VISIBILITY_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_VIEWER_VISIBILITY_H_
#include "xprof/convert/trace_viewer/trace_viewer_visibility.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_TRACE_VIEWER_TRACE_VIEWER_VISIBILITY_H_
tensorflow/core/profiler/convert/xplane_to_op_stats.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_XPLANE_TO_OP_STATS_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_XPLANE_TO_OP_STATS_H_
#include "xprof/convert/xplane_to_op_stats.h" // from @org_xprof // IWYU pragma: export
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_XPLANE_TO_OP_STATS_H_
tensorflow/core/profiler/convert/xplane_to_step_stats.cc
+217
View File
@@ -0,0 +1,217 @@
/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/convert/xplane_to_step_stats.h"
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
#include "absl/container/flat_hash_map.h"
#include "absl/log/check.h"
#include "absl/log/log.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/strings/strip.h"
#include "xla/tsl/profiler/utils/math_utils.h"
#include "xla/tsl/profiler/utils/tf_xplane_visitor.h"
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tensorflow/core/profiler/utils/gpu_event_stats.h"
#include "tensorflow/core/profiler/utils/xplane_schema.h"
#include "tensorflow/core/profiler/utils/xplane_utils.h"
#include "tensorflow/core/profiler/utils/xplane_visitor.h"
#include "tsl/profiler/protobuf/xplane.pb.h"
#include "xprof/utils/gpu_event_stats.h" // from @org_xprof
namespace tensorflow {
namespace profiler {
namespace {
struct CorrelationInfo {
uint32_t thread_id;
uint64_t enqueue_time_ns;
};
enum GpuEventType {
kUnknown,
kKernel,
kMemcpyH2D,
kMemcpyD2H,
kMemcpyD2D,
kMemcpyP2P,
};
GpuEventType ParseMemcpyName(absl::string_view memcpy_name) {
if (absl::ConsumePrefix(&memcpy_name, "Memcpy")) {
if (memcpy_name == "H2D") return GpuEventType::kMemcpyH2D;
if (memcpy_name == "D2H") return GpuEventType::kMemcpyD2H;
if (memcpy_name == "D2D") return GpuEventType::kMemcpyD2D;
if (memcpy_name == "P2P") return GpuEventType::kMemcpyP2P;
}
return GpuEventType::kUnknown;
}
void SetNodeTimes(uint64_t start_time, const XEventVisitor& event,
NodeExecStats* ns) {
// Since XPlane uses relative times, we need to convert event.TimestampNs() to
// absolute times
ns->set_all_start_micros(
tsl::profiler::NanoToMicro(start_time + event.TimestampNs()));
ns->set_op_start_rel_micros(0);
ns->set_op_end_rel_micros(tsl::profiler::NanoToMicro(event.DurationNs()));
ns->set_all_end_rel_micros(tsl::profiler::NanoToMicro(event.DurationNs()));
}
} // namespace
void ConvertGpuXSpaceToStepStats(const XSpace& xspace, StepStats* step_stats) {
std::vector<const XPlane*> device_planes =
FindPlanesWithPrefix(xspace, kGpuPlanePrefix);
if (device_planes.empty()) {
LOG(WARNING) << "GPU trace was not collected.";
return;
}
const XPlane* env_plane = FindPlaneWithName(xspace, kTaskEnvPlaneName);
XPlaneVisitor env_plane_visitor(env_plane, {}, {FindTaskEnvStatType});
uint64_t start_time =
env_plane_visitor.GetStat(TaskEnvStatType::kEnvProfileStartTime)
->IntOrUintValue();
const XPlane* host_plane = FindPlaneWithName(xspace, kHostThreadsPlaneName);
DCHECK_NE(host_plane, nullptr);
absl::flat_hash_map<int64_t /*correlation_id*/, CorrelationInfo>
correlation_info_map;
absl::flat_hash_map<uint32_t /*device_id*/, DeviceStepStats*>
sync_dev_stats_map;
XPlaneVisitor plane = tsl::profiler::CreateTfXPlaneVisitor(host_plane);
plane.ForEachLine([&](const XLineVisitor& line) {
uint32_t thread_id = line.Id();
line.ForEachEvent([&](const XEventVisitor& event) {
LaunchEventStats stats(&event);
if (event.Name() == "cuStreamSynchronize") {
if (stats.device_id.has_value()) {
uint32_t device_ordinal = stats.device_id.value();
DeviceStepStats* sync_dev_stats = sync_dev_stats_map[device_ordinal];
if (sync_dev_stats == nullptr) {
sync_dev_stats = step_stats->add_dev_stats();
sync_dev_stats->set_device(
absl::StrCat("/device:GPU:", device_ordinal, "/sync"));
}
NodeExecStats* ns = sync_dev_stats->add_node_stats();
SetNodeTimes(start_time, event, ns);
ns->set_node_name(event.Name());
ns->set_timeline_label(absl::StrCat("ThreadId ", thread_id));
ns->set_thread_id(thread_id);
}
} else if (stats.correlation_id.has_value()) {
int64_t correlation_id = stats.correlation_id.value();
uint64_t enqueue_time_ns = event.TimestampNs();
correlation_info_map[correlation_id] = {thread_id, enqueue_time_ns};
}
});
});
for (const XPlane* device_plane : device_planes) {
absl::flat_hash_map<std::pair<int64_t /*stream_id*/, GpuEventType>,
DeviceStepStats*>
stream_dev_stats_map;
DeviceStepStats* unknown_stream_dev_stats = nullptr;
DeviceStepStats* all_streams_dev_stats = nullptr;
DeviceStepStats* memcpy_dev_stats = nullptr;
XPlaneVisitor plane = tsl::profiler::CreateTfXPlaneVisitor(device_plane);
uint32_t device_ordinal = plane.Id();
plane.ForEachLine([&](const XLineVisitor& line) {
uint32_t stream_id = line.Id();
line.ForEachEvent([&](const XEventVisitor& event) {
GpuEventStats stats(&event);
auto ns = std::make_unique<NodeExecStats>();
SetNodeTimes(start_time, event, ns.get());
// Get launch information if available.
if (stats.correlation_id.has_value()) {
auto it = correlation_info_map.find(stats.correlation_id.value());
if (it != correlation_info_map.end()) {
const CorrelationInfo& correlation_info = it->second;
ns->set_scheduled_micros(
tsl::profiler::NanoToMicro(correlation_info.enqueue_time_ns));
ns->set_thread_id(correlation_info.thread_id);
}
}
absl::string_view node_name =
stats.IsTfOp() ? stats.tf_op_fullname : event.Name();
ns->set_node_name(node_name);
if (stats.IsKernel()) {
absl::string_view kernel_name = event.Name();
ns->set_timeline_label(
absl::StrCat(kernel_name, " ", stats.kernel_details));
DeviceStepStats*& stream_dev_stats =
stream_dev_stats_map[{stream_id, GpuEventType::kKernel}];
if (stream_dev_stats == nullptr) {
stream_dev_stats = step_stats->add_dev_stats();
stream_dev_stats->set_device(absl::StrCat(
"/device:GPU:", device_ordinal, "/stream:", stream_id));
}
*stream_dev_stats->add_node_stats() = *ns;
if (all_streams_dev_stats == nullptr) {
all_streams_dev_stats = step_stats->add_dev_stats();
all_streams_dev_stats->set_device(
absl::StrCat("/device:GPU:", device_ordinal, "/stream:all"));
}
all_streams_dev_stats->add_node_stats()->Swap(ns.get());
} else if (stats.IsMemCpy()) {
absl::string_view memcpy_name = event.Name();
ns->set_timeline_label(
absl::StrCat(memcpy_name, " ", stats.memcpy_details));
GpuEventType gpu_event_type = ParseMemcpyName(memcpy_name);
DCHECK_NE(gpu_event_type, GpuEventType::kUnknown);
DeviceStepStats*& stream_dev_stats =
stream_dev_stats_map[{stream_id, gpu_event_type}];
if (stream_dev_stats == nullptr) {
stream_dev_stats = step_stats->add_dev_stats();
stream_dev_stats->set_device(
absl::StrCat("/device:GPU:", device_ordinal,
"/stream:", stream_id, "<", memcpy_name, ">"));
}
*stream_dev_stats->add_node_stats() = *ns;
if (memcpy_dev_stats == nullptr) {
memcpy_dev_stats = step_stats->add_dev_stats();
memcpy_dev_stats->set_device(
absl::StrCat("/device:GPU:", device_ordinal, "/memcpy"));
}
memcpy_dev_stats->add_node_stats()->Swap(ns.get());
} else {
ns->set_timeline_label(node_name);
if (unknown_stream_dev_stats == nullptr) {
unknown_stream_dev_stats = step_stats->add_dev_stats();
unknown_stream_dev_stats->set_device(
absl::StrCat("/device:GPU:", device_ordinal, "/stream:"));
}
unknown_stream_dev_stats->add_node_stats()->Swap(ns.get());
}
});
});
}
}
} // namespace profiler
} // namespace tensorflow
tensorflow/core/profiler/convert/xplane_to_step_stats.h
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_XPLANE_TO_STEP_STATS_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_XPLANE_TO_STEP_STATS_H_
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tsl/profiler/protobuf/xplane.pb.h"
namespace tensorflow {
namespace profiler {
// Converts XSpace collected by profiling a GPU device to StepStats.
void ConvertGpuXSpaceToStepStats(const XSpace& xspace, StepStats* step_stats);
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_XPLANE_TO_STEP_STATS_H_
tensorflow/core/profiler/g3doc/advise.md
+108
View File
@@ -0,0 +1,108 @@
## Auto Detect and Advise
tfprof analyzes profiles and generates advice for common issues.
### Run Advise.
```python
# First create a profiler. See profiler tutorials for more details.
profiler = tf.profiler.Profiler(sess.graph)
run_meta = config_pb2.RunMetadata()
_ = sess.run(r1,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
profiler.add_step(1, run_meta)
# Then Start advise.
profiler.advise()
# For one-shot API
tf.profiler.advise(
sess.graph, run_meta=run_metadata)
```
```shell
# Run advisor on CLI
# See CLI tutorial on generating the files.
tfprof --graph_path=graph.pbtxt \
--run_meta_path=run_metadata \
--op_log_path=tfprof_log
tfprof> advise
AcceleratorUtilizationChecker:
device: /job:worker/replica:0/task:0/device:GPU:0 low utilization: 0.03
device: /job:worker/replica:0/task:0/device:GPU:1 low utilization: 0.08
device: /job:worker/replica:0/task:0/device:GPU:2 low utilization: 0.04
device: /job:worker/replica:0/task:0/device:GPU:3 low utilization: 0.21
OperationChecker:
Found operation using NHWC data_format on GPU. Maybe NCHW is faster.
ExpensiveOperationChecker:
top 1 operation type: SoftmaxCrossEntropyWithLogits, cpu: 1.37sec, accelerator: 0us, total: 1.37sec (26.68%)
top 2 operation type: MatMul, cpu: 427.39ms, accelerator: 280.76ms, total: 708.14ms (13.83%)
top 3 operation type: ConcatV2, cpu: 357.83ms, accelerator: 31.80ms, total: 389.63ms (7.61%)
seq2seq_attention_model.py:360:build_graph:self._add_seq2seq(), cpu: 3.16sec, accelerator: 214.84ms, total: 3.37sec
seq2seq_attention_model.py:293:_add_seq2seq:decoder_outputs, ..., cpu: 2.46sec, accelerator: 3.25ms, total: 2.47sec
seq2seq_lib.py:181:sampled_sequence_...:average_across_ti..., cpu: 2.46sec, accelerator: 3.24ms, total: 2.47sec
seq2seq_lib.py:147:sequence_loss_by_...:crossent = loss_f..., cpu: 2.46sec, accelerator: 3.06ms, total: 2.46sec
seq2seq_attention_model.py:289:sampled_loss_func:num_classes=vsize), cpu: 2.46sec, accelerator: 3.06ms, total: 2.46sec
seq2seq_attention_model.py:282:sampled_loss_func:labels = tf.resha..., cpu: 164us, accelerator: 0us, total: 164us
seq2seq_lib.py:148:sequence_loss_by_...:log_perp_list.app..., cpu: 1.33ms, accelerator: 120us, total: 1.45ms
seq2seq_lib.py:151:sequence_loss_by_...:total_size = tf.a..., cpu: 154us, accelerator: 23us, total: 177us
seq2seq_lib.py:184:sampled_sequence_...:return cost / tf...., cpu: 97us, accelerator: 8us, total: 105us
math_ops.py:690:cast:return gen_math_o..., cpu: 62us, accelerator: 3us, total: 65us
math_ops.py:839:binary_op_wrapper:return func(x, y,..., cpu: 35us, accelerator: 5us, total: 40us
seq2seq_attention_model.py:192:_add_seq2seq:sequence_length=a..., cpu: 651.56ms, accelerator: 158.92ms, total: 810.48ms
seq2seq_lib.py:104:bidirectional_rnn:sequence_length, ..., cpu: 306.58ms, accelerator: 73.54ms, total: 380.12ms
core_rnn.py:195:static_rnn:state_size=cell.s..., cpu: 306.52ms, accelerator: 73.54ms, total: 380.05ms
rnn.py:218:_rnn_step:_maybe_copy_some_..., cpu: 303.76ms, accelerator: 73.54ms, total: 377.30ms
rnn.py:216:_rnn_step:time >= max_seque..., cpu: 2.75ms, accelerator: 0us, total: 2.75ms
core_rnn.py:179:static_rnn:max_sequence_leng..., cpu: 67us, accelerator: 0us, total: 67us
seq2seq_lib.py:110:bidirectional_rnn:initial_state_bw,..., cpu: 296.21ms, accelerator: 73.54ms, total: 369.75ms
core_rnn.py:195:static_rnn:state_size=cell.s..., cpu: 296.11ms, accelerator: 73.54ms, total: 369.65ms
rnn.py:218:_rnn_step:_maybe_copy_some_..., cpu: 292.04ms, accelerator: 73.54ms, total: 365.58ms
rnn.py:216:_rnn_step:time >= max_seque..., cpu: 4.07ms, accelerator: 0us, total: 4.07ms
core_rnn.py:178:static_rnn:min_sequence_leng..., cpu: 85us, accelerator: 0us, total: 85us
core_rnn.py:179:static_rnn:max_sequence_leng..., cpu: 16us, accelerator: 0us, total: 16us
seq2seq_lib.py:113:bidirectional_rnn:outputs = [tf.con..., cpu: 46.88ms, accelerator: 3.87ms, total: 50.75ms
...(omitted)
top 1 graph node: seq2seq/loss/sampled_sequence_loss/sequence_loss_by_example/SoftmaxCrossEntropyWithLogits_11, cpu: 89.92ms, accelerator: 0us, total: 89.92ms
top 2 graph node: train_step/update_seq2seq/output_projection/w/ApplyAdam, cpu: 84.52ms, accelerator: 0us, total: 84.52ms
top 3 graph node: seq2seq/loss/sampled_sequence_loss/sequence_loss_by_example/SoftmaxCrossEntropyWithLogits_19, cpu: 73.02ms, accelerator: 0us, total: 73.02ms
```
### Checker
There is no magic behind advise mode. tfprof builds the profiles first, then
it runs through a list of `Checkers`, each one responsible for checking one
area with the profile and report issues. A `Checker` is like a plugin.
For example:
#### JobChecker (Not Available OSS)
* Checks RecvTensor RPC latency and bandwidth.
* Checks CPU/Memory utilization of the job.
#### AcceleratorUtilization Checker
* Checks what percentage of time the accelerator spends on computation.
#### OperationChecker
* Checks whether the operation runs with optimal options.
* Checks if there is a better implementation to replace the current operation.
#### ExpensiveOperationChecker
* Checks the most expensive operation type.
* Checks the most expensive graph nodes.
* Checks the most expensive graph-building Python codes.
#### Contribute Your Checker
Follow examples of accelerator_utilization_checker.h
tensorflow/core/profiler/g3doc/code_timeline.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 45 KiB

tensorflow/core/profiler/g3doc/command_line.md
+336
View File
@@ -0,0 +1,336 @@
## Command Line Interface Tutorials
* [Command Line Inputs](#command-line-inputs)
* [Start `tfprof`](#start-tfprof)
* [Examples](#examples)
* [Profile Python Time](#profile-python-time)
* [Profile Graph Time](#profile-graph-time)
* [Profile Checkpoint Value](#profile-checkpoint-value)
* [Profile Model Parameter](#profile-model-parameter)
* [Profile Device Placement](#profile-device-placement)
* [Define Customized Operation Type](#define-customized-operation-type)
* [Non-interactive Mode](#non-interactive-mode)
### Command Line Inputs
tfprof command line tool uses the following input:
<b>--profile_path:</b> A ProfileProto binary proto file.
See QuickStart on generating the file.
<b>THE OLD WAY BELOW IS DEPRECATED:</b>
<b>--graph_path:</b> GraphDef proto file (optional in eager execution).
Used to build in-memory
data structure of the model. For example, graph.pbtxt written by tf.Supervisor
can be passed to --graph_path. You can also easily get GraphDef using
tf.get_default_graph().as_graph_def(add_shapes=True) or other API.
<b>--run_meta_path:</b> RunMetadata proto file (optional).
Used to get the memory consumption and execution time of
each op of the model.
The following code snippet writes a RunMetadata file:
```python
run_options = config_pb2.RunOptions(trace_level=config_pb2.RunOptions.FULL_TRACE)
run_metadata = config_pb2.RunMetadata()
_ = self._sess.run(..., options=run_options, run_metadata=run_metadata)
with tf.gfile.Open(os.path.join(output_dir, "run_meta"), "w") as f:
f.write(run_metadata.SerializeToString())
```
<b>--op_log_path:</b>
tensorflow.tfprof.OpLogProto (optional). A proto used to provide extra operation
information. 1) float operations. 2) code traces. 3) define customized operation
type for `-account_type_regexes` option.
The following code snippet writes a OpLogProto file.
```python
tf.profiler.write_op_log(graph, log_dir, op_log=None)
```
<b>--checkpoint_path:</b> TensorFlow checkpoint (optional).
It defines _checkpoint_variable op type. It also provides checkpointed tensors' values.
Note: this feature is not well maintained now.
### Start `tfprof`
#### Build `tfprof`
```shell
# Build the tool.
bazel build --config opt tensorflow/core/profiler:profiler
# Help information, including detail 'option' instructions.
bazel-bin/tensorflow/core/profiler/profiler help
```
#### Start `tfprof` Interactive Mode
```shell
# The following commands will start tfprof interactive mode.
#
# Recommended:
#
# The file contains the binary string of ProfileProto.
# It contains all needed information in one file.
bazel-bin/tensorflow/core/profiler/profiler \
--profile_path=profile_xxx
#
# Alternatively, user can pass separate files.
#
# --graph_path contains the model architecture and tensor shapes.
# --run_meta_path contains the memory and time information.
# --op_log_path contains float operation and code traces.
# --checkpoint_path contains the model checkpoint data.
#
# Only includes model architecture, parameters and shapes.
bazel-bin/tensorflow/core/profiler/profiler \
--graph_path=graph.pbtxt
# For profiling eager execution, user can only specify run_meta_path
# and profile execution info of each operation.
bazel-bin/tensorflow/core/profiler/profiler \
--run_meta_path=run_meta
#
# Additionally profile ops memory and timing.
bazel-bin/tensorflow/core/profiler/profiler \
--graph_path=graph.pbtxt \
--run_meta_path=run_meta \
#
# tfprof_log is used to define customized op types, float ops and code traces.
# Use tfprof_logger.write_op_log() to create tfprof_log.
bazel-bin/tensorflow/core/profiler/profiler \
--graph_path=graph.pbtxt \
--run_meta_path=run_meta \
--op_log_path=tfprof_log \
#
# Additionally profile checkpoint statistics and values.
# Use '-account_type_regexes _checkpoint_variables' to select
# checkpoint tensors.
bazel-bin/tensorflow/core/profiler/profiler \
--graph_path=graph.pbtxt \
--run_meta_path=run_meta \
--op_log_path=tfprof_log \
--checkpoint_path=model.ckpt
```
#### Start `tfprof` Non-interactive Mode.
```python
# Runs tfprof in one-shot.
bazel-bin/tensorflow/core/profiler/profiler scope \
--graph_path=graph.pbtxt \
--max_depth=3
```
#### Press enter to show the default options
Refer to [Options](options.md) for option instructions.
```shell
tfprof>
-max_depth 4
-min_bytes 0
-min_micros 0
-min_params 0
-min_float_ops 0
-min_occurrence 0
-step -1
-order_by name
-account_type_regexes Variable,VariableV2
-start_name_regexes .*
-trim_name_regexes
-show_name_regexes .*
-hide_name_regexes IsVariableInitialized_[0-9]+,save\/.*,^zeros[0-9_]*
-account_displayed_op_only false
# supported select fields. Availability depends on --[run_meta|checkpoint|op_log]_path.
# [bytes|micros|params|float_ops|occurrence|tensor_value|device|op_types]
-select params
# format: output_type:key=value,key=value...
# output_types: stdout (default), timeline, file.
# key=value pairs:
# 1. timeline: outfile=<filename>
# 2. file: outfile=<filename>
# 3. stdout: None.
# E.g. timeline:outfile=/tmp/timeline.json
-output
```
### Examples
#### Profile Python Time
```shell
# Requires --graph_path --op_log_path
tfprof> code -max_depth 1000 -show_name_regexes .*model_analyzer.*py.* -select micros -account_type_regexes .* -order_by micros
_TFProfRoot (0us/22.44ms)
model_analyzer_test.py:149:run_filename_as_m...:none (0us/22.44ms)
model_analyzer_test.py:33:_run_code_in_main:none (0us/22.44ms)
model_analyzer_test.py:208:<module>:test.main() (0us/22.44ms)
model_analyzer_test.py:132:testComplexCodeView:x = lib.BuildFull... (0us/22.44ms)
model_analyzer_testlib.py:63:BuildFullModel:return sgd_op.min... (0us/21.83ms)
model_analyzer_testlib.py:58:BuildFullModel:cell, array_ops.c... (0us/333us)
model_analyzer_testlib.py:54:BuildFullModel:seq.append(array_... (0us/254us)
model_analyzer_testlib.py:42:BuildSmallModel:x = nn_ops.conv2d... (0us/134us)
model_analyzer_testlib.py:46:BuildSmallModel:initializer=init_... (0us/40us)
...
model_analyzer_testlib.py:61:BuildFullModel:loss = nn_ops.l2_... (0us/28us)
model_analyzer_testlib.py:60:BuildFullModel:target = array_op... (0us/0us)
model_analyzer_test.py:134:testComplexCodeView:sess.run(variable... (0us/0us)
```
Set ```-output timeline:outfile=<filename>``` to generate timeline instead of stdout.
<left>
![CodeTimeline](code_timeline.png)
</left>
#### Profile Graph Time
```shell
# I defined an op named ‘cost’ to calculate the loss. I want to know what ops
# it depends on take a long time to run.
# Requires --graph_path, --run_meta_path.
tfprof> graph -start_name_regexes cost.* -max_depth 100 -min_micros 10000 -select micros -account_type_regexes .*
_TFProfRoot (0us/3.61sec)
init/init_conv/Conv2D (11.75ms/3.10sec)
random_shuffle_queue_DequeueMany (3.09sec/3.09sec)
unit_1_0/sub2/conv2/Conv2D (74.14ms/3.19sec)
unit_1_3/sub2/conv2/Conv2D (60.75ms/3.34sec)
unit_2_4/sub2/conv2/Conv2D (73.58ms/3.54sec)
unit_3_3/sub2/conv2/Conv2D (10.26ms/3.60sec)
```
#### Profile Checkpoint Value
```shell
# Requires --graph_path, --checkpoint_path.
tfprof> scope -show_name_regexes unit_1_0.*gamma -select tensor_value -max_depth 5
_TFProfRoot ()
unit_1_0/shared_activation/init_bn/gamma ()
[1.80 2.10 2.06 1.91 2.26 1.86 1.81 1.37 1.78 1.85 1.96 1.54 2.04 2.34 2.22 1.99 ],
unit_1_0/sub2/bn2/gamma ()
[1.57 1.83 1.30 1.25 1.59 1.14 1.26 0.82 1.19 1.10 1.48 1.01 0.82 1.23 1.21 1.14 ],
```
#### Profile Model Parameter
```shell
# Show the number of parameters of all `tf.trainable_variables()` in the model.
# Requires --graph_path --op_log_path.
# store option for future commands.
tfprof> set -account_type_regexes _trainable_variables
tfprof> scope -max_depth 4 -select params
_TFProfRoot (--/464.15k params)
init/init_conv/DW (3x3x3x16, 432/432 params)
pool_logit/DW (64x10, 640/640 params)
pool_logit/biases (10, 10/10 params)
unit_last/final_bn/beta (64, 64/64 params)
unit_last/final_bn/gamma (64, 64/64 params)
```
Where does `_trainable_variables` come from? It is customized operation type
defined through the OpLogProto file.
Users can [Define Customized Operation Type](#define-customized-operation-type)
<b>Following example shows importance of defining customized operation type.</b>
In this example, extra `Variables` are created by TensorFlow
implicitly and “/Momentum” is appended to their names. They shouldn't be
included in you “model capacity” calculation.
```shell
tfprof> scope -account_type_regexes VariableV2 -max_depth 4 -select params
_TFProfRoot (--/930.58k params)
global_step (1/1 params)
init/init_conv/DW (3x3x3x16, 432/864 params)
pool_logit/DW (64x10, 640/1.28k params)
pool_logit/DW/Momentum (64x10, 640/640 params)
pool_logit/biases (10, 10/20 params)
pool_logit/biases/Momentum (10, 10/10 params)
unit_last/final_bn/beta (64, 64/128 params)
unit_last/final_bn/gamma (64, 64/128 params)
unit_last/final_bn/moving_mean (64, 64/64 params)
unit_last/final_bn/moving_variance (64, 64/64 params)
```
#### Profile Device Placement
In this tutorial, a model is split
on several gpus at workers and several parameter servers.
In tfprof, 'device' is an op_type. For example, if op1 and op2 are placed on
gpu:0. They share an operation type.
```shell
bazel-bin/tensorflow/core/profiler/profiler \
--graph_path=/tmp/graph.pbtxt \
--run_meta_path=/tmp/run_meta
# Looks like ps task 1 is holding twice more parameters than task 0.
tfprof> scope -select device,params -account_type_regexes .*ps.*task:0.* -max_depth 1
_TFProfRoot (--/25.81m params)
tfprof> scope -select device,params -account_type_regexes .*ps.*task:1.* -max_depth 1
_TFProfRoot (--/58.84m params)
```
#### Define Customized Operation Type
First, in Python code, create an `OpLogProto` proto and add op type
information to it:
```python
op_log = tfprof_log_pb2.OpLogProto()
entry = op_log.log_entries.add()
entry.name = 'pool_logit/DW'
entry.types.append('pool_logit')
entry = op_log.log_entries.add()
entry.name = 'pool_logit/biases'
entry.types.append('pool_logit')
```
Second, call write_op_log to write the OpLogProto proto.
```python
tf.profiler.write_op_log(
sess.graph, /tmp/my_op_log_dir, op_log)
# Get run-time shape information in order to fill shapes and get flops.
tf.profiler.write_op_log(
sess.graph, /tmp/my_op_log_dir, op_log, run_meta)
```
Third, when starting the tfprof tool, specify
"--op_log_path=/tmp/my_op_log_dir/op_log"
```shell
tfprof> scope -account_type_regexes pool_logit -max_depth 4 -select params
_TFProfRoot (--/650 params)
pool_logit/DW (64x10, 640/640 params)
pool_logit/biases (10, 10/10 params)
```
Note that `tf.profiler.write_op_log(...)` automatically
assigns all `Variables` inside `tf.trainable_variables()` a customized
operation type: `_trainable_variables`.
#### Non-interactive Mode
12) Run tfprof in one-shot mode and dump result to file.
```shell
# By default output to stdout. Use -output option to change output types.
tfprof scope --graph_path=graph.pbtxt \
--max_depth=3 \
--output="file:outfile=/tmp/dump"
Reading Files...
Parsing GraphDef...
Preparing Views...
cat /tmp/dump
_TFProfRoot (--/930.58k params)
global_step (0/0 params)
pool_logit/DW (64x10, 640/1.28k params)
pool_logit/biases (10, 10/20 params)
```
tensorflow/core/profiler/g3doc/graph_timeline.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 91 KiB

tensorflow/core/profiler/g3doc/options.md
+135
View File
@@ -0,0 +1,135 @@
## Options
### Overview
For all tfprof views, the profiles are processed with the following procedures
1) An in-memory data structure is built represent the view.
* graph view. Graph. Each profiler node corresponds to a
TensorFlow graph node.
* scope view. Tree. Each profiler node corresponds to a
TensorFlow graph node.
* code view. Tree. Each profiler node includes to all TensorFlow
graph nodes created by the profiler node (python code).
* op view. List. Each profiler node includes to all TensorFlow
graph nodes belonging to an operation type.
2) `-account_type_regexes` is used to first select the nodes that includes
the specified operation types. An operation has its default type
(e.g. MatMul, Conv2D). `tfprof` also considers device as operation type.
User can also define customized operation type. Hence, an operation has
multiple types. Profiler nodes containing matched
types are selected for display and their statistics are aggregated by the
parents of the in-memory data structure.
3) Various `-xxx_name_regexes`, `-min_xxx`, `-max_depth` etc options are then
applied to further filter based on profiler node names and statistics.
It's no limited operation name. In code view,
it's the code string. In op view, it's the operation type name. Different
from `-account_type_regexes`, Statistics are used even if a profiler node is not displayed.
For example, in code view, a callee might be hidden, but its statistics is
still aggregated by it's caller. `-account_displayed_op_only`, however,
breaks the rule and only aggregates statistics of displayed names.
4) Finally, the filtered data structure is output in a format depending
on the `-output` option.
#### Option Semantics In Different View
options usually have the same semantics in different views. However, some
can vary. For example `-max_depth` in scope view means the depth of
name scope <b>tree</b>. In op view, it means the length of operation <b>list</b>.
In graph view, in means the number of hops in the <b>graph</b>.
### Times
Most machines have multi-core CPUs. Some installs one or more accelerators.
Each accelerator usually performs massive parallel processing. The profiler
tracks the accumulated processing times. Hence, the accumulated processing
time is likely larger than the time of each step.
micros: This is the sum of cpu and accelerator times.
accelerator_micros: This is the accelerator times.
cpu_micros: This is the cpu times.
### Memory
Tensor memory are usually ref-counted. The memory is released when there is no
more reference to it. It will be difficult to track the release of memory.
Currently, profiler only tracks the allocation of memory. As a result, the
accumulated memory request is usually larger than the peak memory of the overall
model.
It's recommended to generate timeline to see the allocator memory usage over
time.
`bytes`: The memory allocations requested by the operation.
`peak_bytes`: The peak requested memory (not de-allocated) by the operation.
`residual_bytes`: The memory requested by the operation and not de-allocated
when Compute finishes.
`output_bytes`: The memory output by the operation. It's not necessarily requested
by the current operation. For example, it can be a tensor
forwarded from input to output, with in-place mutation.
### Docs
`-max_depth`: Show nodes that are at most this number of hops from starting node in the data structure.
`-min_bytes`: Show nodes that request at least this number of bytes.
`-min_peak_bytes`: Show nodes that using at least this number of bytes during peak memory usage.
`-min_residual_bytes`: Show nodes that have at least this number of bytes not being de-allocated after Compute.
`-min_output_bytes`: Show nodes that have at least this number of bytes output (no necessarily allocated by the nodes).
`-min_micros`: Show nodes that spend at least this number of microseconds to run. It sums
accelerator_micros and cpu_micros. Note: cpu and accelerator can run in parallel.
`-min_accelerator_micros`: Show nodes that spend at least this number of microseconds to run on accelerator (e.g. GPU).
`-min_cpu_micros`: Show nodes that spend at least this number of microseconds to run on CPU.
`-min_params`: Show nodes that contains at least this number of parameters.
`-min_float_ops`: Show nodes that contain at least this number of float operations. Only available if an node has op.RegisterStatistics() defined and OpLogProto is provided
`-min_occurrence`: Show nodes that appear at least this number of times..
`-step`: Show the stats of the this step when multiple steps of RunMetadata were added. By default, show the average of all steps."
`-order_by`: Order the results by [name|depth|bytes|peak_bytes|residual_bytes|output_bytes|micros|accelerator_micros|cpu_micros|params|float_ops|occurrence]
`-account_type_regexes`: Account and display the nodes whose types match one of
the type regexes specified. tfprof allow user to define extra operation types
for graph nodes through tensorflow.tfprof.OpLogProto proto. regexes are
comma-separated.
`-start_name_regexes`: Show node starting from the node that matches the regexes, recursively. regexes are comma-separated.
`-trim_name_regexes`: Hide node starting from the node that matches the regexes,
recursively, regexes are comma-separated.
`-show_name_regexes`: Show node that match the regexes. regexes are
comma-separated.
`-hide_name_regexes`: Hide node that match the regexes. regexes are
comma-separated.
`-account_displayed_op_only`: If True, only account the statistics of ops eventually displayed. If False, account all op statistics matching -account_type_regexes recursively.
Notes: See <b>overview</b> session on how does above options play with each
other to decide the output and counting.
`-select`: Comma-separated list of attributes to show. Supported attributes:
[bytes|peak_bytes|residual_bytes|output_bytes|micros|accelerator_micros|cpu_micros|params|float_ops|occurrence|tensor_value|device|op_types|input_shapes].
`-output`: Output results as stdout, file or timeline.
The format is ```output_type:key=value,key=value```.
For example: ```-output timeline:outfile=<filename>```.
```shell
timeline: key=outfile, value=<filename>.
stdout: none.
file: key=outfile, value=<filename>.
```
tensorflow/core/profiler/g3doc/pprof.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 249 KiB

tensorflow/core/profiler/g3doc/profile_memory.md
+77
View File
@@ -0,0 +1,77 @@
## Profile Memory
It is generally a good idea to visualize the memory usage in timeline.
It allows you to see the memory consumption of each GPU over time.
```python
#To get memory information, you need --graph_path and --run_meta_path
tfprof> graph -max_depth 10000000 -step 0 -account_type_regexes .* -output timeline:outfile=<filename>
generating trace file.
******************************************************
Timeline file is written to <filename>
Open a Chrome browser, enter URL chrome://tracing and load the timeline file.
******************************************************
```
![Timeline](graph_timeline.png)
```python
# You can also visualize the memory information through other methods.
# With op view, it shows you the aggregated output tensor bytes of each
# operation type.
tfprof> op -select bytes -order_by bytes
node name | requested bytes
Identity 32515.37MB (100.00%, 27.02%)
FusedBatchNormGrad 10802.14MB (72.98%, 8.98%)
FusedBatchNorm 10517.52MB (64.01%, 8.74%)
Conv2D 10509.25MB (55.27%, 8.73%)
Conv2DBackpropInput 9701.39MB (46.54%, 8.06%)
ReluGrad 9206.45MB (38.48%, 7.65%)
Relu 8462.80MB (30.83%, 7.03%)
DepthwiseConv2dNativeBackpropInput 7899.35MB (23.80%, 6.56%)
DepthwiseConv2dNative 7425.17MB (17.23%, 6.17%)
MaxPoolGrad 3015.44MB (11.06%, 2.51%)
AddN 2741.49MB (8.56%, 2.28%)
# With scope view, you can see the operations that outputs largest tensors.
tfprof> scope -order_by bytes -select bytes -min_bytes 100000000
node name | requested bytes
_TFProfRoot (--/120356.38MB)
tower_3/SepConv2d_2b_3x3/separable_conv2d (346.85MB/854.00MB)
tower_3/SepConv2d_2b_3x3/separable_conv2d/depthwise (507.15MB/507.15MB)
tower_0/SepConv2d_2b_3x3/separable_conv2d (346.85MB/693.71MB)
tower_0/SepConv2d_2b_3x3/separable_conv2d/depthwise (346.85MB/346.85MB)
tower_2/SepConv2d_2b_3x3/separable_conv2d (346.85MB/693.71MB)
tower_2/SepConv2d_2b_3x3/separable_conv2d/depthwise (346.85MB/346.85MB)
tower_1/SepConv2d_2b_3x3/separable_conv2d (346.85MB/693.71MB)
tower_1/SepConv2d_2b_3x3/separable_conv2d/depthwise (346.85MB/346.85MB)
tower_3/SepConv2d_2a_3x3/separable_conv2d (346.85MB/520.28MB)
tower_3/SepConv2d_2a_3x3/separable_conv2d/depthwise (173.43MB/173.43MB)
tower_2/SepConv2d_2a_3x3/separable_conv2d (346.85MB/520.28MB)
tower_2/SepConv2d_2a_3x3/separable_conv2d/depthwise (173.43MB/173.43MB)
tower_0/SepConv2d_2a_3x3/separable_conv2d (346.85MB/520.28MB)
tower_0/SepConv2d_2a_3x3/separable_conv2d/depthwise (173.43MB/173.43MB)
...
# code view.
tfprof> code -max_depth 10 -select bytes -order_by bytes -start_name_regexes .*seq2seq.* -min_bytes 1
node name | requested bytes
_TFProfRoot (--/74148.60MB)
seq2seq_attention.py'>:168:run_filename_from...:none (0B/74148.60MB)
seq2seq_attention.py'>:33:_run_code_in_main:none (0B/74148.60MB)
seq2seq_attention.py:316:<module>:app.run() (0B/74148.60MB)
app.py:432:run:_run_main(main or... (0B/74148.60MB)
app.py:352:_run_main:sys.exit(main(arg... (0B/74148.60MB)
seq2seq_attention.py:270:main:_Train(model, bat... (0B/74148.60MB)
seq2seq_attention.py:128:_Train:model.build_graph() (0B/74148.60MB)
seq2seq_attention_model.py:363:build_graph:self._add_train_o... (0B/48931.86MB)
seq2seq_attention_model.py:307:_add_train_op:tf.gradients(self... (0B/46761.06MB)
seq2seq_attention_model.py:322:_add_train_op:zip(grads, tvars)... (0B/2170.80MB)
seq2seq_attention_model.py:312:_add_train_op:tf.train.exponent... (0B/2.56KB)
seq2seq_attention_model.py:308:_add_train_op:tf.summary.scalar... (0B/64B)
seq2seq_attention_model.py:320:_add_train_op:tf.summary.scalar... (0B/64B)
seq2seq_attention_model.py:360:build_graph:self._add_seq2seq() (0B/25216.74MB)
seq2seq_attention_model.py:192:_add_seq2seq:sequence_length=a... (0B/21542.55MB)
```
tensorflow/core/profiler/g3doc/profile_model_architecture.md
+92
View File
@@ -0,0 +1,92 @@
## Profile Model Architecture
* [Profile Model Parameters](#profile-model-parameters)
* [Profile Model Float Operations](#profile-model-float-operations)
### Profile Model Parameters
<b>Notes:</b>
`VariableV2` operation type might contain variables created by TensorFlow
implicitly. User normally don't want to count them as "model capacity".
We can use customized operation type to select a subset of variables.
For example `_trainable_variables` is created automatically by tfprof Python
API. User can also define customized operation type.
```
# parameters are created by operation type 'VariableV2' (For older model,
# it's 'Variable'). scope view is usually suitable in this case.
tfprof> scope -account_type_regexes VariableV2 -max_depth 4 -select params
_TFProfRoot (--/930.58k params)
global_step (1/1 params)
init/init_conv/DW (3x3x3x16, 432/864 params)
pool_logit/DW (64x10, 640/1.28k params)
pool_logit/DW/Momentum (64x10, 640/640 params)
pool_logit/biases (10, 10/20 params)
pool_logit/biases/Momentum (10, 10/10 params)
unit_last/final_bn/beta (64, 64/128 params)
unit_last/final_bn/gamma (64, 64/128 params)
unit_last/final_bn/moving_mean (64, 64/64 params)
unit_last/final_bn/moving_variance (64, 64/64 params)
# The Python API profiles tf.trainable_variables() instead of VariableV2.
#
# By default, it's printed to stdout. User can update options['output']
# to write to file. The result is always returned as a proto buffer.
param_stats = tf.profiler.profile(
tf.get_default_graph(),
options=tf.profiler.ProfileOptionBuilder
.trainable_variables_parameter())
sys.stdout.write('total_params: %d\n' % param_stats.total_parameters)
```
### Profile Model Float Operations
#### Caveats
For an operation to have float operation statistics:
* It must have `RegisterStatistics('flops')` defined in TensorFlow. tfprof
uses the definition to calculate float operations. Contributions are
welcomed.
* It must have known "shape" information for RegisterStatistics('flops') to
calculate the statistics. It is suggested to pass in `-run_meta_path` if
shape is only known during runtime. tfprof can fill in the missing shape
with the runtime shape information from RunMetadata. Hence, it is suggested
to use `-account_displayed_op_only` option so that you know the statistics
are only for the operations printed out.
* If no RunMetadata is provided, tfprof counts float_ops of each graph node
once, even if it is defined in a tf.while_loop. This is because tfprof
doesn't know statically how many times each graph node is run. If
RunMetadata is provided, tfprof calculates float_ops as float_ops *
run_count.
```python
# To profile float operations in commandline, you need to pass --graph_path
# and --op_log_path.
tfprof> scope -min_float_ops 1 -select float_ops -account_displayed_op_only
node name | # float_ops
_TFProfRoot (--/17.63b flops)
gradients/pool_logit/xw_plus_b/MatMul_grad/MatMul (163.84k/163.84k flops)
gradients/pool_logit/xw_plus_b/MatMul_grad/MatMul_1 (163.84k/163.84k flops)
init/init_conv/Conv2D (113.25m/113.25m flops)
pool_logit/xw_plus_b (1.28k/165.12k flops)
pool_logit/xw_plus_b/MatMul (163.84k/163.84k flops)
unit_1_0/sub1/conv1/Conv2D (603.98m/603.98m flops)
unit_1_0/sub2/conv2/Conv2D (603.98m/603.98m flops)
unit_1_1/sub1/conv1/Conv2D (603.98m/603.98m flops)
unit_1_1/sub2/conv2/Conv2D (603.98m/603.98m flops)
# Some might prefer op view that aggregate by operation type.
tfprof> op -min_float_ops 1 -select float_ops -account_displayed_op_only -order_by float_ops
node name | # float_ops
Conv2D 17.63b float_ops (100.00%, 100.00%)
MatMul 491.52k float_ops (0.00%, 0.00%)
BiasAdd 1.28k float_ops (0.00%, 0.00%)
# You can also do that in Python API.
tf.profiler.profile(
tf.get_default_graph(),
options=tf.profiler.ProfileOptionBuilder.float_operation())
```
tensorflow/core/profiler/g3doc/profile_time.md
+179
View File
@@ -0,0 +1,179 @@
## Profile Time
* [Times in TensorFlow and tfprof](#times-in-tensorflow-and-tfprof)
* [Profile by Python Code](#profile-by-python-code)
* [Profile by Operation Type](#profile-by-operation-type)
* [Profile by Graph](#profile-by-graph)
* [Profile by Name Scope](#profile-by-name-scope)
### Times in TensorFlow and tfprof
When we run a model, Tensorflow schedules and runs the nodes (operations)
in the graph. An operation can be placed on an accelerator or on CPU.
#### On Accelerator
When an operation is placed on accelerator, it will first be scheduled
by TensorFlow on CPU. Normally, it's the code in OpKernel::Compute.
OpKernel::Compute can decide to dispatch some of the computations on the
accelerator. While some computation (e.g. pre-processing) is still done
in CPU. OpKernel::Compute can dispatch computation on accelerator
and return, or it can also wait for the accelerator to finish.
tfprof reports 3 execution times:
* <b>accelerator_micros</b>, which is the part of computation time spent on accelerator.
* <b>cpu_micros</b>, which is the part of computation time spent on cpu, including
any wait times that might happen if OpKernel::Compute decides to wait.
* <b>exec_micros</b>, which is the sum of accelerator_micros and cpu_micros.
Since accelerator, such as GPU, usually runs operation asynchronously, you
might notice an operation finishes on cpu before it starts running on
accelerator.
#### On CPU
When an operation is placed on CPU, it will completely run on CPU. Hence,
<b>exec_micros</b> is equal to <b>cpu_micros</b> and <b>accelerator_micros</b>
should be 0.
### Profile by Python Code
```python
# In code view, the time of each line of Python code is the aggregated
# times of all operations created by that line.
# In command line, it requires --graph_path --op_log_path and --run_meta_path.
# --op_log_path provides the code traces information.
# --run_meta_path provides the time information.
tfprof> code -show_name_regexes seq2seq_attention.* -max_depth 10 -select micros -order_by micros
node name | execution time
_TFProfRoot (--/3.74sec)
seq2seq_attention.py'>:168:run_filename_from...:none (0us/3.74sec)
seq2seq_attention.py'>:33:_run_code_in_main:none (0us/3.74sec)
seq2seq_attention.py:316:<module>:app.run() (0us/3.74sec)
seq2seq_attention.py:270:main:_Train(model, bat... (0us/3.74sec)
seq2seq_attention.py:128:_Train:model.build_graph() (0us/3.74sec)
seq2seq_attention_model.py:360:build_graph:self._add_seq2seq() (0us/2.79sec)
seq2seq_attention_model.py:293:_add_seq2seq:decoder_outputs, ... (0us/2.46sec)
seq2seq_attention_model.py:192:_add_seq2seq:sequence_length=a... (0us/265.31ms)
seq2seq_attention_model.py:253:_add_seq2seq:initial_state_att... (0us/50.35ms)
seq2seq_attention_model.py:173:_add_seq2seq:for x in encoder_... (0us/8.72ms)
seq2seq_attention_model.py:218:_add_seq2seq:w_t = tf.transpos... (0us/2.39ms)
...
seq2seq_attention_model.py:363:build_graph:self._add_train_o... (0us/949.10ms)
seq2seq_attention_model.py:307:_add_train_op:tf.gradients(self... (0us/641.44ms)
seq2seq_attention_model.py:322:_add_train_op:zip(grads, tvars)... (0us/307.56ms)
...
seq2seq_attention_model.py:364:build_graph:self._summaries =... (0us/13us)
seq2seq_attention_model.py:361:build_graph:self.global_step ... (0us/12us)
...
seq2seq_attention.py:129:_Train:saver = tf.train.... (0us/0us)
seq2seq_attention.py:140:_Train:global_step=model... (0us/0us)
# Sometimes you want to explore a specific function. You can do that
# with -start_name_regexes.
tfprof> code -start_name_regexes .*_add_seq2seq.* -show_name_regexes seq2seq_attention.* -max_depth 10 -select micros -order_by micros
node name | execution time
_TFProfRoot (--/3.74sec)
seq2seq_attention_model.py:360:build_graph:self._add_seq2seq() (0us/2.79sec)
seq2seq_attention_model.py:293:_add_seq2seq:decoder_outputs, ... (0us/2.46sec)
seq2seq_attention_model.py:289:sampled_loss_func:num_classes=vsize) (0us/2.46sec)
seq2seq_attention_model.py:282:sampled_loss_func:labels = tf.resha... (0us/164us)
# You can also dive deeper into tensorflow's libraries.
tfprof> code -max_depth 5 -select micros -order_by micros -start_name_regexes .*_add_seq2seq.* -min_micros 100000
_TFProfRoot (--/3.74sec)
seq2seq_attention_model.py:360:build_graph:self._add_seq2seq() (0us/2.79sec)
seq2seq_attention_model.py:293:_add_seq2seq:decoder_outputs, ... (0us/2.46sec)
seq2seq_lib.py:181:sampled_sequence_...:average_across_ti... (0us/2.46sec)
seq2seq_lib.py:147:sequence_loss_by_...:crossent = loss_f... (0us/2.46sec)
seq2seq_attention_model.py:192:_add_seq2seq:sequence_length=a... (0us/265.31ms)
seq2seq_lib.py:104:bidirectional_rnn:sequence_length, ... (0us/127.27ms)
core_rnn.py:195:static_rnn:state_size=cell.s... (0us/127.20ms)
seq2seq_lib.py:110:bidirectional_rnn:initial_state_bw,... (0us/125.96ms)
core_rnn.py:195:static_rnn:state_size=cell.s... (0us/125.86ms)
# It can also be done in Python API
opts = model_analyzer.TRAINABLE_VARS_PARAMS_STAT_OPTIONS.copy()
opts['account_type_regexes'] = ['.*']
opts['show_name_regexes'] = ['.*model_analyzer_testlib.py.*']
opts['account_displayed_op_only'] = False
opts['select'] = ['micros']
tfprof_node = model_analyzer.print_model_analysis(
sess.graph, run_meta, cmd='code', options=opts)
```
You can generate some visualization in code view:
Set ```-output timeline:outfile=<filename>``` to generate timeline instead of stdout.
<left>
![CodeTimeline](code_timeline.png)
</left>
### Profile by Operation Type
```python
# In op view, you can view the aggregated time of each operation type.
tfprof> op -select micros,occurrence -order_by micros
node name | execution time | op occurrence
SoftmaxCrossEntropyWithLogits 1.37sec (100.00%, 36.44%), 30
MatMul 618.97ms (63.56%, 16.51%), 3450
Add 273.76ms (47.06%, 7.30%), 2180
Sub 215.41ms (39.76%, 5.74%), 4372
ConcatV2 203.88ms (34.01%, 5.44%), 6098
Mul 134.32ms (28.58%, 3.58%), 9427
ApplyAdam 92.66ms (25.00%, 2.47%), 27
Switch 72.43ms (22.53%, 1.93%), 30654
LogUniformCandidateSampler 69.01ms (20.59%, 1.84%), 30
Unique 53.50ms (18.75%, 1.43%), 2
AddN 50.10ms (17.33%, 1.34%), 5481
# You might be surprised to see that SoftmaxCrossEntropyWithLogits is
# that expensive. As shown below, it is placed on cpu.
tfprof> op -select micros,device -order_by micros
node name | execution time | assigned devices
SoftmaxCrossEntropyWithLogits 1.37sec (100.00%, 36.44%), /job:worker/replica:0/task:0/cpu:0
MatMul 618.97ms (63.56%, 16.51%), |/job:worker/replica:0/task:0/cpu:0|/job:worker/replica:0/task:0/device:GPU:0|/job:worker/replica:0/task:0/device:GPU:1|/job:worker/replica:0/task:0/device:GPU:2|/job:worker/replica:0/task:0/device:GPU:3
```
### Profile by Graph
Usually, use graph view to generate a timeline to visualize the result.
In the chrome://tracing UI, click "Flow Event" in "View Options" of upper
right corner to see the flow of tensors.
<left>
TODO(xpan): Show the image correctly in github.
![Timeline](graph_timeline.png)
</left>
tfprof options allow users to generate timeline in some advanced ways.
```python
# Only generate timeline for gpu3 and cpu on workers.
graph -max_depth 10000000 -step 0 -account_type_regexes .*gpu:3.*,.*worker.*cpu:0.* -output timeline:outfile=<filename.json>
generating trace file.
******************************************************
Timeline file is written to <filename.json>.
Open a Chrome browser, enter URL chrome://tracing and load the timeline file.
******************************************************
```
### Profile by Name Scope
Usually scope view allows you to pin point the problematic places if you
have properly named your operations with tf.name_scope or tf.variable_scope.
```python
tfprof> scope -max_depth 30 -select micros -min_micros 100000 -order_by micros
node name | execution time
_TFProfRoot (--/8.12sec)
tower_3/gradients/tower_3/Conv2d_1a_3x3/convolution_grad/Conv2DBackpropFilter (126.34ms/126.34ms)
tower_1/gradients/tower_1/Conv2d_1a_3x3/convolution_grad/Conv2DBackpropFilter (125.44ms/125.44ms)
tower_2/gradients/tower_2/Conv2d_1a_3x3/convolution_grad/Conv2DBackpropFilter (124.85ms/124.85ms)
tower_0/gradients/tower_0/Conv2d_1a_3x3/convolution_grad/Conv2DBackpropFilter (124.45ms/124.45ms)
```
tensorflow/core/profiler/g3doc/profiler_ui.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 186 KiB

tensorflow/core/profiler/g3doc/python_api.md
+151
View File
@@ -0,0 +1,151 @@
## Python API Tutorials
* [Parameters and Shapes](#parameters-and-shapes)
* [Float Operations](#float-operations)
* [Time and Memory](#time-and-memory)
* [Visualize](#visualize)
* [Multi-step Profiling](#multi-step-profiling)
```import tensorflow as tf```.
### Parameters and Shapes.
```python
# Print trainable variable parameter statistics to stdout.
ProfileOptionBuilder = tf.profiler.ProfileOptionBuilder
param_stats = tf.profiler.profile(
tf.get_default_graph(),
options=ProfileOptionBuilder.trainable_variables_parameter())
# Use code view to associate statistics with Python codes.
opts = ProfileOptionBuilder(
ProfileOptionBuilder.trainable_variables_parameter()
).with_node_names(show_name_regexes=['.*my_code1.py.*', '.*my_code2.py.*']
).build()
param_stats = tf.profiler.profile(
tf.get_default_graph(),
cmd='code',
options=opts)
# param_stats can be tensorflow.tfprof.GraphNodeProto or
# tensorflow.tfprof.MultiGraphNodeProto, depending on the view.
# Let's print the root below.
sys.stdout.write('total_params: %d\n' % param_stats.total_parameters)
```
### Float Operations
#### Note: See [Caveats](profile_model_architecture.md#caveats) in "Profile Model Architecture" Tutorial
``` python
# Print to stdout an analysis of the number of floating point operations in the
# model broken down by individual operations.
tf.profiler.profile(
tf.get_default_graph(),
options=tf.profiler.ProfileOptionBuilder.float_operation())
```
### Time and Memory
You will first need to run the following set up in your model in order to
compute the memory and timing statistics.
```python
# Generate the RunMetadata that contains the memory and timing information.
#
# Note: When run on accelerator (e.g. GPU), an operation might perform some
# cpu computation, enqueue the accelerator computation. The accelerator
# computation is then run asynchronously. The profiler considers 3
# times: 1) accelerator computation. 2) cpu computation (might wait on
# accelerator). 3) the sum of 1 and 2.
#
run_metadata = tf.RunMetadata()
with tf.Session() as sess:
_ = sess.run(train_op,
options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE),
run_metadata=run_metadata)
```
Finally, you may run `tf.profiler.profile` to explore the timing and memory
information of the model.
``` python
# Print to stdout an analysis of the memory usage and the timing information
# broken down by python codes.
ProfileOptionBuilder = tf.profiler.ProfileOptionBuilder
opts = ProfileOptionBuilder(ProfileOptionBuilder.time_and_memory()
).with_node_names(show_name_regexes=['.*my_code.py.*']).build()
tf.profiler.profile(
tf.get_default_graph(),
run_meta=run_metadata,
cmd='code',
options=opts)
# Print to stdout an analysis of the memory usage and the timing information
# broken down by operation types.
tf.profiler.profile(
tf.get_default_graph(),
run_meta=run_metadata,
cmd='op',
options=tf.profiler.ProfileOptionBuilder.time_and_memory())
```
### Visualize
```
To visualize the result of Python API results:
Call `with_step(0).with_timeline_output(filename)` to generate a timeline json file.
Open a Chrome Browser, type URL `chrome://tracing`, and load the json file.
```
Below are 2 examples of graph view and scope view.
<left>
![CodeTimeline](graph_timeline.png)
![CodeTimeline](scope_timeline.png)
</left>
### Multi-step Profiling
tfprof allows you to profile statistics across multiple steps.
```python
opts = model_analyzer.PRINT_ALL_TIMING_MEMORY.copy()
opts['account_type_regexes'] = ['.*']
with session.Session() as sess:
r1, r2, r3 = lib.BuildSplittableModel()
sess.run(variables.global_variables_initializer())
# Create a profiler.
profiler = model_analyzer.Profiler(sess.graph)
# Profile without RunMetadata of any step.
pb0 = profiler.profile_name_scope(opts)
run_meta = config_pb2.RunMetadata()
_ = sess.run(r1,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta)
# Add run_meta of step 1.
profiler.add_step(1, run_meta)
pb1 = profiler.profile_name_scope(opts)
run_meta2 = config_pb2.RunMetadata()
_ = sess.run(r2,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta2)
# Add run_meta of step 2.
profiler.add_step(2, run_meta2)
pb2 = profiler.profile_name_scope(opts)
run_meta3 = config_pb2.RunMetadata()
_ = sess.run(r3,
options=config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE),
run_metadata=run_meta3)
# Add run_meta of step 3.
profiler.add_step(3, run_meta3)
pb3 = profiler.profile_name_scope(opts)
```
tensorflow/core/profiler/g3doc/scope_timeline.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 24 KiB

tensorflow/core/profiler/internal/BUILD
+337
View File
@@ -0,0 +1,337 @@
load("//tensorflow:tensorflow.bzl", "if_not_windows", "tf_cc_test")
load("//tensorflow/core/platform:rules_cc.bzl", "cc_library")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:internal"],
licenses = ["notice"],
)
cc_library(
name = "tfprof_stats",
srcs = ["tfprof_stats.cc"],
hdrs = ["tfprof_stats.h"],
deps = [
":tfprof_code",
":tfprof_graph",
":tfprof_node",
":tfprof_op",
":tfprof_scope",
":tfprof_show",
":tfprof_timeline",
":tfprof_utils",
"//tensorflow/c:checkpoint_reader",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/platform:regexp",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "tfprof_timeline",
srcs = ["tfprof_timeline.cc"],
hdrs = ["tfprof_timeline.h"],
deps = [
":tfprof_node_show",
":tfprof_utils",
"//tensorflow/core:lib",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@jsoncpp_git//:jsoncpp",
],
)
cc_library(
name = "tfprof_node",
srcs = ["tfprof_node.cc"],
hdrs = ["tfprof_node.h"],
deps = [
":tfprof_utils",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/platform:regexp",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "tfprof_scope",
srcs = ["tfprof_scope.cc"],
hdrs = ["tfprof_scope.h"],
deps = [
":tfprof_constants",
":tfprof_node",
":tfprof_show",
":tfprof_tensor",
":tfprof_utils",
"//tensorflow/c:checkpoint_reader",
"//tensorflow/core:lib",
"//tensorflow/core/platform:regexp",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "tfprof_op",
srcs = ["tfprof_op.cc"],
hdrs = ["tfprof_op.h"],
deps = [
":tfprof_constants",
":tfprof_node",
":tfprof_show_multi",
":tfprof_tensor",
":tfprof_utils",
"//tensorflow/core:lib",
"//tensorflow/core/platform:regexp",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "tfprof_code",
srcs = ["tfprof_code.cc"],
hdrs = ["tfprof_code.h"],
deps = [
":tfprof_constants",
":tfprof_node",
":tfprof_show_multi",
":tfprof_timeline",
":tfprof_utils",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core/platform:regexp",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@tsl//tsl/profiler/protobuf:profile_proto_cc",
],
)
cc_library(
name = "tfprof_graph",
srcs = ["tfprof_graph.cc"],
hdrs = ["tfprof_graph.h"],
deps = [
":tfprof_constants",
":tfprof_node",
":tfprof_show",
":tfprof_tensor",
":tfprof_utils",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/platform:regexp",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "tfprof_node_show",
srcs = ["tfprof_node_show.cc"],
hdrs = ["tfprof_node_show.h"],
deps = [
":tfprof_constants",
":tfprof_node",
":tfprof_utils",
"//tensorflow/core:lib",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
],
)
cc_library(
name = "tfprof_show",
srcs = ["tfprof_show.cc"],
hdrs = ["tfprof_show.h"],
deps = [
":tfprof_constants",
":tfprof_node",
":tfprof_node_show",
":tfprof_tensor",
":tfprof_timeline",
":tfprof_utils",
"//tensorflow/c:checkpoint_reader",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/platform:regexp",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "tfprof_show_multi",
srcs = ["tfprof_show_multi.cc"],
hdrs = ["tfprof_show_multi.h"],
deps = [
":tfprof_constants",
":tfprof_node",
":tfprof_node_show",
":tfprof_scope",
":tfprof_show",
":tfprof_tensor",
":tfprof_timeline",
":tfprof_utils",
"//tensorflow/core:lib",
"//tensorflow/core/platform:regexp",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
tf_cc_test(
name = "tfprof_show_test",
size = "small",
srcs = ["tfprof_show_test.cc"],
data = ["//tensorflow/core/profiler/internal/testdata:profiler_testdata"],
deps = [
":tfprof_constants",
":tfprof_stats",
":tfprof_tf_testlib",
":tfprof_utils",
"//tensorflow/core:lib",
"//tensorflow/core:test",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
],
)
tf_cc_test(
name = "tfprof_timeline_test",
size = "small",
srcs = ["tfprof_timeline_test.cc"],
data = ["//tensorflow/core/profiler/internal/testdata:profiler_testdata"],
deps = [
":tfprof_constants",
":tfprof_stats",
":tfprof_tf_testlib",
":tfprof_utils",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:test",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
],
)
cc_library(
name = "tfprof_utils",
srcs = ["tfprof_utils.cc"],
hdrs = ["tfprof_utils.h"],
copts = if_not_windows(["-Wno-sign-compare"]),
deps = [
"//tensorflow/core:lib",
"//tensorflow/core/platform:regexp",
"//tensorflow/core/profiler:tfprof_options",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "print_model_analysis_hdr",
hdrs = ["print_model_analysis.h"],
deps = ["//tensorflow/core:framework"],
)
cc_library(
name = "print_model_analysis",
srcs = ["print_model_analysis.cc"],
hdrs = ["print_model_analysis.h"],
deps = [
":tfprof_stats",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
"//tensorflow/core/profiler/internal/advisor:tfprof_advisor",
"@com_google_absl//absl/strings:str_format",
],
alwayslink = 1,
)
tf_cc_test(
name = "tfprof_stats_test",
size = "small",
srcs = ["tfprof_stats_test.cc"],
data = ["//tensorflow/core/profiler/internal/testdata:profiler_testdata"],
deps = [
":tfprof_constants",
":tfprof_stats",
":tfprof_tf_testlib",
":tfprof_utils",
"//tensorflow/core:lib",
"//tensorflow/core:test",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
],
)
cc_library(
name = "tfprof_tensor",
srcs = ["tfprof_tensor.cc"],
hdrs = ["tfprof_tensor.h"],
copts = if_not_windows(["-Wno-sign-compare"]),
deps = [
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core/profiler:protos_all_cc",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
tf_cc_test(
name = "tfprof_tensor_test",
size = "small",
srcs = ["tfprof_tensor_test.cc"],
data = ["//tensorflow/core/profiler/internal/testdata:profiler_testdata"],
deps = [
":tfprof_stats",
":tfprof_tf_testlib",
":tfprof_utils",
"//tensorflow/core:lib",
"//tensorflow/core:test",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler:tfprof_options",
],
)
cc_library(
name = "tfprof_constants",
hdrs = ["tfprof_constants.h"],
)
cc_library(
name = "tfprof_tf_testlib",
testonly = 1,
deps = [
":tfprof_tf_lib",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
],
)
cc_library(
name = "tfprof_tf_lib",
deps = [
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/platform:regexp",
],
)
tensorflow/core/profiler/internal/advisor/BUILD
+89
View File
@@ -0,0 +1,89 @@
load("//tensorflow:tensorflow.bzl", "tf_cc_test")
load("//tensorflow/core/platform:rules_cc.bzl", "cc_library")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:internal"],
licenses = ["notice"],
)
cc_library(
name = "checker",
hdrs = ["checker.h"],
deps = [
"//tensorflow/core:lib",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler/internal:tfprof_stats",
],
)
cc_library(
name = "internal_checker_runner_dummy",
srcs = ["internal_checker_runner_dummy.cc"],
hdrs = ["internal_checker_runner.h"],
deps = [
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler/internal:tfprof_utils",
],
)
cc_library(
name = "accelerator_utilization_checker",
hdrs = ["accelerator_utilization_checker.h"],
deps = [
":checker",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "operation_checker",
hdrs = ["operation_checker.h"],
deps = [
":checker",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "expensive_operation_checker",
hdrs = ["expensive_operation_checker.h"],
deps = [
":checker",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)
cc_library(
name = "tfprof_advisor",
hdrs = ["tfprof_advisor.h"],
deps = [
":accelerator_utilization_checker",
":checker",
":expensive_operation_checker",
":internal_checker_runner_dummy",
":operation_checker",
"//tensorflow/core/profiler:protos_all_cc",
"@com_google_absl//absl/strings:str_format",
],
)
tf_cc_test(
name = "tfprof_advisor_test",
srcs = ["tfprof_advisor_test.cc"],
deps = [
":checker",
":tfprof_advisor",
"//tensorflow/core:lib",
"//tensorflow/core:test",
"//tensorflow/core/framework:graph_proto_cc",
"//tensorflow/core/framework:node_def_proto_cc",
"//tensorflow/core/framework:step_stats_proto_cc",
"//tensorflow/core/profiler:protos_all_cc",
"//tensorflow/core/profiler/internal:tfprof_node",
"//tensorflow/core/profiler/internal:tfprof_stats",
"//tensorflow/core/profiler/internal:tfprof_tf_testlib",
"@com_google_absl//absl/strings",
],
)
tensorflow/core/profiler/internal/advisor/accelerator_utilization_checker.h
+113
View File
@@ -0,0 +1,113 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This checker checks the accelerator's utilization.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_ACCELERATOR_UTILIZATION_CHECKER_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_ACCELERATOR_UTILIZATION_CHECKER_H_
#include <algorithm>
#include <map>
#include "absl/strings/str_format.h"
#include "tensorflow/core/profiler/internal/advisor/checker.h"
namespace tensorflow {
namespace tfprof {
struct ExecStats {
public:
// Earliest start time of a step.
int64_t start_micros;
// Latest finish time of a step.
int64_t end_micros;
// The duration spent on running a kernel during a step.
int64_t exec_micros;
};
class AcceleratorUtilizationChecker : public Checker {
public:
std::string name() const override { return kCheckers[0]; }
private:
AdviceProto::Checker Check(const AdvisorOptionsProto::CheckerOption& options,
const TFStats* stats) override {
if (!stats) {
absl::FPrintF(
stderr, "Missing profiles (e.g. graph, run_meta). Skip %s\n", name());
return reports_;
}
for (const auto& n : stats->nodes()) {
BuildExecStats(n.second.get());
}
return CheckInternal();
}
AdviceProto::Checker CheckInternal() {
for (const auto& s : accelerator_exec_stats_) {
const ExecStats& stat = s.second;
int64_t total_micros = stat.end_micros - stat.start_micros;
if (total_micros <= 0) continue;
double utilization = 1.0 * stat.exec_micros / total_micros;
if (utilization >= 0.5) {
reports_.add_reports(absl::StrFormat("device: %s utilization: %.2f",
s.first, utilization));
} else if (utilization < 0.5 && utilization > 0.2) {
reports_.add_reports(absl::StrFormat("device: %s low utilization: %.2f",
s.first, utilization));
} else if (utilization <= 0.2) {
reports_.add_reports(absl::StrFormat("device: %s low utilization: %.2f",
s.first, utilization));
}
}
return reports_;
}
void BuildExecStats(const TFGraphNode* node) {
const auto& execs = node->all_op_execs();
if (execs.empty()) {
return;
}
if (!IsPlacedOnAccelerator(node->canonical_device())) {
return;
}
if (accelerator_exec_stats_.find(node->canonical_device()) ==
accelerator_exec_stats_.end()) {
accelerator_exec_stats_.insert(std::pair<std::string, ExecStats>(
node->canonical_device(), ExecStats()));
}
ExecStats& stats = accelerator_exec_stats_.at(node->canonical_device());
// TODO(xpan): Use multiple steps?
const ExecStep& exec = execs.rbegin()->second;
if (stats.start_micros == 0) {
stats.start_micros = exec.all_start_micros();
} else if (exec.all_start_micros() != 0) {
stats.start_micros =
std::min(stats.start_micros, exec.all_start_micros());
}
stats.end_micros = std::max(stats.end_micros, exec.latest_end_micros());
stats.exec_micros += exec.accelerator_exec_micros();
}
std::map<std::string, ExecStats> accelerator_exec_stats_;
std::map<std::string, int64_t> ps_placement_;
AdviceProto::Checker reports_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_ACCELERATOR_UTILIZATION_CHECKER_H_
tensorflow/core/profiler/internal/advisor/checker.h
+51
View File
@@ -0,0 +1,51 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_CHECKER_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_CHECKER_H_
#include "tensorflow/core/profiler/internal/tfprof_stats.h"
#include "tensorflow/core/profiler/tfprof_options.pb.h"
namespace tensorflow {
namespace tfprof {
// Append only.
static const char* const kCheckers[] = {
"AcceleratorUtilizationChecker", "OperationChecker",
"ExpensiveOperationChecker",
"JobChecker", // Internal checker.
};
class Checker {
public:
virtual ~Checker() = default;
virtual std::string name() const = 0;
AdviceProto::Checker Run(const AdvisorOptionsProto::CheckerOption& options,
const TFStats* stats) {
return Check(options, stats);
}
protected:
virtual AdviceProto::Checker Check(
const AdvisorOptionsProto::CheckerOption& options,
const TFStats* stats) = 0;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_CHECKER_H_
tensorflow/core/profiler/internal/advisor/expensive_operation_checker.h
+143
View File
@@ -0,0 +1,143 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This checker checks the most expensive operations.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_EXPENSIVE_OPERATION_CHECKER_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_EXPENSIVE_OPERATION_CHECKER_H_
#include <vector>
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "tensorflow/core/profiler/internal/advisor/checker.h"
namespace tensorflow {
namespace tfprof {
class ExpensiveOperationChecker : public Checker {
public:
std::string name() const override { return kCheckers[2]; }
private:
AdviceProto::Checker Check(const AdvisorOptionsProto::CheckerOption& options,
const TFStats* stats) override {
if (!stats) {
absl::FPrintF(
stderr, "Missing profiles (e.g. graph, run_meta). Skip %s\n", name());
return reports_;
}
if (stats->steps().empty()) {
absl::FPrintF(stderr, "Missing RunMetadata info. Skip %s\n", name());
}
CheckOpView(stats);
CheckScopeView(stats);
CheckCodeView(stats);
return reports_;
}
void CheckOpView(const TFStats* stats) {
if (stats->steps().empty()) {
absl::FPrintF(stderr, "Missing run_meta for %s\n", name());
return;
}
Options opts(3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, -1, "micros", {".*"}, {".*"},
{}, {".*"}, {}, false, {"micros", "occurrence"}, "none", {});
const MultiGraphNodeProto root = stats->ShowMultiGraphNode("op", opts);
if (root.children_size() == 0) {
return;
}
const MultiGraphNodeProto* node = &root;
std::vector<std::string> outputs;
for (int i = 0; i < 3 && node->children_size() > 0; ++i) {
node = &node->children(0);
outputs.push_back(absl::StrFormat(
"top %d operation type: %s, "
"cpu: %s, accelerator: %s, total: %s (%.2f%%)",
i + 1, node->name(), FormatTime(node->cpu_exec_micros()),
FormatTime(node->accelerator_exec_micros()),
FormatTime(node->exec_micros()),
100.0 * node->exec_micros() / (root.total_exec_micros() + 1e-10)));
}
reports_.add_reports(absl::StrJoin(outputs, "\n"));
}
void CheckCodeView(const TFStats* stats) {
if (!stats->has_code_traces()) {
absl::FPrintF(stderr, "Missing op_log (code traces) for %s\n", name());
return;
}
Options opts(100, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, -1, "micros", {".*"},
{".*"}, {}, {".*"}, {}, false, {"micros"}, "none", {});
const MultiGraphNodeProto root = stats->ShowMultiGraphNode("code", opts);
const MultiGraphNodeProto* node = &root;
// A trick here is: Usually, codes in library file are usually referenced
// only once, while user's own code are referenced multiple times.
while (node->children_size() == 1) {
node = &node->children(0);
}
if (node->children_size() == 0) {
return;
}
std::vector<std::string> outputs;
CodeViewHelper(node, 0, &outputs);
reports_.add_reports(absl::StrJoin(outputs, "\n"));
}
void CheckScopeView(const TFStats* stats) {
Options opts(100, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, -1, "micros", {".*"},
{".*"}, {}, {".*"}, {}, false, {"micros"}, "none", {});
const GraphNodeProto root = stats->ShowGraphNode("scope", opts);
if (root.children_size() == 0) {
return;
}
std::vector<std::string> outputs;
for (int i = 0; i < 3 && i < root.children_size(); ++i) {
const GraphNodeProto& node = root.children(i);
outputs.push_back(absl::StrFormat(
"top %d graph node: %s, cpu: %s, accelerator: %s, total: %s", i + 1,
node.name(), FormatTime(node.cpu_exec_micros()),
FormatTime(node.accelerator_exec_micros()),
FormatTime(node.exec_micros())));
}
reports_.add_reports(absl::StrJoin(outputs, "\n"));
}
void CodeViewHelper(const MultiGraphNodeProto* node, int depth,
std::vector<std::string>* outputs) {
if (node->children_size() <= 1 || depth > 3) {
return;
}
for (int j = 0; j < 3 && j < node->children_size(); ++j) {
const MultiGraphNodeProto* c = &node->children(j);
if (c->total_exec_micros() < 1000) {
continue;
}
outputs->push_back(
absl::StrFormat("%s%s, cpu: %s, accelerator: %s, total: %s",
std::string(depth * 2, ' '), c->name(),
FormatTime(c->total_cpu_exec_micros()),
FormatTime(c->total_accelerator_exec_micros()),
FormatTime(c->total_exec_micros())));
CodeViewHelper(c, depth + 1, outputs);
}
}
AdviceProto::Checker reports_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_EXPENSIVE_OPERATION_CHECKER_H_
tensorflow/core/profiler/internal/advisor/internal_checker_runner.h
+34
View File
@@ -0,0 +1,34 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_INTERNAL_CHECKER_RUNNER_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_INTERNAL_CHECKER_RUNNER_H_
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_options.pb.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class TFStats;
AdviceProto RunInternalCheckers(const AdvisorOptionsProto& options,
const TFStats* stats);
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_INTERNAL_CHECKER_RUNNER_H_
tensorflow/core/profiler/internal/advisor/internal_checker_runner_dummy.cc
+28
View File
@@ -0,0 +1,28 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/advisor/internal_checker_runner.h"
#include "tensorflow/core/profiler/tfprof_options.pb.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
AdviceProto RunInternalCheckers(const AdvisorOptionsProto& options,
const TFStats* stats) {
return AdviceProto();
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/advisor/operation_checker.h
+78
View File
@@ -0,0 +1,78 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This checker checks common wrong configurations of operations.
//
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_OPERATION_CHECKER_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_OPERATION_CHECKER_H_
#include "absl/strings/str_format.h"
#include "tensorflow/core/profiler/internal/advisor/checker.h"
namespace tensorflow {
namespace tfprof {
class OperationChecker : public Checker {
public:
std::string name() const override { return kCheckers[1]; }
private:
AdviceProto::Checker Check(const AdvisorOptionsProto::CheckerOption& options,
const TFStats* stats) override {
if (!stats) {
absl::FPrintF(
stderr, "Missing profiles (e.g. graph, run_meta). Skip %s\n", name());
return reports_;
}
bool use_batch_norm = false;
bool use_fused_batch_norm = false;
bool recommend_nchw = false;
for (const auto& n : stats->nodes()) {
const TFGraphNode* node = n.second.get();
if (node->name().find("BatchNorm") != node->name().npos) {
use_batch_norm = true;
}
if (node->op_types().find("FusedBatchNorm") != node->op_types().end()) {
use_fused_batch_norm = true;
}
const AttrValue* attr = node->op_attrs("data_format");
if (attr) {
if (attr->s() == "NHWC" &&
IsPlacedOnAccelerator(node->canonical_device())) {
recommend_nchw = true;
}
}
}
if (use_batch_norm && !use_fused_batch_norm) {
reports_.add_reports(
"Maybe use faster FusedBatchNorm instead of BatchNorm");
}
if (recommend_nchw) {
// TODO(xpan): Maybe print which Op supports NCHW.
reports_.add_reports(
"Found operation using NHWC data_format on GPU. Maybe "
"NCHW is faster.");
}
return reports_;
}
private:
AdviceProto::Checker reports_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_OPERATION_CHECKER_H_
tensorflow/core/profiler/internal/advisor/tfprof_advisor.h
+84
View File
@@ -0,0 +1,84 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_TFPROF_ADVISOR_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_TFPROF_ADVISOR_H_
#include <vector>
#include "absl/strings/str_format.h"
#include "tensorflow/core/profiler/internal/advisor/accelerator_utilization_checker.h"
#include "tensorflow/core/profiler/internal/advisor/checker.h"
#include "tensorflow/core/profiler/internal/advisor/expensive_operation_checker.h"
#include "tensorflow/core/profiler/internal/advisor/internal_checker_runner.h"
#include "tensorflow/core/profiler/internal/advisor/operation_checker.h"
#include "tensorflow/core/profiler/tfprof_options.pb.h"
namespace tensorflow {
namespace tfprof {
// The Advisor runs a list of Checkers, each checks a specific area.
class Advisor {
public:
Advisor(const TFStats* stats) : stats_(stats) {}
static AdvisorOptionsProto DefaultOptions() {
AdvisorOptionsProto options;
std::vector<std::string> checkers(
kCheckers, kCheckers + sizeof(kCheckers) / sizeof(*kCheckers));
for (const std::string& checker : checkers) {
(*options.mutable_checkers())[checker];
}
return options;
}
AdviceProto Advise(const AdvisorOptionsProto& options) {
// Note: Release a checker's memory ASAP.
AdviceProto ret = RunInternalCheckers(options, stats_);
if (options.checkers().find(kCheckers[0]) != options.checkers().end()) {
AcceleratorUtilizationChecker au_checker;
(*ret.mutable_checkers())[kCheckers[0]].MergeFrom(
au_checker.Run(options.checkers().at(kCheckers[0]), stats_));
}
if (options.checkers().find(kCheckers[1]) != options.checkers().end()) {
OperationChecker op_checker;
(*ret.mutable_checkers())[kCheckers[1]].MergeFrom(
op_checker.Run(options.checkers().at(kCheckers[1]), stats_));
}
if (options.checkers().find(kCheckers[2]) != options.checkers().end()) {
ExpensiveOperationChecker expensive_op_checker;
(*ret.mutable_checkers())[kCheckers[2]].MergeFrom(
expensive_op_checker.Run(options.checkers().at(kCheckers[2]),
stats_));
}
for (const auto& checker : ret.checkers()) {
absl::FPrintF(stdout, "\n%s:\n", checker.first);
for (const std::string& r : checker.second.reports()) {
absl::FPrintF(stdout, "%s\n", r);
}
}
fflush(stdout);
return ret;
}
private:
const TFStats* stats_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_ADVISOR_TFPROF_ADVISOR_H_
tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc
+121
View File
@@ -0,0 +1,121 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/advisor/tfprof_advisor.h"
#include <cstdint>
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "absl/strings/match.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/node_def.pb.h"
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/profiler/internal/advisor/checker.h"
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include "tensorflow/core/profiler/internal/tfprof_stats.h"
#include "tensorflow/core/profiler/tfprof_options.pb.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class TFProfAdvisorTest : public ::testing::Test {
protected:
TFProfAdvisorTest() {
stats_ = std::make_unique<TFStats>(std::make_unique<GraphDef>(), nullptr,
nullptr, nullptr);
stats_->AddNodeForTest(
0, CreateNode("n1", "Conv2D", {{"data_format", "NHWC"}}, 0, 10, 2));
stats_->AddNodeForTest(0, CreateNode("n2", "Conv2D", {}, 0, 20, 2));
stats_->BuildAllViews();
advisor_ = std::make_unique<Advisor>(stats_.get());
}
std::unique_ptr<TFGraphNode> CreateNode(
const std::string& name, const std::string& type,
std::map<std::string, std::string> attrs, int64_t step,
int64_t start_miros, int64_t end_rel_micros) {
node_defs_.push_back(std::make_unique<NodeDef>());
NodeDef* def = node_defs_.back().get();
def->set_name(name);
def->set_op(type);
for (const auto& attr : attrs) {
(*def->mutable_attr())[attr.first].set_s(attr.second);
}
std::unique_ptr<TFGraphNode> node =
std::make_unique<TFGraphNode>(def, -1, nullptr);
NodeExecStats node_stat;
node_stat.set_all_start_micros(start_miros);
node_stat.set_op_end_rel_micros(end_rel_micros);
node->AddStepStat(step, "/job:localhost/replica:0/task:0/device:GPU:0",
node_stat);
node->AddStepStat(step,
"/job:localhost/replica:0/task:0/device:GPU:0:stream:all",
node_stat);
node->AddStepStat(step,
"/job:localhost/replica:0/task:0/device:GPU:0:stream:0",
node_stat);
return node;
}
std::unique_ptr<TFStats> stats_;
std::unique_ptr<Advisor> advisor_;
std::vector<std::unique_ptr<NodeDef>> node_defs_;
};
TEST_F(TFProfAdvisorTest, Basics) {
AdvisorOptionsProto options = Advisor::DefaultOptions();
AdviceProto advice = advisor_->Advise(options);
EXPECT_TRUE(advice.checkers().find(kCheckers[0]) != advice.checkers().end());
EXPECT_TRUE(advice.checkers().find(kCheckers[1]) != advice.checkers().end());
EXPECT_TRUE(advice.checkers().find(kCheckers[2]) != advice.checkers().end());
}
TEST_F(TFProfAdvisorTest, OperationChecker) {
AdvisorOptionsProto options;
(*options.mutable_checkers())[kCheckers[1]];
AdviceProto advice = advisor_->Advise(options);
EXPECT_EQ(advice.checkers().at(kCheckers[1]).reports_size(), 1);
EXPECT_TRUE(
absl::StrContains(advice.checkers().at(kCheckers[1]).reports(0), "NCHW"));
}
TEST_F(TFProfAdvisorTest, UtilizationChecker) {
AdvisorOptionsProto options;
(*options.mutable_checkers())[kCheckers[0]];
AdviceProto advice = advisor_->Advise(options);
EXPECT_EQ(advice.checkers().at(kCheckers[0]).reports_size(), 1);
EXPECT_TRUE(absl::StrContains(advice.checkers().at(kCheckers[0]).reports(0),
"low utilization"));
}
TEST_F(TFProfAdvisorTest, ExpensiveOperationChecker) {
AdvisorOptionsProto options;
(*options.mutable_checkers())[kCheckers[2]];
AdviceProto advice = advisor_->Advise(options);
EXPECT_TRUE(absl::StrContains(advice.checkers().at(kCheckers[2]).reports(0),
"top 1 operation type: Conv2D"));
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/print_model_analysis.cc
+207
View File
@@ -0,0 +1,207 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/print_model_analysis.h"
#include <stdio.h>
#include <memory>
#include <utility>
#include "absl/strings/str_format.h"
#include "tensorflow/core/profiler/internal/advisor/tfprof_advisor.h"
#include "tensorflow/core/profiler/internal/tfprof_stats.h"
#include "tensorflow/core/profiler/tfprof_log.pb.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_options.pb.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
namespace {
TFStats* tf_stat = nullptr;
std::string RunProfile(const std::string& command, const std::string& options,
TFStats* tf_stats) {
if (command == kCmds[4]) {
AdvisorOptionsProto option_pb;
if (!option_pb.ParseFromString(options)) {
absl::FPrintF(stderr, "Cannot parse AdvisorOptionsProto\n");
return "";
}
tf_stats->BuildAllViews();
return Advisor(tf_stats).Advise(option_pb).SerializeAsString();
} else {
tf_stats->BuildView(command);
}
Options opts;
absl::Status s = Options::FromProtoStr(options, &opts);
if (!s.ok()) {
absl::FPrintF(stderr, "%s\n", s.ToString());
return "";
}
if (opts.output_type == kOutput[1]) {
absl::PrintF(
"\n=========================Options=============================\n");
absl::PrintF("%s", opts.ToString());
absl::PrintF(
"\n==================Model Analysis Report======================\n");
std::string ret = "";
if (command == kCmds[2] || command == kCmds[3]) {
ret = tf_stats->ShowMultiGraphNode(command, opts).SerializeAsString();
} else if (command == kCmds[0] || command == kCmds[1]) {
ret = tf_stats->ShowGraphNode(command, opts).SerializeAsString();
} else {
absl::FPrintF(stderr, "Unknown command: %s\n", command);
}
absl::PrintF(
"\n======================End of Report==========================\n");
fflush(stdout);
return ret;
}
if (command == kCmds[2] || command == kCmds[3]) {
return tf_stats->ShowMultiGraphNode(command, opts).SerializeAsString();
} else if (command == kCmds[0] || command == kCmds[1]) {
return tf_stats->ShowGraphNode(command, opts).SerializeAsString();
} else {
absl::FPrintF(stderr, "Unknown command: %s\n", command);
return "";
}
}
} // namespace
bool NewProfiler(const std::string* graph, const std::string* op_log) {
std::unique_ptr<GraphDef> graph_ptr = std::make_unique<GraphDef>();
if (graph && !graph->empty()) {
if (!graph_ptr->ParseFromString(*graph)) {
if (!protobuf::TextFormat::ParseFromString(*graph, graph_ptr.get())) {
absl::FPrintF(stderr, "Failed to parse graph\n");
return false;
}
}
}
std::unique_ptr<OpLogProto> op_log_ptr;
if (op_log && !op_log->empty()) {
op_log_ptr = std::make_unique<OpLogProto>();
if (!op_log_ptr->ParseFromString(*op_log)) {
absl::FPrintF(stderr, "Failed to parse OpLogProto.\n");
return false;
}
}
tf_stat = new TFStats(std::move(graph_ptr), nullptr, std::move(op_log_ptr),
nullptr);
return true;
}
void ProfilerFromFile(const std::string* filename) {
CHECK(!tf_stat) << "Currently only 1 living tfprof profiler is allowed";
CHECK(filename) << "Missing profile filename to init profiler from file";
tf_stat = new TFStats(*filename, nullptr);
}
void DeleteProfiler() {
if (tf_stat) {
delete tf_stat;
tf_stat = nullptr;
}
}
double AddStep(int64_t step, const std::string* graph,
const std::string* run_meta, const std::string* op_log) {
CHECK(tf_stat);
if (graph && !graph->empty()) {
std::unique_ptr<GraphDef> graph_ptr = std::make_unique<GraphDef>();
if (!graph_ptr->ParseFromString(*graph)) {
if (!protobuf::TextFormat::ParseFromString(*graph, graph_ptr.get())) {
absl::FPrintF(stderr, "Failed to parse graph\n");
}
}
tf_stat->AddGraph(std::move(graph_ptr));
}
CHECK(run_meta && !run_meta->empty());
// TODO(xpan): Better error handling.
std::unique_ptr<RunMetadata> run_meta_ptr = std::make_unique<RunMetadata>();
run_meta_ptr->ParseFromString(*run_meta);
tf_stat->AddRunMeta(step, std::move(run_meta_ptr));
if (op_log && !op_log->empty()) {
std::unique_ptr<OpLogProto> op_log_ptr;
op_log_ptr = std::make_unique<OpLogProto>();
op_log_ptr->ParseFromString(*op_log);
tf_stat->AddOpLogProto(std::move(op_log_ptr));
}
return tf_stat->run_coverage();
}
std::string Profile(const std::string* command, const std::string* options) {
CHECK(tf_stat);
CHECK(command) << "command mustn't be null";
CHECK(options) << "options mustn't be null";
return RunProfile(*command, *options, tf_stat);
}
std::string SerializeToString() {
CHECK(tf_stat);
std::string content;
tf_stat->SerializeToString(&content);
return content;
}
void WriteProfile(const std::string* filename) {
CHECK(tf_stat);
CHECK(filename) << "empty file name when asking to write profile.";
tf_stat->WriteProfile(*filename);
}
std::string PrintModelAnalysis(const std::string* graph,
const std::string* run_meta,
const std::string* op_log,
const std::string* command,
const std::string* options) {
CHECK(command) << "command mustn't be null";
CHECK(options) << "options mustn't be null";
std::unique_ptr<GraphDef> graph_ptr = std::make_unique<GraphDef>();
if (graph && !graph->empty()) {
graph_ptr->ParseFromString(*graph);
}
std::unique_ptr<RunMetadata> run_meta_ptr;
if (run_meta && !run_meta->empty()) {
run_meta_ptr = std::make_unique<RunMetadata>();
run_meta_ptr->ParseFromString(*run_meta);
}
std::unique_ptr<OpLogProto> op_log_ptr;
if (op_log && !op_log->empty()) {
op_log_ptr = std::make_unique<OpLogProto>();
op_log_ptr->ParseFromString(*op_log);
}
// TODO(xpan): Maybe need to init the checkpoint reader?
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader;
TFStats tf_stats(std::move(graph_ptr), std::move(run_meta_ptr),
std::move(op_log_ptr), std::move(ckpt_reader));
return RunProfile(*command, *options, &tf_stats);
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/print_model_analysis.h
+66
View File
@@ -0,0 +1,66 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_PRINT_MODEL_ANALYSIS_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_PRINT_MODEL_ANALYSIS_H_
#include <string>
namespace tensorflow {
namespace tfprof {
struct Options;
// **********************
// APIs in this file are only for swig.
// Talk to xpan@ if you want to call it directly!
// *********************
// Multi-step Profiler.
//
bool NewProfiler(const std::string* graph, const std::string* op_log);
void DeleteProfiler();
double AddStep(int64_t step, const std::string* graph,
const std::string* run_meta, const std::string* op_log);
// Write the profiler's profile to a proto buffer.
void WriteProfile(const std::string* filename);
// Load the profile to profiler from a proto buffer file.
void ProfilerFromFile(const std::string* filename);
// Returns a binary string that represents the serialized ProfileProto.
std::string SerializeToString();
std::string Profile(const std::string* command, const std::string* options);
// Single-step Profiler.
//
// Interface defined for Python API swig. Calls the tfprof core API.
// 'graph', 'run_meta', 'op_log' are serialized GraphDef, RunMetadata,
// OpLogProto strings, respectively.
// 'graph', 'command' and 'options' are required. Others can be nullptr
// if not available.
std::string PrintModelAnalysis(const std::string* graph,
const std::string* run_meta,
const std::string* op_log,
const std::string* command,
const std::string* options);
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_PRINT_MODEL_ANALYSIS_H_
tensorflow/core/profiler/internal/testdata/BUILD
Vendored
+22
View File
@@ -0,0 +1,22 @@
# Description:
# Profiler test data packages.
load("//tensorflow:tensorflow.default.bzl", "filegroup")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
filegroup(
name = "profiler_testdata",
srcs = [
"ckpt.data-00000-of-00001",
"ckpt.index",
"ckpt.meta",
"graph.pbtxt",
"run_meta",
"tfprof_log",
],
visibility = ["//tensorflow/core/profiler/internal:__pkg__"],
)
tensorflow/core/profiler/internal/testdata/ckpt.data-00000-of-00001
Vendored
BIN
View File
Binary file not shown.
tensorflow/core/profiler/internal/testdata/ckpt.index
Vendored
BIN
View File
Binary file not shown.
tensorflow/core/profiler/internal/testdata/ckpt.meta
Vendored
BIN
View File
Binary file not shown.
tensorflow/core/profiler/internal/testdata/graph.pbtxt
Vendored
+1370
View File
File diff suppressed because it is too large Load Diff
tensorflow/core/profiler/internal/testdata/run_meta
Vendored
BIN
View File
Binary file not shown.
tensorflow/core/profiler/internal/testdata/tfprof_log
Vendored
+11
View File
@@ -0,0 +1,11 @@

DW2_trainable_variables

ScalarW_trainable_variables

DW_trainable_variables
Conv2DÈ-
Conv2D_1€$
tensorflow/core/profiler/internal/tfprof_code.cc
+725
View File
@@ -0,0 +1,725 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_code.h"
#include <stdio.h>
#include <algorithm>
#include <iterator>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/io/zlib_compression_options.h"
#include "tensorflow/core/lib/io/zlib_outputbuffer.h"
#include "tensorflow/core/platform/regexp.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tsl/profiler/protobuf/profile.pb.h"
namespace tensorflow {
namespace tfprof {
namespace {
const char* const kGradientSuffix = " (gradient)";
// Convert to Trace proto into a short readable string.
std::string GetTraceString(const CallStack::Trace& trace) {
std::string ntrace =
absl::StrCat(io::Basename(trace.file()), ":", trace.lineno());
if (trace.function().length() < 20) {
absl::StrAppend(&ntrace, ":", trace.function());
} else {
absl::StrAppend(&ntrace, ":", trace.function().substr(0, 17), "...");
}
return ntrace;
}
bool IsGradNode(const std::string& name, std::string* forward_name) {
// Given a forward operation with name op, its gradient op has the following
// name: ...gradients/op_grad/...
// TODO(xpan): This is hacky.
auto grad_prefix = name.find("gradients/");
auto grad_suffix = name.find("_grad/");
if (grad_prefix == name.npos || grad_suffix == name.npos) {
return false;
}
auto start = grad_prefix + std::string("gradients/").length();
auto len = grad_suffix - start;
if (len <= 0) {
return false;
}
*forward_name = name.substr(start, len);
return true;
}
// StringTable maps each string to an id.
class StringTable {
public:
StringTable() {
// Pprof requires first entry in string_table to be ''.
string_id_[""] = 0;
all_strings_.push_back("");
}
// Returns the index of a string. If not found, inserts the string and
// return the inserted index.
uint64_t GetIndex(const std::string& str) {
auto idx = string_id_.find(str);
if (idx != string_id_.end()) {
return idx->second;
}
all_strings_.push_back(str);
return string_id_
.insert(std::pair<std::string, int64_t>(str, string_id_.size()))
.first->second;
}
const std::vector<std::string>& strings() const { return all_strings_; }
private:
std::map<std::string, uint64_t> string_id_;
std::vector<std::string> all_strings_;
};
// FunctionTable maps each function to an id.
class FunctionTable {
public:
explicit FunctionTable(StringTable* string_table)
: string_table_(string_table) {}
// Returns the index of a function. If not found, adds a function proto
// and returns the function index.
uint64_t GetIndex(const std::string& file_path, const std::string& func_name,
uint64_t func_start_line) {
auto key = std::tuple<std::string, std::string, uint64_t>(
file_path, func_name, func_start_line);
auto idx = function_table_.find(key);
if (idx != function_table_.end()) {
return idx->second.id();
}
pprof::Function* func_pb = &function_table_[key];
// function index should start from 1.
func_pb->set_id(function_table_.size());
std::string file_base(io::Basename(file_path));
file_base = file_base.substr(0, file_base.find_last_of('.'));
func_pb->set_name(
string_table_->GetIndex(absl::StrCat(file_base, ":", func_name)));
func_pb->set_filename(string_table_->GetIndex(file_path));
func_pb->set_start_line(func_start_line);
return func_pb->id();
}
const std::map<std::tuple<std::string, std::string, uint64_t>,
pprof::Function>&
functions() const {
return function_table_;
}
private:
StringTable* string_table_;
std::map<std::tuple<std::string, std::string, uint64_t>, pprof::Function>
function_table_;
};
// LocationTable maps each function call to an id.
class LocationTable {
public:
explicit LocationTable(FunctionTable* function_table)
: function_table_(function_table) {}
// Returns the index of a function call location. If not found, adds a
// location proto and returns the location index.
uint64_t GetIndex(const std::string& file_path, uint64_t line_number,
const std::string& called_function_name,
const std::string& called_file_path,
uint64_t called_func_start_line) {
auto key = std::tuple<std::string, std::string, uint64_t>(
file_path, called_function_name, line_number);
auto idx = location_table_.find(key);
if (idx != location_table_.end()) {
return idx->second.id();
}
pprof::Location* location_pb = &location_table_[key];
location_pb->set_id(location_table_.size());
pprof::Line* line_pb = location_pb->add_line();
line_pb->set_function_id(function_table_->GetIndex(
called_file_path, called_function_name, called_func_start_line));
line_pb->set_line(line_number);
return location_pb->id();
}
const std::map<std::tuple<std::string, std::string, uint64_t>,
pprof::Location>&
locations() const {
return location_table_;
}
private:
FunctionTable* function_table_;
std::map<std::tuple<std::string, std::string, uint64_t>, pprof::Location>
location_table_;
};
// Samples stores samples of all calls. A sample is a single call trace,
// that is, the call path from top caller to the leaf callee.
class Samples {
public:
explicit Samples(StringTable* string_table, const Options* opts)
: string_table_(string_table), opts_(opts) {}
// 'node' is the leaf of the displayed trace. It includes all graph nodes
// created by it. 'location_ids' contains
// the call stack, from callee to caller.
// This method adds the statistics of graph nodes created by the python
// call.
void Add(const CodeNode* node, const std::vector<uint64_t>& location_ids) {
// displayed leaf might not be true leaf. Retrieve the true leaves for
// stats.
std::vector<const CodeNode*> all_leaf = FetchAllLeaf(node);
CHECK(!all_leaf.empty()) << node->name();
for (const CodeNode* cn : all_leaf) {
for (const auto& gn_it : cn->node->graph_nodes()) {
const TFGraphNode* gn = gn_it.second;
std::string name = gn->name();
// Generate a new trace name, in case the name is taken.
while (sample_table_.find(name) != sample_table_.end()) {
name += '@';
}
pprof::Sample* sample_pb = &sample_table_[name];
for (uint64_t id : location_ids) {
sample_pb->mutable_location_id()->Add(id);
}
pprof::Label* label_pb = sample_pb->mutable_label()->Add();
label_pb->set_key(string_table_->GetIndex("graph node:"));
label_pb->set_str(string_table_->GetIndex(gn->name()));
sample_pb->mutable_value()->Add(1);
std::string type = *opts_->select.begin();
if (type == kShown[1]) {
sample_pb->mutable_value()->Add(gn->exec_micros(node->node->step()));
} else if (type == kShown[9]) {
sample_pb->mutable_value()->Add(
gn->accelerator_exec_micros(node->node->step()));
} else if (type == kShown[10]) {
sample_pb->mutable_value()->Add(
gn->cpu_exec_micros(node->node->step()));
} else if (type == kShown[0]) {
sample_pb->mutable_value()->Add(
gn->requested_bytes(node->node->step()));
} else if (type == kShown[11]) {
sample_pb->mutable_value()->Add(gn->peak_bytes(node->node->step()));
} else if (type == kShown[12]) {
sample_pb->mutable_value()->Add(
gn->residual_bytes(node->node->step()));
} else if (type == kShown[13]) {
sample_pb->mutable_value()->Add(gn->output_bytes(node->node->step()));
} else if (type == kShown[2]) {
sample_pb->mutable_value()->Add(gn->parameters());
} else if (type == kShown[3]) {
sample_pb->mutable_value()->Add(gn->float_ops(node->node->step()));
} else {
absl::FPrintF(stderr, "pprof doesn't support -select=%s\n", type);
}
}
}
}
const std::map<std::string, pprof::Sample>& samples() const {
return sample_table_;
}
private:
std::vector<const CodeNode*> FetchAllLeaf(const CodeNode* root) {
if (root->children.empty()) {
return {root};
}
std::vector<const CodeNode*> ret;
for (auto& n : root->children) {
std::vector<const CodeNode*> nodes = FetchAllLeaf(n);
ret.insert(ret.end(), nodes.begin(), nodes.end());
}
return ret;
}
StringTable* string_table_;
const Options* opts_;
std::map<std::string, pprof::Sample> sample_table_;
};
class PprofProfileImpl : public PprofProfile {
public:
explicit PprofProfileImpl(const Options* opts)
: opts_(opts),
func_table_(new FunctionTable(&string_table_)),
loc_table_(new LocationTable(func_table_.get())),
samples_(new Samples(&string_table_, opts)) {}
uint64_t AddLocation(const CodeNode* callee,
const CodeNode* caller) override {
const std::string& file_path = caller->file();
uint64_t lineno = caller->lineno();
const std::string& callee_file_path = callee->file();
const std::string& callee_function = callee->function();
uint64_t callee_func_start_line = callee->func_start_line();
return loc_table_->GetIndex(file_path, lineno, callee_function,
callee_file_path, callee_func_start_line);
}
void AddSample(const CodeNode* leaf,
std::vector<uint64_t>* call_ids) override {
std::vector<uint64_t> reversed_call_ids;
std::reverse_copy(call_ids->begin(), call_ids->end(),
std::back_inserter(reversed_call_ids));
samples_->Add(leaf, reversed_call_ids);
}
absl::Status WritePprofProfile(const std::string& filename) override {
pprof::Profile profile_pb;
Build(&profile_pb);
std::unique_ptr<WritableFile> file;
absl::Status s = Env::Default()->NewWritableFile(filename, &file);
if (!s.ok()) return s;
int32_t buf_size = 1024 * 1024;
io::ZlibOutputBuffer* zlib_output_buffer = new io::ZlibOutputBuffer(
file.get(), buf_size, buf_size, io::ZlibCompressionOptions::GZIP());
s = zlib_output_buffer->Init();
if (!s.ok()) {
delete zlib_output_buffer;
return s;
}
s = zlib_output_buffer->Append(profile_pb.SerializeAsString());
if (!s.ok()) {
delete zlib_output_buffer;
return s;
}
s = zlib_output_buffer->Close();
if (!s.ok()) {
delete zlib_output_buffer;
return s;
}
absl::FPrintF(stdout,
"\nRun pprof -png --nodecount=100 --sample_index=1 <%s>\n",
filename);
delete zlib_output_buffer;
return s;
}
private:
void Build(pprof::Profile* profile_pb) {
std::string sample_type_description = "count";
auto sample_type = profile_pb->mutable_sample_type()->Add();
sample_type->set_type(string_table_.GetIndex(sample_type_description));
sample_type->set_unit(string_table_.GetIndex("count"));
std::string type = *opts_->select.begin();
sample_type_description = type;
sample_type = profile_pb->mutable_sample_type()->Add();
sample_type->set_type(string_table_.GetIndex(sample_type_description));
if (type == kShown[1] || type == kShown[9] || type == kShown[10]) {
sample_type->set_unit(string_table_.GetIndex("microseconds"));
if (type == kShown[1]) {
profile_pb->mutable_comment()->Add(string_table_.GetIndex(
"Sum of accelerator execution time and cpu execution time."));
} else if (type == kShown[9]) {
profile_pb->mutable_comment()->Add(
string_table_.GetIndex("Accelerator execution time."));
} else if (type == kShown[10]) {
profile_pb->mutable_comment()->Add(
string_table_.GetIndex("CPU execution time."));
}
} else if (type == kShown[0]) {
sample_type->set_unit(string_table_.GetIndex("bytes"));
profile_pb->mutable_comment()->Add(
string_table_.GetIndex("Sum of operation total memory requests, "
"excluding deallocations."));
} else if (type == kShown[11]) {
sample_type->set_unit(string_table_.GetIndex("bytes"));
profile_pb->mutable_comment()->Add(
string_table_.GetIndex("Sum of operation peak memory usage."));
} else if (type == kShown[12]) {
sample_type->set_unit(string_table_.GetIndex("bytes"));
profile_pb->mutable_comment()->Add(string_table_.GetIndex(
"Sum of operation allocated memory after finish."));
} else if (type == kShown[13]) {
sample_type->set_unit(string_table_.GetIndex("bytes"));
profile_pb->mutable_comment()->Add(
string_table_.GetIndex("Sum of operation output size."));
} else if (type == kShown[2]) {
sample_type->set_unit(string_table_.GetIndex("count"));
profile_pb->mutable_comment()->Add(
string_table_.GetIndex("Model parameters."));
} else if (type == kShown[3]) {
sample_type->set_unit(string_table_.GetIndex("count"));
profile_pb->mutable_comment()->Add(string_table_.GetIndex(
"Model float operations (Only available if defined)."));
} else {
absl::FPrintF(stderr, "pprof doesn't support selecting: %s\n", type);
}
for (const std::string& str : string_table_.strings()) {
*profile_pb->mutable_string_table()->Add() = str;
}
for (const auto& sample_it : samples_->samples()) {
// TODO(xpan): Consider swap.
profile_pb->mutable_sample()->Add()->MergeFrom(sample_it.second);
}
for (const auto& function_it : func_table_->functions()) {
profile_pb->mutable_function()->Add()->MergeFrom(function_it.second);
}
for (const auto& location_it : loc_table_->locations()) {
profile_pb->mutable_location()->Add()->MergeFrom(location_it.second);
}
}
const Options* opts_;
StringTable string_table_;
std::unique_ptr<FunctionTable> func_table_;
std::unique_ptr<LocationTable> loc_table_;
std::unique_ptr<Samples> samples_;
};
} // namespace
void TFCode::AddNode(TFGraphNode* node) {
if (!node->call_stack() || node->call_stack()->traces().empty()) {
return;
}
// We infer the forward operation name from gradient op name. So, we can
// map gradient op traces to forward op traces.
// E.g. gradient node of 'inp_1/Conv2D' would be 'gradients/inp_1/Conv2D_grad.
std::string forward_name;
if (IsGradNode(node->name(), &forward_name)) {
auto grad_nodes_it = grad_nodes_.find(forward_name);
if (grad_nodes_it != grad_nodes_.end()) {
grad_nodes_it->second.push_back(node);
} else {
grad_nodes_.insert(std::pair<std::string, std::vector<TFGraphNode*>>(
forward_name, {node}));
}
return;
} else {
forward_nodes_[node->name()] = node;
}
if (!root_) {
graph_root_ = std::make_unique<TFMultiGraphNode>(kTFProfRoot);
root_ = std::make_unique<CodeNode>(graph_root_.get(), nullptr, "");
}
CodeNode* pre_code_node = root_.get();
// TODO(xpan): Consider to release CodeDef after TFCode is built. It
// takes a lot of memory.
std::set<std::string> traces;
for (int i = 0, end = node->call_stack()->traces().size(); i < end; ++i) {
// Unlike op name, which is globally unique, trace name is only unique
// w.r.t. it's parent.
const std::string& trace =
GetTraceString(node->call_stack()->traces().at(i));
traces.insert(trace);
pre_code_node = pre_code_node->AddChildren(
trace, &node->call_stack()->traces().at(i), "");
const int64_t last_index = node->call_stack()->traces().size() - 1;
if (i == last_index) {
pre_code_node->node->AddGraphNode(node);
}
}
}
void TFCode::Build() {
int64_t unaccounted_nodes = 0;
for (const auto& it : grad_nodes_) {
const std::string& forward_name = it.first;
auto forward_it = forward_nodes_.find(forward_name);
if (forward_it == forward_nodes_.end()) {
unaccounted_nodes += 1;
continue;
}
TFGraphNode* fn = forward_it->second;
CodeNode* leaf = nullptr;
CodeNode* pre_code_node = root_.get();
for (int i = 0, end = fn->call_stack()->traces().size(); i < end; ++i) {
const std::string& trace =
GetTraceString(fn->call_stack()->traces().at(i)) + kGradientSuffix;
pre_code_node = pre_code_node->AddChildren(
trace, &fn->call_stack()->traces().at(i), kGradientSuffix);
const int64_t last_trace = fn->call_stack()->traces().size() - 1;
if (i == last_trace) {
leaf = pre_code_node;
}
}
for (TFGraphNode* gn : it.second) {
leaf->node->AddGraphNode(gn);
}
}
if (unaccounted_nodes > 0) {
absl::FPrintF(stderr, "%d gradient nodes not accounted\n",
unaccounted_nodes);
}
}
const ShowMultiNode* TFCode::ShowInternal(const Options& opts,
Timeline* timeline) {
root_->ResetTotalStats();
if (opts.output_type == kOutput[3]) {
if (opts.select.size() != 1) {
absl::FPrintF(stderr, "Can only select 1 attribute for pprof output.\n");
return root_.get();
}
std::string select = *opts.select.begin();
if (select != kShown[0] && select != kShown[1] && select != kShown[2] &&
select != kShown[3] && select != kShown[9] && select != kShown[10] &&
select != kShown[11] && select != kShown[12] && select != kShown[13]) {
absl::FPrintF(stderr, "pprof doesn't support -select=%s\n", select);
return root_.get();
}
}
if (opts.account_displayed_op_only) {
absl::FPrintF(stderr,
"Note: code view ignores account_displayed_op_only\n");
}
std::vector<CodeNode*> roots = Account(root_->children, opts);
root_->show_children.clear();
for (CodeNode* n : roots) {
root_->AggregateTotalStats(n);
}
if (opts.start_name_regexes.size() != 1 ||
opts.start_name_regexes[0] != ".*") {
roots = SearchRoot(roots, opts.start_name_regexes);
}
root_->show_children.assign(roots.begin(), roots.end());
CodeNode* root = PrintScope({root_.get()}, opts, 1, 0)[0];
root->formatted_str = FormatLegend(opts) + root->formatted_str;
if (opts.output_type == kOutput[3]) {
std::vector<uint64_t> call_ids;
pprof_profile_ = std::make_unique<PprofProfileImpl>(&opts);
Format(root, root->show_children, opts, &root->formatted_str,
root->mutable_proto(), &call_ids);
absl::Status s = pprof_profile_->WritePprofProfile(
opts.output_options.at(kPprofOpts[0]));
if (!s.ok()) {
absl::FPrintF(stderr, "%s\n", s.ToString());
}
} else {
Format(root, root->show_children, opts, &root->formatted_str,
root->mutable_proto(), nullptr);
if (timeline) {
timeline->GenerateCodeTimeline(root);
}
}
return root;
}
void TFCode::Format(const CodeNode* root, const std::vector<CodeNode*>& nodes,
const Options& opts, std::string* display_str,
MultiGraphNodeProto* proto,
std::vector<uint64_t>* call_ids) {
if (nodes.empty() && root->has_trace() && opts.output_type == kOutput[3]) {
pprof_profile_->AddSample(root, call_ids);
}
for (CodeNode* node : nodes) {
if (root->has_trace() && opts.output_type == kOutput[3]) {
uint64_t loc_id = pprof_profile_->AddLocation(node, root);
call_ids->push_back(loc_id);
}
display_str->append(node->formatted_str);
MultiGraphNodeProto* child = proto->add_children();
child->MergeFrom(node->proto());
Format(node, node->show_children, opts, display_str, child, call_ids);
if (root->has_trace() && opts.output_type == kOutput[3]) {
call_ids->pop_back();
}
}
}
std::vector<CodeNode*> TFCode::SearchRoot(
std::vector<CodeNode*> roots, const std::vector<std::string>& regexes) {
std::vector<CodeNode*> res;
if (roots.empty()) {
return res;
}
for (CodeNode* root : roots) {
bool match_start_node = false;
for (const std::string& regex : regexes) {
if (RE2::FullMatch(root->name(), regex)) {
res.push_back(root);
match_start_node = true;
break;
}
}
if (match_start_node) {
// Found a start node at this branch, no need to continue.
continue;
}
std::vector<CodeNode*> nroots = SearchRoot(root->show_children, regexes);
res.insert(res.end(), nroots.begin(), nroots.end());
}
return res;
}
std::vector<CodeNode*> TFCode::PrintScope(const std::vector<CodeNode*> roots,
const Options& opts, int depth,
int last_ident) {
std::vector<CodeNode*> show_nodes;
for (CodeNode* node : roots) {
if (ShouldTrim(node, opts.trim_name_regexes) || depth > opts.max_depth) {
continue;
}
int ident = last_ident;
bool show = ShouldShow(node, opts, depth);
if (show) ident += 2;
std::vector<CodeNode*> show_cnodes =
PrintScope(node->show_children, opts, depth + 1, ident);
if (show) {
node->show_children.clear();
show_cnodes = SortNodes(show_cnodes, opts);
for (CodeNode* sc : show_cnodes) {
node->show_children.push_back(sc);
}
node->formatted_str = FormatNode(node, opts, last_ident);
if (opts.select.find(kShown[4]) != opts.select.end()) {
absl::FPrintF(stderr, "code view has no tensor value to show\n");
}
show_nodes.push_back(node);
} else {
show_nodes.insert(show_nodes.end(), show_cnodes.begin(),
show_cnodes.end());
}
}
return show_nodes;
}
std::vector<CodeNode*> TFCode::Account(const std::vector<CodeNode*>& roots,
const Options& opts) {
std::vector<CodeNode*> act_nodes;
for (CodeNode* node : roots) {
node->ResetTotalStats();
std::vector<CodeNode*> act_cnodes = Account(node->children, opts);
node->account = ReAccount(node, opts);
if (node->account || !act_cnodes.empty()) {
node->show_children.clear();
node->ResetTotalStats();
node->AddSelfToTotalStats();
for (CodeNode* c : act_cnodes) {
node->AggregateTotalStats(c);
node->show_children.push_back(c);
}
act_nodes.push_back(node);
}
}
return act_nodes;
}
std::string TFCode::FormatNodeMemory(CodeNode* node, int64_t bytes,
int64_t total_bytes) const {
std::string memory = FormatMemory(total_bytes);
if (node->account) {
memory = FormatMemory(bytes) + "/" + memory;
} else {
memory = "--/" + memory;
}
return memory;
}
std::string TFCode::FormatNode(CodeNode* node, const Options& opts,
int64_t indent) const {
std::vector<std::string> attrs;
if (opts.select.find(kShown[0]) != opts.select.end()) {
attrs.push_back(FormatNodeMemory(node, node->proto().requested_bytes(),
node->proto().total_requested_bytes()));
}
if (opts.select.find(kShown[11]) != opts.select.end()) {
attrs.push_back(FormatNodeMemory(node, node->proto().peak_bytes(),
node->proto().total_peak_bytes()));
}
if (opts.select.find(kShown[12]) != opts.select.end()) {
attrs.push_back(FormatNodeMemory(node, node->proto().residual_bytes(),
node->proto().total_residual_bytes()));
}
if (opts.select.find(kShown[13]) != opts.select.end()) {
attrs.push_back(FormatNodeMemory(node, node->proto().output_bytes(),
node->proto().total_output_bytes()));
}
std::vector<std::string> time_attrs = FormatTimes(node, opts);
attrs.insert(attrs.end(), time_attrs.begin(), time_attrs.end());
if (opts.select.find(kShown[2]) != opts.select.end()) {
std::string params =
FormatNumber(node->proto().total_parameters()) + " params";
if (node->account) {
params = FormatNumber(node->proto().parameters()) + "/" + params;
} else {
params = "--/" + params;
}
attrs.push_back(params);
}
if (opts.select.find(kShown[3]) != opts.select.end()) {
std::string fops = FormatNumber(node->proto().total_float_ops()) + " flops";
if (node->account) {
fops = FormatNumber(node->proto().float_ops()) + "/" + fops;
} else {
fops = "--/" + fops;
}
attrs.push_back(fops);
}
if (opts.select.find(kShown[5]) != opts.select.end() &&
!node->node->devices().empty()) {
attrs.push_back(absl::StrJoin(node->node->devices(), "|"));
}
if (opts.select.find(kShown[6]) != opts.select.end()) {
std::set<std::string> op_types = node->node->op_types();
attrs.push_back(absl::StrJoin(op_types, "|"));
}
if (opts.select.find(kShown[7]) != opts.select.end()) {
// TODO(xpan): Make op count available in code view?
attrs.push_back(absl::StrFormat("%s N/A in code view", kShown[7]));
}
if (opts.select.find(kShown[8]) != opts.select.end()) {
attrs.push_back(absl::StrFormat("%s N/A in code view", kShown[8]));
}
return absl::StrFormat("%s%s (%s)\n", std::string(indent, ' '), node->name(),
absl::StrJoin(attrs, ", "));
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_code.h
+97
View File
@@ -0,0 +1,97 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Build a tree structure based on the TensorFlow model's python code stacks.
// Stats are aggregated from descendants to ancestors.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_CODE_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_CODE_H_
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include "tensorflow/core/profiler/internal/tfprof_show_multi.h"
#include "tensorflow/core/profiler/internal/tfprof_timeline.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/profile.pb.h"
#include "tensorflow/core/profiler/tfprof_log.pb.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class PprofProfile {
public:
virtual ~PprofProfile() = default;
virtual uint64_t AddLocation(const CodeNode* callee,
const CodeNode* caller) = 0;
virtual void AddSample(const CodeNode* leaf,
std::vector<uint64_t>* call_ids) = 0;
virtual absl::Status WritePprofProfile(const std::string& filename) = 0;
};
class TFCode : public TFMultiShow {
public:
TFCode() = default;
~TFCode() override = default;
// Add nodes to the code view. Called before Build()
void AddNode(TFGraphNode* node) override;
// Build the code view structure. Called after all nodes
// are added via AddNode().
void Build() override;
private:
const ShowMultiNode* ShowInternal(const Options& opts,
Timeline* timeline) override;
std::vector<CodeNode*> SearchRoot(std::vector<CodeNode*> roots,
const std::vector<std::string>& regexes);
std::vector<CodeNode*> PrintScope(std::vector<CodeNode*> roots,
const Options& opts, int depth,
int last_ident);
std::vector<CodeNode*> Account(const std::vector<CodeNode*>& roots,
const Options& opts);
void Format(const CodeNode* root, const std::vector<CodeNode*>& nodes,
const Options& opts, std::string* display_str,
MultiGraphNodeProto* proto, std::vector<uint64_t>* call_ids);
std::string FormatNode(CodeNode* node, const Options& opts,
int64_t indent) const;
std::string FormatNodeMemory(CodeNode* node, int64_t bytes,
int64_t total_bytes) const;
std::unique_ptr<CodeNode> root_;
std::unique_ptr<TFMultiGraphNode> graph_root_;
std::unique_ptr<PprofProfile> pprof_profile_;
std::map<std::string, std::vector<TFGraphNode*>> grad_nodes_;
std::map<std::string, TFGraphNode*> forward_nodes_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_CODE_H_
tensorflow/core/profiler/internal/tfprof_constants.h
+37
View File
@@ -0,0 +1,37 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_CONSTANTS_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_CONSTANTS_H_
namespace tensorflow {
namespace tfprof {
// Op name of root of everything. Aggregates all stats.
static const char* const kTFProfRoot = "_TFProfRoot";
// Op type for nodes that doesn't represent a physical node in the
// TensorFlow model. Only exist as a placehold to aggregate children.
// For example, kTFProfRoot belongs to this type.
static const char* const kTFGraphParent = "_TFGraphParent";
static const char* const kTFScopeParent = "_kTFScopeParent";
// Op type for tf.trainable_variables().
static const char* const kTrainableVarType = "_trainable_variables";
// Op type for tensors in the checkpoint file.
static const char* const kCkptVarType = "_checkpoint_variables";
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_CONSTANTS_H_
tensorflow/core/profiler/internal/tfprof_graph.cc
+246
View File
@@ -0,0 +1,246 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_graph.h"
#include <stdio.h>
#include <map>
#include <memory>
#include <set>
#include <utility>
#include <vector>
#include "absl/strings/str_format.h"
#include "tensorflow/core/platform/regexp.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tensorflow/core/profiler/internal/tfprof_tensor.h"
namespace tensorflow {
namespace tfprof {
GraphNode* TFGraph::CreateParentNode(const std::string& name) {
node_defs_.push_back(std::make_unique<NodeDef>());
node_defs_.back()->set_name(name);
node_defs_.back()->set_op(kTFGraphParent);
parent_nodes_[name] =
std::make_unique<TFGraphNode>(node_defs_.back().get(), -1, nullptr);
nodes_map_[name] = std::make_unique<GraphNode>(parent_nodes_[name].get());
return nodes_map_[name].get();
}
void TFGraph::AddNode(TFGraphNode* node) {
std::string name = node->name();
nodes_map_[name] = std::make_unique<GraphNode>(node);
}
void TFGraph::Build() {
if (root_) return;
std::set<std::string> nonroots;
// Filter out the root nodes (node not input of any other node).
for (auto it = nodes_map_.begin(); it != nodes_map_.end(); it++) {
GraphNode* node = it->second.get();
const std::map<int, std::string>& inputs = node->node->inputs();
for (auto inputs_it = inputs.cbegin(); inputs_it != inputs.cend();
inputs_it++) {
nonroots.insert(inputs_it->second);
auto child_it = nodes_map_.find(inputs_it->second);
if (child_it != nodes_map_.end()) {
node->children.push_back(child_it->second.get());
}
}
}
std::vector<GraphNode*> roots;
for (auto it = nodes_map_.begin(); it != nodes_map_.end(); it++) {
if (nonroots.find(it->first) == nonroots.end()) {
roots.push_back(it->second.get());
}
}
root_ = CreateParentNode(kTFProfRoot);
root_->children.insert(root_->children.end(), roots.begin(), roots.end());
}
const ShowNode* TFGraph::ShowInternal(const Options& opts, Timeline* timeline) {
root_->ResetTotalStats();
root_->show_children.clear();
if (opts.output_type == kOutput[3]) {
absl::FPrintF(stderr, "Only 'code' view supports pprof output now.\n");
return root_;
}
if (timeline && timeline->step() < 0) {
// TODO(xpan): Maybe pick a default step for users.
absl::FPrintF(
stderr,
"Must specify -step option to generate timeline in graph view.\n");
return root_;
}
// 1. Account and aggregate the stats based on the graph structure.
// Returns a graph consists of accounted nodes.
std::set<std::string> visits;
std::vector<GraphNode*> roots = Account(root_->children, opts, &visits);
for (GraphNode* n : roots) {
root_->AggregateTotalStats(n);
}
// 2. Trim the nodes before start_name_regexes.
if (opts.start_name_regexes.size() != 1 ||
opts.start_name_regexes[0] != ".*") {
visits.clear();
roots = SearchRoot(roots, opts.start_name_regexes, &visits);
}
// 3. Trim the nodes not matching show/hide/trim_name_regexes.
// If account_displayed_op_only=true, redo the accounting.
visits.clear();
root_->show_children.assign(roots.begin(), roots.end());
GraphNode* root = PrintGraph({root_}, opts, 1, 0, &visits)[0];
// 4. Prepare output based on the final graphs.
root->formatted_str = FormatLegend(opts) + root->formatted_str;
Format(root->show_children, &root->formatted_str, root->mutable_proto());
if (timeline) {
timeline->GenerateGraphTimeline(root->show_children);
}
return root;
}
std::vector<GraphNode*> TFGraph::SearchRoot(
const std::vector<GraphNode*>& roots,
const std::vector<std::string>& regexes, std::set<std::string>* visited) {
std::vector<GraphNode*> res;
if (roots.empty()) {
return res;
}
for (GraphNode* root : roots) {
if (visited->find(root->name()) != visited->end()) continue;
visited->insert(root->name());
// If the parent is a start point, don't search its children.
// Note that its children can still be added as start node through
// another route.
bool match_start_node = false;
for (const std::string& regex : regexes) {
if (RE2::FullMatch(root->name(), regex)) {
res.push_back(root);
match_start_node = true;
break;
}
}
if (match_start_node) {
continue;
}
std::vector<GraphNode*> nroot =
SearchRoot(root->show_children, regexes, visited);
res.insert(res.end(), nroot.begin(), nroot.end());
}
return res;
}
void TFGraph::Format(const std::vector<GraphNode*> roots,
std::string* display_str, GraphNodeProto* proto) {
for (GraphNode* node : roots) {
display_str->append(node->formatted_str);
GraphNodeProto* child = proto->add_children();
child->MergeFrom(node->proto());
Format(node->show_children, display_str, child);
}
}
std::vector<GraphNode*> TFGraph::PrintGraph(const std::vector<GraphNode*> roots,
const Options& opts, int depth,
int last_ident,
std::set<std::string>* visits) {
std::vector<GraphNode*> show_nodes;
for (GraphNode* node : roots) {
if (visits->find(node->name()) != visits->end()) continue;
visits->insert(node->name());
bool show = ShouldShow(node, opts, depth);
int indent = last_ident;
if (show) indent += 2;
std::vector<GraphNode*> show_cnodes;
if (!ShouldTrim(node, opts.trim_name_regexes) && depth <= opts.max_depth) {
show_cnodes =
PrintGraph(node->show_children, opts, depth + 1, indent, visits);
}
if (show) {
node->show_children.clear();
if (opts.account_displayed_op_only) {
node->ResetTotalStats();
node->AddSelfToTotalStats();
}
show_cnodes = SortNodes(show_cnodes, opts);
for (GraphNode* sc : show_cnodes) {
node->show_children.push_back(sc);
if (opts.account_displayed_op_only) {
node->AggregateTotalStats(sc);
}
}
node->formatted_str = absl::StrFormat(
"%s%s\n", std::string(last_ident, ' '), FormatNode(node, opts));
if (opts.select.find(kShown[4]) != opts.select.end()) {
std::unique_ptr<TFProfTensor> tfprof_tensor;
if (LookUpCheckPoint(node->name(), &tfprof_tensor)) {
std::string value_str;
tfprof_tensor->Display(&value_str,
node->mutable_proto()->mutable_tensor_value());
node->formatted_str += value_str;
}
}
show_nodes.push_back(node);
} else {
show_nodes.insert(show_nodes.end(), show_cnodes.begin(),
show_cnodes.end());
}
}
return show_nodes;
}
std::vector<GraphNode*> TFGraph::Account(const std::vector<GraphNode*>& roots,
const Options& opts,
std::set<std::string>* visits) {
std::vector<GraphNode*> act_nodes;
for (GraphNode* node : roots) {
if (visits->find(node->name()) != visits->end()) continue;
visits->insert(node->name());
// Depth-first.
std::vector<GraphNode*> act_cnodes = Account(node->children, opts, visits);
node->account = ReAccount(node, opts);
if (node->account) {
node->show_children.clear();
node->ResetTotalStats();
node->AddSelfToTotalStats();
// Aggregate its accounted children stats.
for (GraphNode* c : act_cnodes) {
node->AggregateTotalStats(c);
node->show_children.push_back(c);
}
act_nodes.push_back(node);
} else {
// If the current node is not accounted, pass the children to the
// ancestor.
act_nodes.insert(act_nodes.end(), act_cnodes.begin(), act_cnodes.end());
}
}
return act_nodes;
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_graph.h
+88
View File
@@ -0,0 +1,88 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Build a graph structure based on op inputs/outputs. The graph is a directed
// acyclic graph pointing *from outputs to inputs*.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_GRAPH_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_GRAPH_H_
#include <deque>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <vector>
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include "tensorflow/core/profiler/internal/tfprof_show.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
// Organize tensorflow ops in a graph structure, pointing from output ops
// to input ops.
class TFGraph : public TFShow {
public:
explicit TFGraph(checkpoint::CheckpointReader* ckpt_reader)
: TFShow(ckpt_reader), root_(nullptr) {}
~TFGraph() override = default;
void AddNode(TFGraphNode* node) override;
void Build() override;
private:
const ShowNode* ShowInternal(const Options& opts,
Timeline* timeline) override;
bool ShouldShowIfExtra(const ShowNode* node, const Options& opts,
int depth) const override {
return true;
}
GraphNode* CreateParentNode(const std::string& name);
std::vector<GraphNode*> SearchRoot(const std::vector<GraphNode*>& roots,
const std::vector<std::string>& regexes,
std::set<std::string>* visited);
std::vector<GraphNode*> PrintGraph(std::vector<GraphNode*> roots,
const Options& opts, int depth,
int last_ident,
std::set<std::string>* visits);
std::vector<GraphNode*> Account(const std::vector<GraphNode*>& roots,
const Options& opts,
std::set<std::string>* visits);
void Format(std::vector<GraphNode*> roots, std::string* display_str,
GraphNodeProto* proto);
MemoryTracker memory_tracker_;
GraphNode* root_;
std::vector<std::unique_ptr<NodeDef>> node_defs_;
std::map<std::string, std::unique_ptr<TFGraphNode>> parent_nodes_;
std::map<std::string, std::unique_ptr<GraphNode>> nodes_map_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_GRAPH_H_
tensorflow/core/profiler/internal/tfprof_node.cc
+296
View File
@@ -0,0 +1,296 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include <algorithm>
#include <utility>
#include <vector>
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
namespace tensorflow {
namespace tfprof {
bool CountAsAcceleratorTime(const std::string& device) {
return device.find("stream:all") != device.npos;
}
bool CountAsCPUTime(const std::string& device) {
return RE2::FullMatch(device, ".*/(device:gpu|gpu|device:cpu|cpu):\\d+");
}
bool IsCanonicalDevice(const std::string& device) {
return CountAsCPUTime(device);
}
// Notes about start and end time from the NodeExecStats proto:
// For GPU, there is no difference between op_end_rel_micros and
// all_end_rel_micros. All are kernel times.
// For CPU, op_end_rel is the kernel time, while all_end_rel_micros includes
// some post-processing. Besides, currently, there is no way to measure
// the execution time of async ops accurately.
//
// Notes about device:
// For ops on gpu:
// It will appear in three different devices in RunMetadata: 1) gpu:x,
// 2) gpu:x:stream:all and 3) gpu:x:stream:id. 2) is used a combined view
// of all different 3). 1) is the op scheduling, pre-processing and
// post processing time. 3) is the execution time of GPU kernels on a stream.
// For ops on cpu:
// It will only appear as cpu:0.
void ExecStep::AddTimeStats(const std::string& dev,
const NodeExecStats& step_stat) {
devices_.insert(dev);
if (step_stat.all_start_micros() > 0) {
if (exec_.all_start_micros() > 0) {
exec_.set_all_start_micros(
std::min(static_cast<int64_t>(exec_.all_start_micros()),
static_cast<int64_t>(step_stat.all_start_micros())));
} else {
exec_.set_all_start_micros(step_stat.all_start_micros());
}
int64_t op_end_rel_micros = step_stat.op_end_rel_micros();
// Round quick execution to 1 micro to be semantically robust.
if (op_end_rel_micros == 0) {
++op_end_rel_micros;
}
exec_.set_latest_end_micros(
std::max(static_cast<int64_t>(exec_.latest_end_micros()),
step_stat.all_start_micros() + op_end_rel_micros));
const std::pair<int64_t, int64_t> pair =
std::make_pair(step_stat.all_start_micros(), op_end_rel_micros);
if (CountAsAcceleratorTime(dev)) {
accelerator_execs_[dev].push_back(pair);
op_execs_[dev].push_back(pair);
} else if (CountAsCPUTime(dev)) {
cpu_execs_[dev].push_back(pair);
op_execs_[dev].push_back(pair);
// In while-loop, a graph node is executed multiple times under
// the same name.
exec_.set_run_count(exec_.run_count() + 1);
}
}
}
void ExecStep::AddMemoryStats(const std::string& dev,
const NodeExecStats& step_stat) {
ExecMemory exec_mem;
if (step_stat.all_start_micros() > 0) {
exec_mem.set_memory_micros(step_stat.all_start_micros() +
step_stat.op_end_rel_micros());
} else {
absl::FPrintF(stderr, "%s has no start time, skipping\n",
step_stat.node_name());
return;
}
int accelerator_allocator_cnt = 0;
for (const auto& mem : step_stat.memory()) {
// TODO(xpan): Fix this hack. Currently the allocator name seems quite
// ad-hoc.
if (mem.allocator_name().find("GPU") == mem.allocator_name().npos) {
continue;
}
++accelerator_allocator_cnt;
exec_mem.set_allocator_bytes_in_use(
std::max(static_cast<int64_t>(exec_mem.allocator_bytes_in_use()),
static_cast<int64_t>(mem.allocator_bytes_in_use())));
for (const auto& alloc : mem.allocation_records()) {
allocations_.push_back(alloc);
}
}
if (accelerator_allocator_cnt > 1) {
absl::FPrintF(stderr, "found %d gpu allocator for 1 node\n",
accelerator_allocator_cnt);
}
int64_t total_output_bytes = 0;
for (const auto& output : step_stat.output()) {
if (output.has_tensor_description() &&
output.tensor_description().has_allocation_description()) {
// TODO(xpan): Maybe allocated_bytes.
int64_t output_bytes = std::max(output.tensor_description()
.allocation_description()
.allocated_bytes(),
output.tensor_description()
.allocation_description()
.requested_bytes());
uint64_t output_ptr =
output.tensor_description().allocation_description().ptr();
total_output_bytes += output_bytes;
auto& mem = (*exec_mem.mutable_output_memory())[output.slot()];
mem.set_ptr(output_ptr);
mem.set_bytes(output_bytes);
}
}
exec_mem.set_output_bytes(total_output_bytes);
if (step_stat.has_memory_stats()) {
if (IsPlacedOnCPU(dev)) {
// Currently we assume ops placed on gpu only allocate memory on gpu.
exec_mem.set_host_temp_bytes(exec_mem.host_temp_bytes() +
step_stat.memory_stats().temp_memory_size());
exec_mem.set_host_persistent_bytes(
exec_mem.host_persistent_bytes() +
step_stat.memory_stats().persistent_memory_size());
} else {
exec_mem.set_accelerator_temp_bytes(
exec_mem.accelerator_temp_bytes() +
step_stat.memory_stats().temp_memory_size());
exec_mem.set_accelerator_persistent_bytes(
exec_mem.accelerator_persistent_bytes() +
step_stat.memory_stats().persistent_memory_size());
}
}
// TODO(xpan): Make this more accurate:
// High level: Memory tracking is suspicious and requires large scale
// clean up.
// Investigate the memory usage difference between CPU/GPU with OpViewTest.
//
// 1. OpKernelConstruction::allocate_xxx is not traced. Below, we only
// discuss OpKernelContext-related allocations.
// 2. allocate_output calls allocate_tensor, which is properly tracked in
// 'NodeExecStats.memory'.
// 3. allocate_temp is only tracked through record_xxx_temp. It appears
// in 'NodeExecStats.memory_stats'.
// 4. record_xxx_persistent is called when allocate_persistent
// is not used and hence tracks some complementary bytes. It appears in
// 'NodeExecStats.memory_stats'. It's suspicious. But we should
// use it now since it covers constant op.
int64_t residual_bytes = 0;
int64_t requested_bytes = 0;
int64_t peak_bytes = 0;
for (const auto& mem : step_stat.memory()) {
residual_bytes += mem.live_bytes();
requested_bytes += mem.total_bytes();
peak_bytes += mem.peak_bytes();
}
residual_bytes += exec_mem.host_persistent_bytes() +
exec_mem.accelerator_persistent_bytes();
requested_bytes += exec_mem.host_persistent_bytes() +
exec_mem.accelerator_persistent_bytes() +
exec_mem.host_temp_bytes() +
exec_mem.accelerator_temp_bytes();
peak_bytes += exec_mem.host_persistent_bytes() +
exec_mem.accelerator_persistent_bytes() +
exec_mem.host_temp_bytes() + exec_mem.accelerator_temp_bytes();
exec_mem.set_requested_bytes(requested_bytes);
exec_mem.set_residual_bytes(residual_bytes);
exec_mem.set_peak_bytes(peak_bytes);
memory_execs_.emplace_back(exec_mem);
}
void TFGraphNode::AddStepStat(int64_t step, const std::string& device,
const NodeExecStats& step_stat) {
std::string dev = absl::AsciiStrToLower(device);
// TODO(xpan): Make this more robust?
// See run_metadata_test.py
// It can be /job:0/replica:0/xxxx/device:GPU:0, or simply /device:GPU:0.
// It can has some ad-hoc suffix, such as /stream:xx or /memcpy:xx.
if (IsCanonicalDevice(dev)) {
if (!node_.canonical_device().empty()) {
if (node_.canonical_device() != dev) {
// TODO(xpan): Some RunMetadata node appears at multiple devices.
// Need to address it.
return;
}
} else {
node_.set_canonical_device(dev);
// TODO(xpan): Support things other than gpu?
node_.set_host_device(StringReplace(dev, "gpu:\\d+", "cpu:0"));
AddOpType(node_.canonical_device());
}
}
auto exec = execs_.find(step);
if (exec == execs_.end()) {
execs_.insert(std::pair<int64_t, ExecStep>(step, ExecStep()));
exec = execs_.find(step);
}
exec->second.AddTimeStats(dev, step_stat);
if (dev == node_.canonical_device()) {
exec->second.AddMemoryStats(dev, step_stat);
}
}
int64_t ExecStep::exec_micros() const {
return accelerator_exec_micros() + cpu_exec_micros();
}
int64_t ExecStep::accelerator_exec_micros() const {
int64_t total = 0;
// Normally, an op should only be scheduled on 1 accelerator device.
// Hence there should generally be 1 element in accelerator_execs_.
for (const auto& execs : accelerator_execs_) {
// An op can fire multiple kernels or
// being scheduled multiple times in while-loop.
for (const auto& exec : execs.second) {
total += exec.second;
}
}
return total;
}
int64_t ExecStep::cpu_exec_micros() const {
int64_t total = 0;
// Normally, an op can only be scheduled on 1 device.
for (const auto& execs : cpu_execs_) {
// An op can be scheduled multiple times in while-loop.
for (const auto& exec : execs.second) {
total += exec.second;
}
}
return total;
}
std::vector<int64_t> ShapeProtoToVec(const TensorShapeProto& shape_pb) {
std::vector<int64_t> shape_vec;
if (shape_pb.dim_size() == 0 && !shape_pb.unknown_rank()) {
// Scalar parameter with empty shape but known rank.
shape_vec.push_back(1);
} else {
for (const auto& d : shape_pb.dim()) {
shape_vec.push_back(d.size());
}
}
return shape_vec;
}
TensorShapeProto VecToShapeProto(const std::vector<int64_t>& shape_vec) {
TensorShapeProto shape_pb;
if (shape_vec.empty()) {
shape_pb.set_unknown_rank(true);
return shape_pb;
}
for (const int64_t s : shape_vec) {
shape_pb.add_dim()->set_size(s);
}
return shape_pb;
}
bool IsPlacedOnAccelerator(const std::string& device) {
return device.find("gpu") != device.npos;
}
bool IsPlacedOnCPU(const std::string& device) {
return device.find("cpu") != device.npos;
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_node.h
+919
View File
@@ -0,0 +1,919 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_NODE_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_NODE_H_
#include <algorithm>
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>
#include "absl/strings/str_format.h"
#include "tensorflow/core/framework/allocation_description.pb.h"
#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/node_def.pb.h"
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tensorflow/core/framework/tensor_description.pb.h"
#include "tensorflow/core/framework/tensor_shape.pb.h"
#include "tensorflow/core/platform/regexp.h"
#include "tensorflow/core/profiler/tfprof_log.pb.h"
#include "tensorflow/core/profiler/tfprof_options.h"
namespace tensorflow {
namespace tfprof {
std::vector<int64_t> ShapeProtoToVec(const TensorShapeProto& shape_pb);
TensorShapeProto VecToShapeProto(const std::vector<int64_t>& shape_vec);
class TFGraphNode;
class CallStack {
public:
class Trace {
public:
Trace(const CodeDef::Trace* trace,
const std::map<int64_t, std::string>* id_to_string)
: trace_(trace), id_to_string_(id_to_string) {}
int32_t lineno() const { return trace_->lineno(); }
std::string file() const { return id_to_string_->at(trace_->file_id()); }
std::string function() const {
return id_to_string_->at(trace_->function_id());
}
int32_t func_start_line() const { return trace_->func_start_line(); }
private:
const CodeDef::Trace* trace_;
const std::map<int64_t, std::string>* id_to_string_;
};
CallStack(const CodeDef& def,
const std::map<int64_t, std::string>* id_to_string)
: def_(def) {
traces_.reserve(def.traces_size());
for (const auto& t : def_.traces()) {
traces_.emplace_back(&t, id_to_string);
}
}
const CodeDef& code_def() const { return def_; }
const std::vector<Trace>& traces() const { return traces_; }
private:
std::vector<Trace> traces_;
CodeDef def_;
};
class ExecStep {
public:
ExecStep() = default;
void AddTimeStats(const std::string& dev, const NodeExecStats& step_stat);
void AddMemoryStats(const std::string& dev, const NodeExecStats& step_stat);
int64_t run_count() const { return exec_.run_count(); }
// The execution time of an op. If it runs on accelerator, then it's
// accelerator_exec_micros(). Otherwise, it's CPU time.
int64_t exec_micros() const;
// The accelerator execution time of an op. 0 if not run on accelerator.
int64_t accelerator_exec_micros() const;
// The cpu execution time of an op.
int64_t cpu_exec_micros() const;
const std::map<std::string, std::vector<std::pair<int64_t, int64_t>>>&
op_execs() const {
return op_execs_;
}
const std::map<std::string, std::vector<std::pair<int64_t, int64_t>>>&
cpu_execs() const {
return cpu_execs_;
}
int64_t all_start_micros() const { return exec_.all_start_micros(); }
int64_t latest_end_micros() const { return exec_.latest_end_micros(); }
int64_t lastest_schedule_end_micros() const {
int64_t ret = 0;
for (const auto& exec : cpu_execs_) {
for (const auto& pair : exec.second) {
ret = std::max(ret, pair.first + pair.second);
}
}
return ret;
}
int64_t requested_bytes() const {
int64_t requested_bytes = 0;
for (const ExecMemory& exec : memory_execs_) {
requested_bytes += exec.requested_bytes();
}
return requested_bytes;
}
int64_t peak_bytes() const {
int64_t peak_bytes = 0;
for (const ExecMemory& exec : memory_execs_) {
peak_bytes += exec.peak_bytes();
}
return peak_bytes;
}
int64_t residual_bytes() const {
int64_t residual_bytes = 0;
for (const ExecMemory& exec : memory_execs_) {
residual_bytes += exec.residual_bytes();
}
return residual_bytes;
}
int64_t output_bytes() const {
int64_t output_bytes = 0;
for (const ExecMemory& exec : memory_execs_) {
output_bytes += exec.output_bytes();
}
return output_bytes;
}
int64_t accelerator_temp_bytes() const {
int64_t accelerator_temp_bytes = 0;
for (const ExecMemory& exec : memory_execs_) {
accelerator_temp_bytes += exec.accelerator_temp_bytes();
}
return accelerator_temp_bytes;
}
int64_t host_temp_bytes() const {
int64_t host_temp_bytes = 0;
for (const ExecMemory& exec : memory_execs_) {
host_temp_bytes += exec.host_temp_bytes();
}
return host_temp_bytes;
}
int64_t accelerator_persistent_bytes() const {
int64_t accelerator_persistent_bytes = 0;
for (const ExecMemory& exec : memory_execs_) {
accelerator_persistent_bytes += exec.accelerator_persistent_bytes();
}
return accelerator_persistent_bytes;
}
int64_t host_persistent_bytes() const {
int64_t host_persistent_bytes = 0;
for (const ExecMemory& exec : memory_execs_) {
host_persistent_bytes += exec.host_persistent_bytes();
}
return host_persistent_bytes;
}
std::map<int64_t, int64_t> allocator_bytes_in_use() const {
std::map<int64_t, int64_t> bytes_in_use;
for (const ExecMemory& exec : memory_execs_) {
bytes_in_use[exec.memory_micros()] = exec.allocator_bytes_in_use();
}
return bytes_in_use;
}
const std::vector<AllocationRecord>& allocations() const {
return allocations_;
}
const ExecProfile& ToProto() {
exec_.mutable_accelerator_execs()->clear();
for (const auto& e : accelerator_execs_) {
auto& exec_time = (*exec_.mutable_accelerator_execs())[e.first];
for (const auto& p : e.second) {
auto* t = exec_time.mutable_times()->Add();
t->add_int64_values(p.first);
t->add_int64_values(p.second);
}
}
exec_.mutable_cpu_execs()->clear();
for (const auto& e : cpu_execs_) {
auto& exec_time = (*exec_.mutable_cpu_execs())[e.first];
for (const auto& p : e.second) {
auto* t = exec_time.mutable_times()->Add();
t->add_int64_values(p.first);
t->add_int64_values(p.second);
}
}
exec_.mutable_devices()->Clear();
exec_.mutable_devices()->Reserve(devices_.size());
for (const std::string& d : devices_) {
exec_.add_devices(d);
}
exec_.mutable_allocations()->Clear();
for (const auto& r : allocations_) {
exec_.add_allocations()->MergeFrom(r);
}
exec_.mutable_memory_execs()->Clear();
for (const auto& m : memory_execs_) {
exec_.add_memory_execs()->MergeFrom(m);
}
return exec_;
}
void FromProto(const ExecProfile& exec) {
exec_.Clear();
exec_.MergeFrom(exec);
devices_.clear();
devices_.insert(exec.devices().begin(), exec.devices().end());
accelerator_execs_.clear();
cpu_execs_.clear();
op_execs_.clear();
allocations_.clear();
memory_execs_.clear();
for (const auto& exec_time : exec_.accelerator_execs()) {
auto& exec = accelerator_execs_[exec_time.first];
auto& op_exec = op_execs_[exec_time.first];
for (const auto& p : exec_time.second.times()) {
exec.push_back(std::make_pair(p.int64_values(0), p.int64_values(1)));
op_exec.push_back(std::make_pair(p.int64_values(0), p.int64_values(1)));
}
}
for (const auto& exec_time : exec_.cpu_execs()) {
auto& exec = cpu_execs_[exec_time.first];
auto& op_exec = op_execs_[exec_time.first];
for (const auto& p : exec_time.second.times()) {
exec.push_back(std::make_pair(p.int64_values(0), p.int64_values(1)));
op_exec.push_back(std::make_pair(p.int64_values(0), p.int64_values(1)));
}
}
for (const auto& r : exec_.allocations()) {
allocations_.push_back(r);
}
for (const auto& m : exec_.memory_execs()) {
memory_execs_.push_back(m);
}
}
private:
ExecProfile exec_;
// device -> vector of {op_start_micros, op_exec_micros} pairs.
// accelerator_execs: gpu:id/stream:all -> {op_start_micros, op_exec_micros}
// For accelerator, vector size can be larger than 1, multiple kernel fires
// or in tf.while_loop.
std::map<std::string, std::vector<std::pair<int64_t, int64_t>>>
accelerator_execs_;
// cpu_execs: cpu/gpu:id -> {op_start_micros, op_exec_micros}
// For cpu, vector size can be larger than 1 if in tf.while_loop.
std::map<std::string, std::vector<std::pair<int64_t, int64_t>>> cpu_execs_;
// combines accelerator_execs_ and cpu_execs_.
std::map<std::string, std::vector<std::pair<int64_t, int64_t>>> op_execs_;
// Each ExecMemory corresponds to one scheduling of the op. Normally,
// there are multiple schedulings in while_loop.
std::vector<ExecMemory> memory_execs_;
// All devices the op is associated with (e.g. gpu:0 (scheduling),
// gpu:0:stream:xx (kernel exec), cpu:0 host)
std::set<std::string> devices_;
// The history of accelerator allocations and deallocations of this step.
std::vector<AllocationRecord> allocations_;
};
#define GRAPH_NODE_BYTES(type) \
do { \
if (execs_.empty()) { \
return 0; \
} \
if (step >= 0) { \
auto exec = execs_.find(step); \
if (exec == execs_.end()) return 0; \
return exec->second.type##_bytes(); \
} \
\
int64_t bytes = 0; \
for (const auto& exec : execs_) { \
bytes += exec.second.type##_bytes(); \
} \
return bytes / execs_.size(); \
} while (0)
class TFGraphNode {
public:
TFGraphNode(
const ProfileNode& node, const ProfileProto& profile,
const std::map<int64_t, std::string>* id_to_string,
const std::map<std::string, std::unique_ptr<TFGraphNode>>* nodes_map) {
nodes_map_ = nodes_map;
FromProto(node, profile, id_to_string);
}
TFGraphNode(
const NodeDef* node, int64_t id,
const std::map<std::string, std::unique_ptr<TFGraphNode>>* nodes_map) {
nodes_map_ = nodes_map;
node_.set_id(id);
node_.set_name(node->name());
node_.set_op(node->op());
node_.set_float_ops(0);
for (const auto& attr : node->attr()) {
(*node_.mutable_attrs())[attr.first].MergeFrom(attr.second);
if (attr.first == "shape" && attr.second.has_shape()) {
if (!shape_.empty()) {
absl::FPrintF(stderr, "Found duplicated shapes!\n");
continue;
}
shape_ = ShapeProtoToVec(attr.second.shape());
} else if (attr.first == "_output_shapes" && attr.second.has_list()) {
if (!output_shapes_.empty()) {
absl::FPrintF(stderr, "Found duplicated output shapes!\n");
continue;
}
for (int i = 0; i < attr.second.list().shape_size(); ++i) {
output_shapes_[i] = ShapeProtoToVec(attr.second.list().shape(i));
}
}
}
op_types_.insert(node->op());
}
void AddInput(const std::string& input, int64_t output_index, int input_idx) {
inputs_[input_idx] = input;
src_output_idx_[input] = output_index;
}
void AddOpType(const std::string& op_type) { op_types_.insert(op_type); }
void AddStepStat(int64_t step, const std::string& device,
const NodeExecStats& step_stat);
void AddFloatOps(int64_t float_ops) { node_.set_float_ops(float_ops); }
// TODO(xpan): This could take a lot of memory.
void AddCode(const CodeDef& code,
const std::map<int64_t, std::string>* id_to_string) {
if (!call_stack_) {
call_stack_ = std::make_unique<CallStack>(code, id_to_string);
}
}
const std::string& name() const { return node_.name(); }
int64_t id() const { return node_.id(); }
const std::string& op() const { return node_.op(); }
const ProfileNode& node() { return node_; }
bool trackable(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) return false;
if (exec->second.all_start_micros() == 0) return false;
if (node_.canonical_device().empty() || node_.host_device().empty()) {
return false;
}
return true;
}
const ProfileNode& ToProto(
const std::map<std::string, std::unique_ptr<TFGraphNode>>& nodes_map) {
node_.clear_shape();
node_.mutable_shape()->Reserve(shape().size());
for (int64_t s : shape()) {
node_.add_shape(s);
}
node_.clear_op_types();
node_.mutable_op_types()->Reserve(op_types().size());
for (const std::string& t : op_types()) {
node_.add_op_types(t);
}
node_.clear_execs();
for (auto& exec : execs_) {
auto& exec_pb = (*node_.mutable_execs())[exec.first];
exec_pb.MergeFrom(exec.second.ToProto());
}
node_.clear_inputs();
for (const auto& inp : inputs_) {
(*node_.mutable_inputs())[inp.first] = nodes_map.at(inp.second)->id();
}
node_.clear_input_shapes();
for (const auto& s : input_shapes_) {
auto& shape = (*node_.mutable_input_shapes())[s.first];
for (int64_t d : s.second) {
shape.add_int64_values(d);
}
}
node_.clear_output_shapes();
for (const auto& s : output_shapes_) {
auto& shape = (*node_.mutable_output_shapes())[s.first];
for (int64_t d : s.second) {
shape.add_int64_values(d);
}
}
node_.clear_src_output_index();
for (const auto& s : src_output_idx_) {
int64_t id = nodes_map.at(s.first)->id();
(*node_.mutable_src_output_index())[id] = s.second;
}
if (call_stack_) {
node_.clear_trace();
node_.mutable_trace()->MergeFrom(call_stack_->code_def());
}
return node_;
}
void FromProto(const ProfileNode& node, const ProfileProto& profile,
const std::map<int64_t, std::string>* id_to_string) {
node_.Clear();
node_.MergeFrom(node);
call_stack_ = std::make_unique<CallStack>(node.trace(), id_to_string);
op_types_.clear();
op_types_.insert(node_.op_types().begin(), node_.op_types().end());
shape_.clear();
for (int64_t s : node_.shape()) {
shape_.push_back(s);
}
execs_.clear();
for (const auto& exec_pb : node.execs()) {
auto& exec = execs_[exec_pb.first];
exec.FromProto(exec_pb.second);
}
inputs_.clear();
for (const auto& inp : node.inputs()) {
inputs_[inp.first] = profile.nodes().at(inp.second).name();
}
input_shapes_.clear();
for (const auto& s : node.input_shapes()) {
auto& shape = input_shapes_[s.first];
for (const int64_t d : s.second.int64_values()) {
shape.push_back(d);
}
}
output_shapes_.clear();
for (const auto& s : node.output_shapes()) {
auto& shape = output_shapes_[s.first];
for (const int64_t d : s.second.int64_values()) {
shape.push_back(d);
}
}
src_output_idx_.clear();
for (const auto& s : node.src_output_index()) {
src_output_idx_[profile.nodes().at(s.first).name()] = s.second;
}
}
const std::map<int32_t, std::string>& inputs() const { return inputs_; }
// Number of times the graph node is executed. When step < 0, the
// average number of times executed across all steps.
int64_t run_count(int64_t step) const {
if (execs_.empty()) {
return 0;
}
if (step >= 0) {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return 0;
}
return exec->second.run_count();
}
int64_t total_run_count = 0;
for (const auto& exec : execs_) {
total_run_count += exec.second.run_count();
}
return total_run_count / execs_.size();
}
// This is overall computation time, including both cpu and accelerator.
// Note, cpu and accelerator might or might not run in parallel.
int64_t exec_micros(int64_t step) const {
// Empty when no RunMetadata is provided.
if (execs_.empty()) {
return 0;
}
if (step >= 0) {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return 0;
}
return exec->second.exec_micros();
}
int64_t total_micros = 0;
for (const auto& exec : execs_) {
total_micros += exec.second.exec_micros();
}
return total_micros / execs_.size();
}
// This is accelerator computation time of a step, or average of
// multiple step, when step < 0.
int64_t accelerator_exec_micros(int64_t step) const {
// Empty when no RunMetadata is provided.
if (execs_.empty()) {
return 0;
}
if (step >= 0) {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return 0;
}
return exec->second.accelerator_exec_micros();
}
int64_t total_micros = 0;
for (const auto& exec : execs_) {
total_micros += exec.second.accelerator_exec_micros();
}
return total_micros / execs_.size();
}
// This is cpu computation time of a step, or average of
// multiple step, when step < 0.
int64_t cpu_exec_micros(int64_t step) const {
// Empty when no RunMetadata is provided.
if (execs_.empty()) {
return 0;
}
if (step >= 0) {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return 0;
}
return exec->second.cpu_exec_micros();
}
int64_t total_micros = 0;
for (const auto& exec : execs_) {
total_micros += exec.second.cpu_exec_micros();
}
return total_micros / execs_.size();
}
int64_t requested_bytes(int64_t step) const { GRAPH_NODE_BYTES(requested); }
int64_t peak_bytes(int64_t step) const { GRAPH_NODE_BYTES(peak); }
int64_t residual_bytes(int64_t step) const { GRAPH_NODE_BYTES(residual); }
int64_t output_bytes(int64_t step) const { GRAPH_NODE_BYTES(output); }
int64_t all_start_micros(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return 0;
}
return exec->second.all_start_micros();
}
int64_t latest_end_micros(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return 0;
}
return exec->second.latest_end_micros();
}
int64_t lastest_schedule_end_micros(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return 0;
}
return exec->second.lastest_schedule_end_micros();
}
const std::map<std::string, std::vector<std::pair<int64_t, int64_t>>>&
op_execs(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return empty_execs_;
}
return exec->second.op_execs();
}
const std::map<std::string, std::vector<std::pair<int64_t, int64_t>>>&
cpu_execs(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return empty_execs_;
}
return exec->second.cpu_execs();
}
const std::map<int64_t, ExecStep>& all_op_execs() const { return execs_; }
int64_t accelerator_temp_bytes(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return 0;
}
return exec->second.accelerator_temp_bytes();
}
int64_t host_temp_bytes(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return 0;
}
return exec->second.host_temp_bytes();
}
int64_t accelerator_persistent_bytes() const {
int64_t persistent_bytes = 0;
for (const auto& exec : execs_) {
persistent_bytes = std::max(persistent_bytes,
exec.second.accelerator_persistent_bytes());
}
return persistent_bytes;
}
std::map<int64_t, int64_t> allocator_bytes_in_use(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return empty_bytes_in_use_;
}
return exec->second.allocator_bytes_in_use();
}
const std::vector<AllocationRecord>& allocations(int64_t step) const {
auto exec = execs_.find(step);
if (exec == execs_.end()) {
return empty_allocations_;
}
return exec->second.allocations();
}
int64_t parameters() const {
if (!shape().empty()) {
int64_t params = 1;
bool complete_shape = true;
for (int64_t d : shape()) {
// Sometimes parameters could be <0 when a dim is unknown.
if (d < 0) {
complete_shape = false;
break;
}
params *= d;
}
if (complete_shape) {
return params;
} else {
LOG(INFO) << "Incomplete shape.\n";
}
}
return 0;
}
int64_t float_ops(int64_t step) const {
// If not run, return static analysis.
if (execs_.empty()) {
return node_.float_ops();
}
// Otherwise, return dynamic float_ops.
return node_.float_ops() * run_count(step);
}
const CallStack* call_stack() { return call_stack_.get(); }
std::string canonical_device() const { return node_.canonical_device(); }
std::string host_device() const { return node_.host_device(); }
const std::set<std::string>& op_types() const { return op_types_; }
const AttrValue* op_attrs(const std::string& name) const {
const auto it = node_.attrs().find(name);
if (it == node_.attrs().end()) {
return nullptr;
}
return &it->second;
}
const std::vector<int64_t>& shape() const { return shape_; }
const std::map<int, std::vector<int64_t>>& output_shapes() const {
return output_shapes_;
}
std::map<int, std::vector<int64_t>> input_shapes() const {
std::map<int, std::vector<int64_t>> input_shapes;
for (const auto& inp : inputs_) {
// Always create an empty vec even if the shape info might be missing.
std::vector<int64_t>& shape_vec = input_shapes[inp.first];
if (!nodes_map_) continue;
auto input_it = nodes_map_->find(inp.second);
if (input_it == nodes_map_->end()) continue;
auto output_it = src_output_idx_.find(inp.second);
if (output_it == src_output_idx_.end()) continue;
const TFGraphNode* input_node = input_it->second.get();
if (!input_node) continue;
const auto& output_shapes = input_node->output_shapes();
const auto& output_shape = output_shapes.find(output_it->second);
if (output_shape == output_shapes.end()) continue;
if (output_shape != input_node->output_shapes().end()) {
shape_vec.assign(output_shape->second.begin(),
output_shape->second.end());
}
}
return input_shapes;
}
private:
// maps graph node name to TFGraphNode. Not owned.
const std::map<std::string, std::unique_ptr<TFGraphNode>>* nodes_map_;
// inputs to the node. input index -> input node name.
std::map<int, std::string> inputs_;
// The output index of the source node.
std::map<std::string, int32_t> src_output_idx_;
// proto for serialize/deserialized representation of the node.
ProfileNode node_;
// Python call stack that creates the name.
std::unique_ptr<CallStack> call_stack_;
// Shape of the node (e.g. Variable) if available.
std::vector<int64_t> shape_;
// Won't missing input_idx. But some shapes might be empty (unknown).
std::map<int, std::vector<int64_t>> input_shapes_;
// Could miss output_idx if no _output_shapes attr. some shapes can also
// be empty.
std::map<int, std::vector<int64_t>> output_shapes_;
std::set<std::string> op_types_;
std::map<int64_t, ExecStep> execs_;
// Placeholder for empty cases.
std::map<int64_t, int64_t> empty_bytes_in_use_;
std::map<std::string, std::vector<std::pair<int64_t, int64_t>>> empty_execs_;
std::vector<AllocationRecord> empty_allocations_;
};
class TFMultiGraphNode {
public:
TFMultiGraphNode(const std::string& name)
: name_(name),
step_(-1),
run_count_(0),
exec_micros_(0),
accelerator_exec_micros_(0),
cpu_exec_micros_(0),
requested_bytes_(0),
peak_bytes_(0),
residual_bytes_(0),
output_bytes_(0),
float_ops_(0),
parameters_(0) {}
bool SnapshotNodes(int64_t step,
const std::vector<std::string>& type_regexes) {
run_count_ = 0;
exec_micros_ = 0;
accelerator_exec_micros_ = 0;
cpu_exec_micros_ = 0;
requested_bytes_ = 0;
peak_bytes_ = 0;
residual_bytes_ = 0;
output_bytes_ = 0;
float_ops_ = 0;
parameters_ = 0;
op_types_.clear();
shapes_.clear();
devices_.clear();
snapshot_nodes_.clear();
step_ = step;
std::vector<const TFGraphNode*> nodes = pick_nodes(type_regexes);
if (nodes.empty()) {
return (type_regexes.size() == 1 && type_regexes[0] == ".*");
}
for (const TFGraphNode* node : nodes) {
op_types_.insert(node->op_types().begin(), node->op_types().end());
run_count_ += node->run_count(step);
exec_micros_ += node->exec_micros(step);
accelerator_exec_micros_ += node->accelerator_exec_micros(step);
cpu_exec_micros_ += node->cpu_exec_micros(step);
requested_bytes_ += node->requested_bytes(step);
peak_bytes_ += node->peak_bytes(step);
residual_bytes_ += node->residual_bytes(step);
output_bytes_ += node->output_bytes(step);
float_ops_ += node->float_ops(step);
parameters_ += node->parameters();
if (!node->shape().empty()) {
shapes_.push_back(node->shape());
}
devices_.insert(node->canonical_device());
snapshot_nodes_[node->name()] = node;
}
return true;
}
int64_t step() const { return step_; }
void AddGraphNode(const TFGraphNode* node) {
if (nodes_.find(node->name()) != nodes_.end()) {
return;
}
nodes_[node->name()] = node;
}
const std::map<std::string, const TFGraphNode*>& graph_nodes() const {
return snapshot_nodes_;
}
const std::string& name() const { return name_; }
int64_t run_count() const { return run_count_; }
int64_t exec_micros() const { return exec_micros_; }
int64_t accelerator_exec_micros() const { return accelerator_exec_micros_; }
int64_t cpu_exec_micros() const { return cpu_exec_micros_; }
int64_t requested_bytes() const { return requested_bytes_; }
int64_t peak_bytes() const { return peak_bytes_; }
int64_t residual_bytes() const { return residual_bytes_; }
int64_t output_bytes() const { return output_bytes_; }
int64_t float_ops() const { return float_ops_; }
int64_t parameters() const { return parameters_; }
const std::set<std::string>& devices() const { return devices_; }
const std::set<std::string>& op_types() const { return op_types_; }
const std::vector<std::vector<int64_t>>& shapes() const { return shapes_; }
private:
std::vector<const TFGraphNode*> pick_nodes(
const std::vector<std::string>& type_regexes) {
if (type_regexes.empty()) {
return {};
}
std::vector<const TFGraphNode*> ret;
if (type_regexes.size() == 1 && type_regexes[0] == ".*") {
for (const auto& n : nodes_) {
ret.push_back(n.second);
}
return ret;
}
for (const std::string& regex : type_regexes) {
for (const auto& n : nodes_) {
for (const std::string& type : n.second->op_types()) {
if (RE2::FullMatch(type, regex)) {
ret.push_back(n.second);
break;
}
}
}
}
return ret;
}
const std::string name_;
int64_t step_;
// Snapshot based on type_regexes
std::set<std::string> op_types_;
int64_t run_count_;
int64_t exec_micros_;
int64_t accelerator_exec_micros_;
int64_t cpu_exec_micros_;
int64_t requested_bytes_;
int64_t peak_bytes_;
int64_t residual_bytes_;
int64_t output_bytes_;
int64_t float_ops_;
int64_t parameters_;
std::set<std::string> devices_;
std::vector<std::vector<int64_t>> shapes_;
std::map<std::string, const TFGraphNode*> snapshot_nodes_;
// Overall data held by the TFMultiGraphNode.
std::map<std::string, const TFGraphNode*> nodes_;
};
bool IsPlacedOnCPU(const std::string& device);
bool IsPlacedOnAccelerator(const std::string& device);
bool CountAsAcceleratorTime(const std::string& device);
bool CountAsCPUTime(const std::string& device);
bool IsCanonicalDevice(const std::string& device);
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_NODE_H_
tensorflow/core/profiler/internal/tfprof_node_show.cc
+238
View File
@@ -0,0 +1,238 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_node_show.h"
#include <vector>
namespace tensorflow {
namespace tfprof {
namespace {}
ShowNode::ShowNode(const TFGraphNode* node) : node(node), account(false) {
ReInit(-1);
}
void ShowNode::ReInit(int64_t step) {
mutable_proto()->set_name(name());
mutable_proto()->clear_devices();
if (!node->canonical_device().empty()) {
mutable_proto()->add_devices(node->canonical_device());
}
mutable_proto()->set_run_count(node->run_count(step));
mutable_proto()->set_exec_micros(node->exec_micros(step));
mutable_proto()->set_accelerator_exec_micros(
node->accelerator_exec_micros(step));
mutable_proto()->set_cpu_exec_micros(node->cpu_exec_micros(step));
mutable_proto()->set_requested_bytes(node->requested_bytes(step));
mutable_proto()->set_peak_bytes(node->peak_bytes(step));
mutable_proto()->set_residual_bytes(node->residual_bytes(step));
mutable_proto()->set_output_bytes(node->output_bytes(step));
mutable_proto()->set_float_ops(node->float_ops(step));
mutable_proto()->clear_input_shapes();
for (const auto& inp : node->input_shapes()) {
(*mutable_proto()->mutable_input_shapes())[inp.first].MergeFrom(
VecToShapeProto(inp.second));
}
proto_.set_parameters(node->parameters());
}
GraphNodeProto* ShowNode::mutable_proto() { return &proto_; }
const GraphNodeProto& ShowNode::proto() const { return proto_; }
void ShowNode::AggregateTotalStats(ShowNode* node) {
GraphNodeProto* node_pb = node->mutable_proto();
mutable_proto()->set_total_run_count(proto().total_run_count() +
node_pb->total_run_count());
mutable_proto()->set_total_definition_count(
proto().total_definition_count() + node_pb->total_definition_count());
mutable_proto()->set_total_exec_micros(proto().total_exec_micros() +
node_pb->total_exec_micros());
mutable_proto()->set_total_accelerator_exec_micros(
proto().total_accelerator_exec_micros() +
node_pb->total_accelerator_exec_micros());
mutable_proto()->set_total_cpu_exec_micros(proto().total_cpu_exec_micros() +
node_pb->total_cpu_exec_micros());
mutable_proto()->set_total_requested_bytes(proto().total_requested_bytes() +
node_pb->total_requested_bytes());
mutable_proto()->set_total_peak_bytes(proto().total_peak_bytes() +
node_pb->total_peak_bytes());
mutable_proto()->set_total_residual_bytes(proto().total_residual_bytes() +
node_pb->total_residual_bytes());
mutable_proto()->set_total_output_bytes(proto().total_output_bytes() +
node_pb->total_output_bytes());
mutable_proto()->set_total_parameters(proto().total_parameters() +
node_pb->total_parameters());
mutable_proto()->set_total_float_ops(proto().total_float_ops() +
node_pb->total_float_ops());
}
void ShowNode::AddSelfToTotalStats() {
mutable_proto()->set_total_definition_count(proto().total_definition_count() +
1);
mutable_proto()->set_total_run_count(proto().total_run_count() +
proto().run_count());
mutable_proto()->set_total_exec_micros(proto().total_exec_micros() +
proto().exec_micros());
mutable_proto()->set_total_accelerator_exec_micros(
proto().total_accelerator_exec_micros() +
proto().accelerator_exec_micros());
mutable_proto()->set_total_cpu_exec_micros(proto().total_cpu_exec_micros() +
proto().cpu_exec_micros());
mutable_proto()->set_total_requested_bytes(proto().total_requested_bytes() +
proto().requested_bytes());
mutable_proto()->set_total_peak_bytes(proto().total_peak_bytes() +
proto().peak_bytes());
mutable_proto()->set_total_residual_bytes(proto().total_residual_bytes() +
proto().residual_bytes());
mutable_proto()->set_total_output_bytes(proto().total_output_bytes() +
proto().output_bytes());
mutable_proto()->set_total_parameters(proto().total_parameters() +
proto().parameters());
mutable_proto()->set_total_float_ops(proto().total_float_ops() +
proto().float_ops());
}
void ShowNode::ResetTotalStats() {
formatted_str.clear();
mutable_proto()->set_total_definition_count(0);
mutable_proto()->set_total_run_count(0);
mutable_proto()->set_total_exec_micros(0);
mutable_proto()->set_total_accelerator_exec_micros(0);
mutable_proto()->set_total_cpu_exec_micros(0);
mutable_proto()->set_total_requested_bytes(0);
mutable_proto()->set_total_peak_bytes(0);
mutable_proto()->set_total_residual_bytes(0);
mutable_proto()->set_total_output_bytes(0);
mutable_proto()->set_total_parameters(0);
mutable_proto()->set_total_float_ops(0);
mutable_proto()->mutable_children()->Clear();
}
ShowMultiNode::ShowMultiNode(TFMultiGraphNode* node)
: node(node), account(false), show(false) {
ReInit(-1, {".*"});
}
bool ShowMultiNode::ReInit(int64_t step,
const std::vector<std::string>& type_regexes) {
bool has_matched_type = node->SnapshotNodes(step, type_regexes);
std::vector<ShowNode> snodes;
mutable_proto()->mutable_graph_nodes()->Clear();
for (const auto& it : node->graph_nodes()) {
ShowNode snode(it.second);
snodes.push_back(snode);
snodes.back().ReInit(step);
snodes.back().AddSelfToTotalStats();
mutable_proto()->add_graph_nodes()->MergeFrom(snodes.back().proto());
}
mutable_proto()->set_name(name());
mutable_proto()->set_exec_micros(node->exec_micros());
mutable_proto()->set_accelerator_exec_micros(node->accelerator_exec_micros());
mutable_proto()->set_cpu_exec_micros(node->cpu_exec_micros());
mutable_proto()->set_requested_bytes(node->requested_bytes());
mutable_proto()->set_peak_bytes(node->peak_bytes());
mutable_proto()->set_residual_bytes(node->residual_bytes());
mutable_proto()->set_output_bytes(node->output_bytes());
mutable_proto()->set_float_ops(node->float_ops());
mutable_proto()->set_parameters(node->parameters());
return has_matched_type;
}
MultiGraphNodeProto* ShowMultiNode::mutable_proto() { return &proto_; }
const MultiGraphNodeProto& ShowMultiNode::proto() const { return proto_; }
void ShowMultiNode::AggregateTotalStats(ShowMultiNode* node) {
MultiGraphNodeProto* node_pb = node->mutable_proto();
mutable_proto()->set_total_exec_micros(proto().total_exec_micros() +
node_pb->total_exec_micros());
mutable_proto()->set_total_accelerator_exec_micros(
proto().total_accelerator_exec_micros() +
node_pb->total_accelerator_exec_micros());
mutable_proto()->set_total_cpu_exec_micros(proto().total_cpu_exec_micros() +
node_pb->total_cpu_exec_micros());
mutable_proto()->set_total_requested_bytes(proto().total_requested_bytes() +
node_pb->total_requested_bytes());
mutable_proto()->set_total_peak_bytes(proto().total_peak_bytes() +
node_pb->total_peak_bytes());
mutable_proto()->set_total_residual_bytes(proto().total_residual_bytes() +
node_pb->total_residual_bytes());
mutable_proto()->set_total_output_bytes(proto().total_output_bytes() +
node_pb->total_output_bytes());
mutable_proto()->set_total_parameters(proto().total_parameters() +
node_pb->total_parameters());
mutable_proto()->set_total_float_ops(proto().total_float_ops() +
node_pb->total_float_ops());
}
void ShowMultiNode::AddSelfToTotalStats() {
mutable_proto()->set_total_exec_micros(proto().total_exec_micros() +
proto().exec_micros());
mutable_proto()->set_total_accelerator_exec_micros(
proto().total_accelerator_exec_micros() +
proto().accelerator_exec_micros());
mutable_proto()->set_total_cpu_exec_micros(proto().total_cpu_exec_micros() +
proto().cpu_exec_micros());
mutable_proto()->set_total_requested_bytes(proto().total_requested_bytes() +
proto().requested_bytes());
mutable_proto()->set_total_peak_bytes(proto().total_peak_bytes() +
proto().peak_bytes());
mutable_proto()->set_total_residual_bytes(proto().total_residual_bytes() +
proto().residual_bytes());
mutable_proto()->set_total_output_bytes(proto().total_output_bytes() +
proto().output_bytes());
mutable_proto()->set_total_parameters(proto().total_parameters() +
proto().parameters());
mutable_proto()->set_total_float_ops(proto().total_float_ops() +
proto().float_ops());
}
void ShowMultiNode::ResetTotalStats() {
formatted_str.clear();
mutable_proto()->set_total_exec_micros(0);
mutable_proto()->set_total_accelerator_exec_micros(0);
mutable_proto()->set_total_cpu_exec_micros(0);
mutable_proto()->set_total_requested_bytes(0);
mutable_proto()->set_total_peak_bytes(0);
mutable_proto()->set_total_residual_bytes(0);
mutable_proto()->set_total_output_bytes(0);
mutable_proto()->set_total_parameters(0);
mutable_proto()->set_total_float_ops(0);
mutable_proto()->mutable_children()->Clear();
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_node_show.h
+160
View File
@@ -0,0 +1,160 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Node classes used for different views. They are wrappers with "show"
// methods.
//
// ScopeNode is for scope view. GraphNode is for graph view, CodeNode
// is for code view and OpNode for op view.
// ScopeNode and GraphNode each maps to one TFGraphNode.
// CodeNode and OpNode each maps to one TFMultiGraphNode.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_NODE_SHOW_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_NODE_SHOW_H_
#include <algorithm>
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class ShowNode {
public:
explicit ShowNode(const TFGraphNode* node);
virtual ~ShowNode() = default;
const std::string& name() const { return node->name(); }
GraphNodeProto* mutable_proto();
const GraphNodeProto& proto() const;
void ReInit(int64_t step);
void AggregateTotalStats(ShowNode* node);
void AddSelfToTotalStats();
void ResetTotalStats();
const TFGraphNode* node;
bool account;
std::string formatted_str;
protected:
GraphNodeProto proto_;
};
class GraphNode : public ShowNode {
public:
explicit GraphNode(TFGraphNode* node) : ShowNode(node) {}
bool Trackable(int64_t step) const { return node->trackable(step); }
std::vector<GraphNode*> children;
std::vector<GraphNode*> show_children;
};
class ScopeNode : public ShowNode {
public:
explicit ScopeNode(const TFGraphNode* node) : ShowNode(node) {}
~ScopeNode() override = default;
std::vector<ScopeNode*> children;
std::vector<ScopeNode*> show_children;
};
class ShowMultiNode {
public:
explicit ShowMultiNode(TFMultiGraphNode* node);
virtual ~ShowMultiNode() = default;
bool ReInit(int64_t step, const std::vector<std::string>& type_regexes);
const std::string& name() const { return node->name(); }
MultiGraphNodeProto* mutable_proto();
const MultiGraphNodeProto& proto() const;
void AggregateTotalStats(ShowMultiNode* node);
void AddSelfToTotalStats();
void ResetTotalStats();
TFMultiGraphNode* node;
bool account;
bool show;
std::string formatted_str;
protected:
MultiGraphNodeProto proto_;
};
class CodeNode : public ShowMultiNode {
public:
CodeNode(TFMultiGraphNode* node, const CallStack::Trace* trace,
const std::string& suffix)
: ShowMultiNode(node), trace_(trace), suffix_(suffix) {}
~CodeNode() override = default;
CodeNode* AddChildren(const std::string& name, const CallStack::Trace* trace,
const std::string suffix) {
auto it = children_.find(name);
if (it != children_.end()) {
return it->second.get();
}
graph_children_.push_back(std::make_unique<TFMultiGraphNode>(name));
auto child = &children_[name];
*child =
std::make_unique<CodeNode>(graph_children_.back().get(), trace, suffix);
children.push_back(child->get());
return child->get();
}
bool has_trace() const { return trace_ != nullptr; }
int32_t lineno() const { return trace_->lineno(); }
std::string file() const { return trace_->file(); }
std::string function() const { return trace_->function() + suffix_; }
int32_t func_start_line() const { return trace_->func_start_line(); }
std::vector<CodeNode*> children;
std::vector<CodeNode*> show_children;
private:
const CallStack::Trace* trace_;
std::string suffix_;
std::vector<std::unique_ptr<TFMultiGraphNode>> graph_children_;
std::map<std::string, std::unique_ptr<CodeNode>> children_;
};
class OpNode : public ShowMultiNode {
public:
explicit OpNode(TFMultiGraphNode* node) : ShowMultiNode(node) {}
~OpNode() override = default;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_NODE_SHOW_H_
tensorflow/core/profiler/internal/tfprof_op.cc
+326
View File
@@ -0,0 +1,326 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_op.h"
#include <stdio.h>
#include <algorithm>
#include <memory>
#include <set>
#include <utility>
#include <vector>
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "tensorflow/core/platform/regexp.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tensorflow/core/profiler/internal/tfprof_tensor.h"
namespace tensorflow {
namespace tfprof {
namespace {
std::string FormatToalExecTime(const ShowMultiNode* node,
const ShowMultiNode* root) {
double accu_pct = 0.0;
double pct = 0.0;
if (node->proto().total_exec_micros() > 0) {
accu_pct = 100.0 * node->proto().total_exec_micros() /
root->proto().total_exec_micros();
pct =
100.0 * node->proto().exec_micros() / root->proto().total_exec_micros();
}
return absl::StrFormat(
"%30s",
absl::StrFormat("%s (%.2f%%, %.2f%%)",
FormatTime(node->proto().exec_micros()), accu_pct, pct));
}
std::string FormatCPUExecTime(const ShowMultiNode* node,
const ShowMultiNode* root) {
double accu_pct = 0.0;
double pct = 0.0;
if (node->proto().total_cpu_exec_micros() > 0) {
accu_pct = 100.0 * node->proto().total_cpu_exec_micros() /
root->proto().total_cpu_exec_micros();
pct = 100.0 * node->proto().cpu_exec_micros() /
root->proto().total_cpu_exec_micros();
}
return absl::StrFormat(
"%30s", absl::StrFormat("%s (%.2f%%, %.2f%%)",
FormatTime(node->proto().cpu_exec_micros()),
accu_pct, pct));
}
std::string FormatAcceleratorExecTime(const ShowMultiNode* node,
const ShowMultiNode* root) {
double accu_pct = 0.0;
double pct = 0.0;
if (node->proto().total_accelerator_exec_micros() > 0) {
accu_pct = 100.0 * node->proto().total_accelerator_exec_micros() /
root->proto().total_accelerator_exec_micros();
pct = 100.0 * node->proto().accelerator_exec_micros() /
root->proto().total_accelerator_exec_micros();
}
return absl::StrFormat(
"%30s",
absl::StrFormat("%s (%.2f%%, %.2f%%)",
FormatTime(node->proto().accelerator_exec_micros()),
accu_pct, pct));
}
} // namespace
void TFOp::AddNode(TFGraphNode* node) {
const std::string& op = node->op();
if (tfcnodes_map_.find(op) == tfcnodes_map_.end()) {
tfcnodes_map_[op] = std::make_unique<TFMultiGraphNode>(op);
}
TFMultiGraphNode* tfcnode = tfcnodes_map_[op].get();
tfcnode->AddGraphNode(node);
}
void TFOp::Build() {
for (auto& tn : tfcnodes_map_) {
cnodes_map_[tn.first] = std::make_unique<OpNode>(tn.second.get());
}
tfcnodes_map_[kTFProfRoot] = std::make_unique<TFMultiGraphNode>(kTFProfRoot);
root_ = std::make_unique<OpNode>(tfcnodes_map_[kTFProfRoot].get());
}
const ShowMultiNode* TFOp::ShowInternal(const Options& opts,
Timeline* timeline) {
root_->ResetTotalStats();
if (opts.output_type == kOutput[3]) {
absl::FPrintF(stderr, "Only 'code' view supports pprof output now.\n");
return root_.get();
}
if (opts.output_type == kOutput[1] || opts.output_type == kOutput[2]) {
root_->formatted_str = FormatNode(root_.get(), root_.get(), opts);
}
if (timeline) {
absl::FPrintF(stderr,
"op view doesn't support timeline yet. "
"Consider graph/scope/code view.\n");
return root_.get();
}
if (cnodes_map_.empty()) {
return root_.get();
}
std::vector<OpNode*> nodes;
for (auto& n : cnodes_map_) {
n.second->account = ReAccount(n.second.get(), opts);
n.second->ResetTotalStats();
n.second->AddSelfToTotalStats();
nodes.push_back(n.second.get());
}
nodes = SortNodes(nodes, opts);
// pre keeps track of previous visited node.
OpNode* pre = nullptr;
std::vector<OpNode*> account_nodes;
for (auto it = nodes.rbegin(); it != nodes.rend(); ++it) {
if ((*it)->account) {
if (pre) (*it)->AggregateTotalStats(pre);
account_nodes.push_back(*it);
pre = *it;
}
}
std::reverse(std::begin(account_nodes), std::end(account_nodes));
if (pre) {
root_->AggregateTotalStats(pre);
}
// Perform the display and optionally redo accounting.
int64_t depth = 0;
std::vector<OpNode*> show_nodes;
int64_t start = SearchRoot(account_nodes, opts.start_name_regexes);
for (int64_t i = start, end = account_nodes.size(); i < end; ++i, ++depth) {
OpNode* n = account_nodes[i];
if (ShouldTrim(n, opts.trim_name_regexes) || depth > opts.max_depth) {
break;
}
n->show = ShouldShow(n, opts, depth);
if (n->show) show_nodes.push_back(n);
}
pre = nullptr;
for (auto it = show_nodes.rbegin(); it != show_nodes.rend(); ++it) {
if (opts.account_displayed_op_only) {
(*it)->ResetTotalStats();
(*it)->AddSelfToTotalStats();
if (pre) (*it)->AggregateTotalStats(pre);
}
pre = *it;
}
if (opts.account_displayed_op_only) {
root_->ResetTotalStats();
if (pre) {
root_->AggregateTotalStats(pre);
}
}
if (opts.output_type == kOutput[1] || opts.output_type == kOutput[2]) {
std::string display_str = FormatLegend(opts);
for (OpNode* node : show_nodes) {
display_str += FormatNode(node, root_.get(), opts);
}
// In op view, we don't show root (total). But it will still in proto.
// TODO(xpan): Is it the right choice?
root_->formatted_str = display_str;
}
// Populate the children field.
auto* pre_pb = root_->mutable_proto();
for (auto& show_node : show_nodes) {
pre_pb->clear_children();
pre_pb->add_children()->Swap(show_node->mutable_proto());
pre_pb = pre_pb->mutable_children(0);
}
return root_.get();
}
int64_t TFOp::SearchRoot(const std::vector<OpNode*> nodes,
const std::vector<std::string>& regexes) {
if (regexes.empty() || (regexes.size() == 1 && regexes[0] == ".*")) {
return 0;
}
int64_t i = 0;
const int64_t nodes_size = nodes.size();
for (; i < nodes_size; ++i) {
for (const std::string& regex : regexes) {
if (RE2::FullMatch(nodes[i]->name(), regex)) {
return i;
}
}
}
return i;
}
std::string TFOp::FormatMemoryNode(int64_t node_total_bytes,
int64_t root_total_bytes,
int64_t node_bytes) const {
double accu_pct = 0.0;
double pct = 0.0;
if (node_bytes > 0) {
accu_pct = 100.0 * node_total_bytes / root_total_bytes;
pct = 100.0 * node_bytes / root_total_bytes;
}
return absl::StrFormat(
"%30s", absl::StrFormat("%s (%.2f%%, %.2f%%)", FormatMemory(node_bytes),
accu_pct, pct));
}
std::string TFOp::FormatNode(OpNode* node, OpNode* root,
const Options& opts) const {
std::vector<std::string> attrs;
if (opts.select.find(kShown[0]) != opts.select.end()) {
attrs.push_back(FormatMemoryNode(node->proto().total_requested_bytes(),
root->proto().total_requested_bytes(),
node->proto().requested_bytes()));
}
if (opts.select.find(kShown[11]) != opts.select.end()) {
attrs.push_back(FormatMemoryNode(node->proto().total_peak_bytes(),
root->proto().total_peak_bytes(),
node->proto().peak_bytes()));
}
if (opts.select.find(kShown[12]) != opts.select.end()) {
attrs.push_back(FormatMemoryNode(node->proto().total_residual_bytes(),
root->proto().total_residual_bytes(),
node->proto().residual_bytes()));
}
if (opts.select.find(kShown[13]) != opts.select.end()) {
attrs.push_back(FormatMemoryNode(node->proto().total_output_bytes(),
root->proto().total_output_bytes(),
node->proto().output_bytes()));
}
if (opts.select.find(kShown[1]) != opts.select.end()) {
attrs.push_back(FormatToalExecTime(node, root));
attrs.push_back(FormatAcceleratorExecTime(node, root));
attrs.push_back(FormatCPUExecTime(node, root));
}
if (opts.select.find(kShown[9]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
attrs.push_back(FormatAcceleratorExecTime(node, root));
}
if (opts.select.find(kShown[10]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
attrs.push_back(FormatCPUExecTime(node, root));
}
if (opts.select.find(kShown[2]) != opts.select.end()) {
double accu_pct = 0.0;
double pct = 0.0;
if (node->proto().total_parameters() > 0) {
accu_pct = 100.0 * node->proto().total_parameters() /
root->proto().total_parameters();
pct =
100.0 * node->proto().parameters() / root->proto().total_parameters();
}
attrs.push_back(absl::StrFormat(
"%30s", absl::StrFormat("%s params (%.2f%%, %.2f%%)",
FormatNumber(node->proto().parameters()),
accu_pct, pct)));
}
if (opts.select.find(kShown[3]) != opts.select.end()) {
double accu_pct = 0.0;
double pct = 0.0;
if (node->proto().total_float_ops() > 0) {
accu_pct = 100.0 * node->proto().total_float_ops() /
root->proto().total_float_ops();
pct = 100.0 * node->proto().float_ops() / root->proto().total_float_ops();
}
attrs.push_back(absl::StrFormat(
"%30s", absl::StrFormat("%s float_ops (%.2f%%, %.2f%%)",
FormatNumber(node->proto().float_ops()),
accu_pct, pct)));
}
if (opts.select.find(kShown[5]) != opts.select.end()) {
attrs.push_back(absl::StrJoin(node->node->devices(), "|"));
}
if (opts.select.find(kShown[6]) != opts.select.end()) {
std::set<std::string> op_types = node->node->op_types();
attrs.push_back(absl::StrJoin(op_types, "|"));
}
if (opts.select.find(kShown[7]) != opts.select.end()) {
int64_t total_runs = 0;
for (const auto& gnode : node->proto().graph_nodes()) {
total_runs += gnode.run_count();
}
attrs.push_back(absl::StrFormat(
"%10s", absl::StrFormat("%d|%d", total_runs,
node->proto().graph_nodes_size())));
}
std::string node_str =
absl::StrFormat("%-25s%s\n", node->name(), absl::StrJoin(attrs, ", "));
if (opts.select.find(kShown[8]) != opts.select.end()) {
std::string input_shape_str = FormatInputShapes(node->proto());
if (!input_shape_str.empty()) {
node_str = absl::StrFormat("%s\n%s\n\n", node_str, input_shape_str);
}
}
return node_str;
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_op.h
+78
View File
@@ -0,0 +1,78 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Build a flat structure of ops.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_OP_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_OP_H_
#include <deque>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <vector>
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include "tensorflow/core/profiler/internal/tfprof_show_multi.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
// Organize tensorflow ops in a graph structure, pointing from output ops
// to input ops.
class TFOp : public TFMultiShow {
public:
explicit TFOp() : TFMultiShow() {}
~TFOp() override = default;
void AddNode(TFGraphNode* node) override;
void Build() override;
private:
const ShowMultiNode* ShowInternal(const Options& opts,
Timeline* timeline) override;
int64_t SearchRoot(std::vector<OpNode*> nodes,
const std::vector<std::string>& regexes);
bool ShouldShowIfExtra(const ShowMultiNode* node, const Options& opts,
int depth) const override {
const int max_num_graph_nodes = node->node->graph_nodes().size();
if (opts.min_occurrence > max_num_graph_nodes) {
return false;
}
return true;
}
std::string FormatNode(OpNode* node, OpNode* root, const Options& opts) const;
std::string FormatMemoryNode(int64_t node_total_bytes,
int64_t root_total_bytes,
int64_t node_bytes) const;
std::unique_ptr<OpNode> root_;
std::map<std::string, std::unique_ptr<OpNode>> cnodes_map_;
std::map<std::string, std::unique_ptr<TFMultiGraphNode>> tfcnodes_map_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_OP_H_
tensorflow/core/profiler/internal/tfprof_scope.cc
+218
View File
@@ -0,0 +1,218 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_scope.h"
#include <stdio.h>
#include <memory>
#include <utility>
#include <vector>
#include "absl/strings/str_format.h"
#include "tensorflow/core/platform/regexp.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tensorflow/core/profiler/internal/tfprof_tensor.h"
namespace tensorflow {
namespace tfprof {
ScopeNode* TFScope::CreateParentNode(const std::string& name) {
if (nodes_map_.find(name) != nodes_map_.end()) {
return nodes_map_[name].get();
}
node_defs_.push_back(std::make_unique<NodeDef>());
node_defs_.back()->set_name(name);
node_defs_.back()->set_op(kTFScopeParent);
parent_nodes_[name] =
std::make_unique<TFGraphNode>(node_defs_.back().get(), -1, nullptr);
nodes_map_[name] = std::make_unique<ScopeNode>(parent_nodes_[name].get());
return nodes_map_[name].get();
}
void TFScope::AddNode(TFGraphNode* node) {
std::string name = node->name();
if (nodes_map_.find(node->name()) == nodes_map_.end()) {
nodes_map_[name] = std::make_unique<ScopeNode>(node);
}
auto last_slash = name.find_last_of('/');
while (last_slash != name.npos) {
name = name.substr(0, last_slash);
if (nodes_map_.find(name) == nodes_map_.end()) {
CHECK(CreateParentNode(name));
}
last_slash = name.find_last_of('/');
}
}
void TFScope::Build() {
if (root_) return;
std::vector<ScopeNode*> roots;
// Found roots, which are nodes without "/".
for (auto it = nodes_map_.begin(); it != nodes_map_.end(); it++) {
ScopeNode* node = it->second.get();
auto last_slash = node->name().find_last_of('/');
if (last_slash == std::string::npos) {
roots.push_back(node);
} else {
const std::string prefix = node->name().substr(0, last_slash);
nodes_map_[prefix]->children.push_back(node);
}
}
root_ = CreateParentNode(kTFProfRoot);
root_->children.assign(roots.begin(), roots.end());
}
const ShowNode* TFScope::ShowInternal(const Options& opts, Timeline* timeline) {
root_->ResetTotalStats();
if (opts.output_type == kOutput[3]) {
absl::FPrintF(stderr, "Only 'code' view supports pprof output now.\n");
return root_;
}
std::vector<ScopeNode*> roots = Account(root_->children, opts);
root_->show_children.clear();
for (ScopeNode* n : roots) {
root_->AggregateTotalStats(n);
}
if (opts.start_name_regexes.size() != 1 ||
opts.start_name_regexes[0] != ".*") {
roots = SearchRoot(roots, opts.start_name_regexes);
}
root_->show_children.assign(roots.begin(), roots.end());
ScopeNode* root = PrintScope({root_}, opts, 1, 0)[0];
root->formatted_str = FormatLegend(opts) + root->formatted_str;
Format(root->show_children, &root->formatted_str, root->mutable_proto());
if (timeline) {
timeline->GenerateScopeTimeline(root);
}
return root;
}
void TFScope::Format(const std::vector<ScopeNode*> roots,
std::string* display_str, GraphNodeProto* proto) {
for (ScopeNode* node : roots) {
display_str->append(node->formatted_str);
GraphNodeProto* child = proto->add_children();
child->MergeFrom(node->proto());
Format(node->show_children, display_str, child);
}
}
std::vector<ScopeNode*> TFScope::SearchRoot(
std::vector<ScopeNode*> roots, const std::vector<std::string>& regexes) {
std::vector<ScopeNode*> res;
if (roots.empty()) {
return res;
}
for (ScopeNode* root : roots) {
bool match_start_node = false;
for (const std::string& regex : regexes) {
if (RE2::FullMatch(root->name(), regex)) {
res.push_back(root);
match_start_node = true;
break;
}
}
if (match_start_node) {
// Found a start node at this branch, no need to continue.
continue;
}
std::vector<ScopeNode*> nroots = SearchRoot(root->show_children, regexes);
res.insert(res.end(), nroots.begin(), nroots.end());
}
return res;
}
std::vector<ScopeNode*> TFScope::PrintScope(const std::vector<ScopeNode*> roots,
const Options& opts, int depth,
int last_ident) {
std::vector<ScopeNode*> show_nodes;
for (ScopeNode* node : roots) {
int ident = last_ident;
bool show = ShouldShow(node, opts, depth);
if (show) ident += 2;
std::vector<ScopeNode*> show_cnodes;
if (!ShouldTrim(node, opts.trim_name_regexes) && depth <= opts.max_depth) {
show_cnodes = PrintScope(node->show_children, opts, depth + 1, ident);
}
if (show) {
node->show_children.clear();
if (opts.account_displayed_op_only) {
node->ResetTotalStats();
node->AddSelfToTotalStats();
}
show_cnodes = SortNodes(show_cnodes, opts);
for (ScopeNode* sc : show_cnodes) {
node->show_children.push_back(sc);
if (opts.account_displayed_op_only) {
node->AggregateTotalStats(sc);
}
}
node->formatted_str = absl::StrFormat(
"%s%s\n", std::string(last_ident, ' '), FormatNode(node, opts));
if (opts.select.find(kShown[4]) != opts.select.end()) {
std::unique_ptr<TFProfTensor> tfprof_tensor;
if (LookUpCheckPoint(node->name(), &tfprof_tensor)) {
std::string value_str;
tfprof_tensor->Display(&value_str,
node->mutable_proto()->mutable_tensor_value());
node->formatted_str += value_str;
}
}
show_nodes.push_back(node);
} else {
show_nodes.insert(show_nodes.end(), show_cnodes.begin(),
show_cnodes.end());
}
}
return show_nodes;
}
std::vector<ScopeNode*> TFScope::Account(const std::vector<ScopeNode*>& roots,
const Options& opts) {
std::vector<ScopeNode*> act_nodes;
for (ScopeNode* node : roots) {
node->ResetTotalStats();
std::vector<ScopeNode*> act_cnodes = Account(node->children, opts);
node->account = ReAccount(node, opts);
if (node->account || !act_cnodes.empty()) {
node->show_children.clear();
node->ResetTotalStats();
node->AddSelfToTotalStats();
for (ScopeNode* c : act_cnodes) {
node->AggregateTotalStats(c);
node->show_children.push_back(c);
}
act_nodes.push_back(node);
}
}
return act_nodes;
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_scope.h
+76
View File
@@ -0,0 +1,76 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Build a tree structure based on the TensorFlow op names.
// For example, 'name1/name2' is a child of 'name1'.
// Stats are aggregated from descendants to ancestors.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_SCOPE_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_SCOPE_H_
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/c/checkpoint_reader.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include "tensorflow/core/profiler/internal/tfprof_show.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class TFScope : public TFShow {
public:
explicit TFScope(checkpoint::CheckpointReader* ckpt_reader)
: TFShow(ckpt_reader), root_(nullptr) {}
~TFScope() override = default;
void AddNode(TFGraphNode* node) override;
void Build() override;
private:
const ShowNode* ShowInternal(const Options& opts,
Timeline* timeline) override;
ScopeNode* CreateParentNode(const std::string& name);
std::vector<ScopeNode*> SearchRoot(std::vector<ScopeNode*> roots,
const std::vector<std::string>& regexes);
std::vector<ScopeNode*> PrintScope(std::vector<ScopeNode*> roots,
const Options& opts, int depth,
int last_ident);
std::vector<ScopeNode*> Account(const std::vector<ScopeNode*>& roots,
const Options& opts);
void Format(std::vector<ScopeNode*> roots, std::string* display_str,
GraphNodeProto* proto);
ScopeNode* root_;
std::vector<std::unique_ptr<NodeDef>> node_defs_;
std::map<std::string, std::unique_ptr<TFGraphNode>> parent_nodes_;
std::map<std::string, std::unique_ptr<ScopeNode>> nodes_map_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_SCOPE_H_
tensorflow/core/profiler/internal/tfprof_show.cc
+296
View File
@@ -0,0 +1,296 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_show.h"
#include <memory>
#include <set>
#include <utility>
#include <vector>
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/regexp.h"
namespace tensorflow {
namespace tfprof {
const GraphNodeProto& TFShow::Show(const std::string& prefix,
const Options& opts) {
if (opts.output_type == kOutput[0]) {
Timeline timeline(opts.step, opts.output_options.at(kTimelineOpts[0]));
return ShowInternal(opts, &timeline)->proto();
} else {
const ShowNode* ret = ShowInternal(opts, nullptr);
if (opts.output_type == kOutput[1]) {
absl::PrintF("%s", (prefix + ret->formatted_str));
fflush(stdout);
} else if (opts.output_type == kOutput[2]) {
absl::Status s = WriteStringToFile(Env::Default(),
opts.output_options.at(kFileOpts[0]),
prefix + ret->formatted_str);
if (!s.ok()) {
absl::FPrintF(stderr, "%s\n", s.ToString());
}
} else if (opts.output_type == kOutput[3] ||
opts.output_type == kOutput[4]) {
} else {
absl::FPrintF(stderr, "Unknown output type: %s\n", opts.output_type);
}
return ret->proto();
}
}
bool TFShow::LookUpCheckPoint(const std::string& name,
std::unique_ptr<TFProfTensor>* tensor) {
if (name == kTFProfRoot || !ckpt_reader_ || !tensor) {
return false;
}
std::unique_ptr<Tensor> out_tensor;
TF_Status* status = TF_NewStatus();
ckpt_reader_->GetTensor(name, &out_tensor, status);
if (TF_GetCode(status) != TF_OK) {
absl::FPrintF(stderr, "%s\n", TF_Message(status));
TF_DeleteStatus(status);
return false;
}
*tensor = std::make_unique<TFProfTensor>(std::move(out_tensor));
TF_DeleteStatus(status);
return true;
}
bool TFShow::ShouldShow(const ShowNode* node, const Options& opts,
int depth) const {
// Always show kTFProfRoot.
if (node->name() == kTFProfRoot) return true;
if (node->proto().total_requested_bytes() < opts.min_bytes ||
node->proto().total_peak_bytes() < opts.min_peak_bytes ||
node->proto().total_residual_bytes() < opts.min_residual_bytes ||
node->proto().total_output_bytes() < opts.min_output_bytes ||
node->proto().total_exec_micros() < opts.min_micros ||
node->proto().total_accelerator_exec_micros() <
opts.min_accelerator_micros ||
node->proto().total_cpu_exec_micros() < opts.min_cpu_micros ||
node->proto().parameters() < opts.min_params ||
node->proto().float_ops() < opts.min_float_ops ||
node->proto().run_count() < opts.min_occurrence ||
depth > opts.max_depth || !ShouldShowIfExtra(node, opts, depth)) {
return false;
}
bool show = false;
if (opts.show_name_regexes.size() == 1 && opts.show_name_regexes[0] == ".*") {
show = true;
} else {
for (const std::string& regex : opts.show_name_regexes) {
if (RE2::FullMatch(node->name(), regex)) {
show = true;
break;
}
}
}
// Don't show if show_name_regexes don't cover it.
if (!show) return false;
// Don't show if hide_name_regexes cover it.
for (const std::string& regex : opts.hide_name_regexes) {
if (RE2::FullMatch(node->name(), regex)) return false;
}
return true;
}
bool TFShow::ShouldTrim(const ShowNode* node,
const std::vector<std::string>& regexes) const {
for (const std::string& regex : regexes) {
if (RE2::FullMatch(node->name(), regex)) {
return true;
}
}
return false;
}
bool TFShow::ReAccount(ShowNode* node, const Options& opts) {
node->ReInit(opts.step);
if (opts.account_type_regexes.size() == 1 &&
opts.account_type_regexes[0] == ".*") {
return true;
}
for (const std::string& regex : opts.account_type_regexes) {
for (const std::string& type : node->node->op_types()) {
if (RE2::FullMatch(type, regex)) {
return true;
}
}
}
return false;
}
std::string TFShow::FormatNodeMemory(ShowNode* node, int64_t bytes,
int64_t total_bytes) const {
std::string memory = FormatMemory(total_bytes);
if (node->account) {
memory = FormatMemory(bytes) + "/" + memory;
} else {
memory = "--/" + memory;
}
return memory;
}
std::string TFShow::FormatNode(ShowNode* node, const Options& opts) const {
std::vector<std::string> info;
if (opts.select.find(kShown[2]) != opts.select.end()) {
const std::string shape = FormatShapes(node->node->shape());
if (!shape.empty()) {
info.push_back(shape);
}
std::string params =
FormatNumber(node->proto().total_parameters()) + " params";
if (node->account) {
params = FormatNumber(node->proto().parameters()) + "/" + params;
} else {
params = "--/" + params;
}
info.push_back(params);
}
if (opts.select.find(kShown[3]) != opts.select.end()) {
std::string fops = FormatNumber(node->proto().total_float_ops()) + " flops";
if (node->account) {
fops = FormatNumber(node->proto().float_ops()) + "/" + fops;
} else {
fops = "--/" + fops;
}
info.push_back(fops);
}
if (opts.select.find(kShown[0]) != opts.select.end()) {
info.push_back(FormatNodeMemory(node, node->proto().requested_bytes(),
node->proto().total_requested_bytes()));
}
if (opts.select.find(kShown[11]) != opts.select.end()) {
info.push_back(FormatNodeMemory(node, node->proto().peak_bytes(),
node->proto().total_peak_bytes()));
}
if (opts.select.find(kShown[12]) != opts.select.end()) {
info.push_back(FormatNodeMemory(node, node->proto().residual_bytes(),
node->proto().total_residual_bytes()));
}
if (opts.select.find(kShown[13]) != opts.select.end()) {
info.push_back(FormatNodeMemory(node, node->proto().output_bytes(),
node->proto().total_output_bytes()));
}
if (opts.select.find(kShown[1]) != opts.select.end()) {
info.push_back(FormatTotalExecTime(node, opts));
info.push_back(FormatAcceleratorExecTime(node, opts));
info.push_back(FormatCPUExecTime(node, opts));
}
if (opts.select.find(kShown[9]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
info.push_back(FormatAcceleratorExecTime(node, opts));
}
if (opts.select.find(kShown[10]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
info.push_back(FormatCPUExecTime(node, opts));
}
if (opts.select.find(kShown[5]) != opts.select.end()) {
if (node->proto().devices_size() > 0) {
info.push_back(absl::StrJoin(node->proto().devices(), "|"));
}
}
if (opts.select.find(kShown[6]) != opts.select.end()) {
const std::set<std::string>& op_types = node->node->op_types();
info.push_back(absl::StrJoin(op_types, "|"));
}
if (opts.select.find(kShown[7]) != opts.select.end()) {
std::string run = FormatNumber(node->proto().total_run_count());
if (node->account) {
run = FormatNumber(node->proto().run_count()) + "/" + run;
} else {
run = "--/" + run;
}
std::string definition =
FormatNumber(node->proto().total_definition_count());
if (node->account) {
definition = "1/" + definition;
} else {
definition = "--/" + definition;
}
info.push_back(run + "|" + definition);
}
if (opts.select.find(kShown[8]) != opts.select.end()) {
std::vector<std::string> shape_vec;
for (const auto& s : node->node->input_shapes()) {
if (s.second.empty()) {
shape_vec.push_back(absl::StrFormat("%d:unknown", s.first));
} else {
shape_vec.push_back(
absl::StrFormat("%d:%s", s.first, absl::StrJoin(s.second, "x")));
}
}
info.push_back(absl::StrJoin(shape_vec, "|"));
}
return absl::StrFormat("%s (%s)", node->name(), absl::StrJoin(info, ", "));
}
std::string TFShow::FormatLegend(const Options& opts) const {
std::vector<std::string> legends;
if (opts.select.find(kShown[2]) != opts.select.end()) {
legends.push_back("# parameters");
}
if (opts.select.find(kShown[3]) != opts.select.end()) {
legends.push_back("# float_ops");
}
if (opts.select.find(kShown[0]) != opts.select.end()) {
legends.push_back("requested bytes");
}
if (opts.select.find(kShown[11]) != opts.select.end()) {
legends.push_back("peak bytes");
}
if (opts.select.find(kShown[12]) != opts.select.end()) {
legends.push_back("residual bytes");
}
if (opts.select.find(kShown[13]) != opts.select.end()) {
legends.push_back("output bytes");
}
if (opts.select.find(kShown[1]) != opts.select.end()) {
legends.push_back("total execution time");
legends.push_back("accelerator execution time");
legends.push_back("cpu execution time");
}
if (opts.select.find(kShown[9]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
legends.push_back("accelerator execution time");
}
if (opts.select.find(kShown[10]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
legends.push_back("cpu execution time");
}
if (opts.select.find(kShown[5]) != opts.select.end()) {
legends.push_back("assigned devices");
}
if (opts.select.find(kShown[6]) != opts.select.end()) {
legends.push_back("op types");
}
if (opts.select.find(kShown[7]) != opts.select.end()) {
legends.push_back("op count (run|defined)");
}
if (opts.select.find(kShown[8]) != opts.select.end()) {
legends.push_back("input shapes");
}
return absl::StrFormat("node name | %s\n", absl::StrJoin(legends, " | "));
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_show.h
+157
View File
@@ -0,0 +1,157 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Parent class and utilities for tfprof_graph and tfprof_scope.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_SHOW_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_SHOW_H_
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/c/checkpoint_reader.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include "tensorflow/core/profiler/internal/tfprof_node_show.h"
#include "tensorflow/core/profiler/internal/tfprof_tensor.h"
#include "tensorflow/core/profiler/internal/tfprof_timeline.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class TFShow {
public:
explicit TFShow(checkpoint::CheckpointReader* ckpt_reader)
: ckpt_reader_(ckpt_reader) {}
virtual ~TFShow() = default;
virtual void AddNode(TFGraphNode* node) = 0;
virtual void Build() = 0;
virtual const GraphNodeProto& Show(const std::string& prefix,
const Options& opts) final;
protected:
virtual const ShowNode* ShowInternal(const Options& opts,
Timeline* timeline) = 0;
bool LookUpCheckPoint(const std::string& name,
std::unique_ptr<TFProfTensor>* tensor);
// Overridden by subclass if extra requirements need to be met.
virtual bool ShouldShowIfExtra(const ShowNode* node, const Options& opts,
int depth) const {
return true;
}
bool ShouldShow(const ShowNode* node, const Options& opts, int depth) const;
bool ShouldTrim(const ShowNode* node,
const std::vector<std::string>& regexes) const;
bool ReAccount(ShowNode* node, const Options& opts);
std::string FormatNode(ShowNode* node, const Options& opts) const;
std::string FormatNodeMemory(ShowNode* node, int64_t bytes,
int64_t total_bytes) const;
std::string FormatLegend(const Options& opts) const;
template <typename T>
std::vector<T*> SortNodes(const std::vector<T*>& nodes, const Options& opts) {
if (opts.order_by.empty() || nodes.empty()) {
return nodes;
}
std::vector<T*> sorted_nodes = nodes;
std::stable_sort(sorted_nodes.begin(), sorted_nodes.end(),
[&opts](const T* n1, const T* n2) {
if (n1->name() == kTFProfRoot) return true;
if (n2->name() == kTFProfRoot) return false;
bool name_cmp = n1->name() < n2->name();
if (opts.order_by == kOrderBy[0]) {
return name_cmp;
} else if (opts.order_by == kOrderBy[1]) {
return n1->proto().total_requested_bytes() >
n2->proto().total_requested_bytes();
} else if (opts.order_by == kOrderBy[2]) {
return n1->proto().total_peak_bytes() >
n2->proto().total_peak_bytes();
} else if (opts.order_by == kOrderBy[3]) {
return n1->proto().total_residual_bytes() >
n2->proto().total_residual_bytes();
} else if (opts.order_by == kOrderBy[4]) {
return n1->proto().total_output_bytes() >
n2->proto().total_output_bytes();
} else if (opts.order_by == kOrderBy[5]) {
return n1->proto().total_exec_micros() >
n2->proto().total_exec_micros();
} else if (opts.order_by == kOrderBy[6]) {
return n1->proto().total_accelerator_exec_micros() >
n2->proto().total_accelerator_exec_micros();
} else if (opts.order_by == kOrderBy[7]) {
return n1->proto().total_cpu_exec_micros() >
n2->proto().total_cpu_exec_micros();
} else if (opts.order_by == kOrderBy[8]) {
return n1->proto().total_parameters() >
n2->proto().total_parameters();
} else if (opts.order_by == kOrderBy[9]) {
return n1->proto().total_float_ops() >
n2->proto().total_float_ops();
}
return name_cmp;
});
return sorted_nodes;
}
checkpoint::CheckpointReader* ckpt_reader_;
};
template <typename T>
std::string FormatTotalExecTime(const T* node, const Options& opts) {
std::string time = FormatTime(node->proto().total_exec_micros());
if (node->account) {
time = FormatTime(node->proto().exec_micros()) + "/" + time;
} else {
time = "--/" + time;
}
return time;
}
template <typename T>
std::string FormatCPUExecTime(const T* node, const Options& opts) {
std::string time = FormatTime(node->proto().total_cpu_exec_micros());
if (node->account) {
time = FormatTime(node->proto().cpu_exec_micros()) + "/" + time;
} else {
time = "--/" + time;
}
return time;
}
template <typename T>
std::string FormatAcceleratorExecTime(const T* node, const Options& opts) {
std::string time = FormatTime(node->proto().total_accelerator_exec_micros());
if (node->account) {
time = FormatTime(node->proto().accelerator_exec_micros()) + "/" + time;
} else {
time = "--/" + time;
}
return time;
}
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_SHOW_H_
tensorflow/core/profiler/internal/tfprof_show_multi.cc
+249
View File
@@ -0,0 +1,249 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_show_multi.h"
#include <algorithm>
#include <map>
#include <memory>
#include <set>
#include <tuple>
#include <utility>
#include <vector>
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/regexp.h"
#include "tensorflow/core/profiler/internal/tfprof_scope.h"
namespace tensorflow {
namespace tfprof {
const MultiGraphNodeProto& TFMultiShow::Show(const std::string& prefix,
const Options& opts) {
if (opts.output_type == kOutput[0]) {
Timeline timeline(opts.step, opts.output_options.at(kTimelineOpts[0]));
return ShowInternal(opts, &timeline)->proto();
} else {
const ShowMultiNode* ret = ShowInternal(opts, nullptr);
if (opts.output_type == kOutput[1]) {
absl::PrintF("%s%s", prefix, ret->formatted_str);
fflush(stdout);
} else if (opts.output_type == kOutput[2]) {
absl::Status s = WriteStringToFile(Env::Default(),
opts.output_options.at(kFileOpts[0]),
prefix + ret->formatted_str);
if (!s.ok()) {
absl::FPrintF(stderr, "%s\n", s.ToString());
}
} else if (opts.output_type == kOutput[3] ||
opts.output_type == kOutput[4]) {
} else {
absl::FPrintF(stderr, "Unknown output type: %s\n", opts.output_type);
}
return ret->proto();
}
}
bool TFMultiShow::ShouldShow(const ShowMultiNode* node, const Options& opts,
int depth) const {
// Always show kTFProfRoot.
if (node->name() == kTFProfRoot) return true;
// TODO(xpan): Think more carefully about node filtering in code view.
// Unlike graph/scope view, which users want to see the exact leaf op.
// In code view, users want to see the middle code traces they wrote.
//
// This is a subtle difference from scope/graph view. Usually mostly
// want to see the middle code traces (i.e. their own codes.), instead
// of the TensorFlow internal codes traces.
if (node->proto().total_requested_bytes() < opts.min_bytes ||
node->proto().total_peak_bytes() < opts.min_peak_bytes ||
node->proto().total_residual_bytes() < opts.min_residual_bytes ||
node->proto().total_output_bytes() < opts.min_output_bytes ||
node->proto().total_exec_micros() < opts.min_micros ||
node->proto().total_accelerator_exec_micros() <
opts.min_accelerator_micros ||
node->proto().total_cpu_exec_micros() < opts.min_cpu_micros ||
node->proto().total_parameters() < opts.min_params ||
node->proto().total_float_ops() < opts.min_float_ops ||
depth > opts.max_depth || !ShouldShowIfExtra(node, opts, depth)) {
return false;
}
bool show = false;
if (opts.show_name_regexes.size() == 1 && opts.show_name_regexes[0] == ".*") {
show = true;
} else {
for (const std::string& regex : opts.show_name_regexes) {
if (RE2::FullMatch(node->name(), regex)) {
show = true;
break;
}
}
}
// Don't show if show_name_regexes don't cover it.
if (!show) return false;
// Don't show if hide_name_regexes cover it.
for (const std::string& regex : opts.hide_name_regexes) {
if (RE2::FullMatch(node->name(), regex)) return false;
}
return true;
}
bool TFMultiShow::ShouldTrim(const ShowMultiNode* node,
const std::vector<std::string>& regexes) const {
for (const std::string& regex : regexes) {
if (RE2::FullMatch(node->name(), regex)) {
return true;
}
}
return false;
}
bool TFMultiShow::ReAccount(ShowMultiNode* node, const Options& opts) {
return node->ReInit(opts.step, opts.account_type_regexes);
}
std::string TFMultiShow::FormatLegend(const Options& opts) const {
std::vector<std::string> legends;
if (opts.select.find(kShown[0]) != opts.select.end()) {
legends.push_back("requested bytes");
}
if (opts.select.find(kShown[11]) != opts.select.end()) {
legends.push_back("peak bytes");
}
if (opts.select.find(kShown[12]) != opts.select.end()) {
legends.push_back("residual bytes");
}
if (opts.select.find(kShown[13]) != opts.select.end()) {
legends.push_back("output bytes");
}
if (opts.select.find(kShown[1]) != opts.select.end()) {
legends.push_back("total execution time");
legends.push_back("accelerator execution time");
legends.push_back("cpu execution time");
}
if (opts.select.find(kShown[9]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
legends.push_back("accelerator execution time");
}
if (opts.select.find(kShown[10]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
legends.push_back("cpu execution time");
}
if (opts.select.find(kShown[2]) != opts.select.end()) {
legends.push_back("# parameters");
}
if (opts.select.find(kShown[3]) != opts.select.end()) {
legends.push_back("# float_ops");
}
if (opts.select.find(kShown[5]) != opts.select.end()) {
legends.push_back("assigned devices");
}
if (opts.select.find(kShown[6]) != opts.select.end()) {
legends.push_back("op types");
}
if (opts.select.find(kShown[7]) != opts.select.end()) {
legends.push_back("op occurrence (run|defined)");
}
if (opts.select.find(kShown[8]) != opts.select.end()) {
legends.push_back("input shapes");
}
return absl::StrFormat("node name | %s\n", absl::StrJoin(legends, " | "));
}
std::string TFMultiShow::FormatInputShapes(
const MultiGraphNodeProto& proto) const {
// input_shape string -> (static defined count, run count, run_micros)
std::map<std::string, std::tuple<int64_t, int64_t, int64_t>>
input_shapes_attr;
for (int i = 0; i < proto.graph_nodes_size(); ++i) {
const GraphNodeProto& gnode = proto.graph_nodes(i);
// Convert and sort by input_idx.
std::map<int, std::vector<int64_t>> input_shapes;
for (const auto& inp : gnode.input_shapes()) {
input_shapes[inp.first] = ShapeProtoToVec(inp.second);
}
std::vector<std::string> input_vec;
for (const auto& s : input_shapes) {
if (s.second.empty()) {
input_vec.push_back(absl::StrFormat("%d:unknown", s.first));
} else {
input_vec.push_back(
absl::StrFormat("%d:%s", s.first, absl::StrJoin(s.second, "x")));
}
}
std::string shape_type_str =
absl::StrFormat("input_type: %s", absl::StrJoin(input_vec, ",\t"));
auto t = input_shapes_attr.find(shape_type_str);
if (t == input_shapes_attr.end()) {
input_shapes_attr.insert(
std::make_pair(shape_type_str, std::make_tuple(0, 0, 0)));
t = input_shapes_attr.find(shape_type_str);
}
input_shapes_attr[shape_type_str] = std::make_tuple(
std::get<0>(t->second) + 1, std::get<1>(t->second) + gnode.run_count(),
std::get<2>(t->second) + gnode.exec_micros());
}
if (input_shapes_attr.empty()) {
return "";
}
std::vector<std::pair<std::string, std::tuple<int64_t, int64_t, int64_t>>>
shape_count_vec(input_shapes_attr.begin(), input_shapes_attr.end());
std::stable_sort(
shape_count_vec.begin(), shape_count_vec.end(),
[](const std::pair<const std::string,
std::tuple<int64_t, int64_t, int64_t>>& a,
const std::pair<const std::string,
std::tuple<int64_t, int64_t, int64_t>>& b) {
return std::get<1>(a.second) > std::get<1>(b.second);
});
std::vector<std::string> input_types;
input_types.reserve(shape_count_vec.size());
for (const auto& s : shape_count_vec) {
std::tuple<int64_t, int64_t, int64_t> t = s.second;
input_types.push_back(absl::StrFormat(
"%s\t(run*%d|defined*%d)\texec_time: %s", s.first, std::get<1>(t),
std::get<0>(t), FormatTime(std::get<2>(t))));
}
return absl::StrJoin(input_types, "\n");
}
std::vector<std::string> TFMultiShow::FormatTimes(const ShowMultiNode* node,
const Options& opts) const {
std::vector<std::string> attrs;
if (opts.select.find(kShown[1]) != opts.select.end()) {
attrs.push_back(FormatTotalExecTime(node, opts));
attrs.push_back(FormatAcceleratorExecTime(node, opts));
attrs.push_back(FormatCPUExecTime(node, opts));
}
if (opts.select.find(kShown[9]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
attrs.push_back(FormatAcceleratorExecTime(node, opts));
}
if (opts.select.find(kShown[10]) != opts.select.end() &&
opts.select.find(kShown[1]) == opts.select.end()) {
attrs.push_back(FormatCPUExecTime(node, opts));
}
return attrs;
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_show_multi.h
+128
View File
@@ -0,0 +1,128 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Parent class and utilities for tfprof_code.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_SHOW_MULTI_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_SHOW_MULTI_H_
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include "tensorflow/core/profiler/internal/tfprof_node_show.h"
#include "tensorflow/core/profiler/internal/tfprof_show.h"
#include "tensorflow/core/profiler/internal/tfprof_tensor.h"
#include "tensorflow/core/profiler/internal/tfprof_timeline.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class TFMultiShow {
public:
explicit TFMultiShow() = default;
virtual ~TFMultiShow() = default;
virtual void AddNode(TFGraphNode* node) = 0;
virtual void Build() = 0;
const MultiGraphNodeProto& Show(const std::string& prefix,
const Options& opts);
protected:
virtual const ShowMultiNode* ShowInternal(const Options& opts,
Timeline* timeline) = 0;
bool LookUpCheckPoint(const std::string& name,
std::unique_ptr<TFProfTensor>* tensor);
// Overridden by subclass if extra requirements need to be met.
virtual bool ShouldShowIfExtra(const ShowMultiNode* node, const Options& opts,
int depth) const {
return true;
}
bool ShouldShow(const ShowMultiNode* node, const Options& opts,
int depth) const;
bool ShouldTrim(const ShowMultiNode* node,
const std::vector<std::string>& regexes) const;
bool ReAccount(ShowMultiNode* node, const Options& opts);
std::string FormatLegend(const Options& opts) const;
std::string FormatInputShapes(const MultiGraphNodeProto& proto) const;
std::vector<std::string> FormatTimes(const ShowMultiNode* node,
const Options& opts) const;
template <typename T>
std::vector<T*> SortNodes(const std::vector<T*>& nodes, const Options& opts) {
if (opts.order_by.empty() || nodes.empty()) {
return nodes;
}
std::vector<T*> sorted_nodes = nodes;
std::stable_sort(sorted_nodes.begin(), sorted_nodes.end(),
[&opts](const T* n1, const T* n2) {
if (n1->name() == kTFProfRoot) return true;
if (n2->name() == kTFProfRoot) return false;
bool name_cmp = n1->name() < n2->name();
if (opts.order_by == kOrderBy[0]) {
return name_cmp;
} else if (opts.order_by == kOrderBy[1]) {
return n1->proto().total_requested_bytes() >
n2->proto().total_requested_bytes();
} else if (opts.order_by == kOrderBy[2]) {
return n1->proto().total_peak_bytes() >
n2->proto().total_peak_bytes();
} else if (opts.order_by == kOrderBy[3]) {
return n1->proto().total_residual_bytes() >
n2->proto().total_residual_bytes();
} else if (opts.order_by == kOrderBy[4]) {
return n1->proto().total_output_bytes() >
n2->proto().total_output_bytes();
} else if (opts.order_by == kOrderBy[5]) {
return n1->proto().total_exec_micros() >
n2->proto().total_exec_micros();
} else if (opts.order_by == kOrderBy[6]) {
return n1->proto().total_accelerator_exec_micros() >
n2->proto().total_accelerator_exec_micros();
} else if (opts.order_by == kOrderBy[7]) {
return n1->proto().total_cpu_exec_micros() >
n2->proto().total_cpu_exec_micros();
} else if (opts.order_by == kOrderBy[8]) {
return n1->proto().total_parameters() >
n2->proto().total_parameters();
} else if (opts.order_by == kOrderBy[9]) {
return n1->proto().total_float_ops() >
n2->proto().total_float_ops();
} else if (opts.order_by == kOrderBy[10]) {
return n1->node->graph_nodes().size() >
n2->node->graph_nodes().size();
}
return name_cmp;
});
return sorted_nodes;
}
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_SHOW_MULTI_H_
tensorflow/core/profiler/internal/tfprof_show_test.cc
+217
View File
@@ -0,0 +1,217 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <memory>
#include <utility>
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tensorflow/core/profiler/internal/tfprof_stats.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_log.pb.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
std::string CheckAndRemoveDoc(const std::string& doc) {
auto pos = doc.find("Profile:");
CHECK(pos != doc.npos);
return doc.substr(pos + 9);
}
class TFProfShowTest : public ::testing::Test {
protected:
TFProfShowTest() {
std::string graph_path =
io::JoinPath(testing::TensorFlowSrcRoot(),
"core/profiler/internal/testdata/graph.pbtxt");
std::unique_ptr<tensorflow::GraphDef> graph_pb =
std::make_unique<tensorflow::GraphDef>();
TF_CHECK_OK(
ReadProtoFile(Env::Default(), graph_path, graph_pb.get(), false));
std::unique_ptr<tensorflow::RunMetadata> run_meta_pb =
std::make_unique<tensorflow::RunMetadata>();
std::string run_meta_path =
io::JoinPath(testing::TensorFlowSrcRoot(),
"core/profiler/internal/testdata/run_meta");
TF_CHECK_OK(
ReadProtoFile(Env::Default(), run_meta_path, run_meta_pb.get(), true));
std::unique_ptr<OpLogProto> op_log_pb = std::make_unique<OpLogProto>();
std::string op_log_path =
io::JoinPath(testing::TensorFlowSrcRoot(),
"core/profiler/internal/testdata/tfprof_log");
TF_CHECK_OK(ReadBinaryProto(Env::Default(), op_log_path, op_log_pb.get()));
std::string ckpt_path = io::JoinPath(
testing::TensorFlowSrcRoot(), "core/profiler/internal/testdata/ckpt");
TF_Status* status = TF_NewStatus();
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader =
std::make_unique<checkpoint::CheckpointReader>(ckpt_path, status);
CHECK(TF_GetCode(status) == TF_OK);
TF_DeleteStatus(status);
tf_stats_ =
std::make_unique<TFStats>(std::move(graph_pb), std::move(run_meta_pb),
std::move(op_log_pb), std::move(ckpt_reader));
tf_stats_->BuildAllViews();
}
std::string TestToFromProto(const std::string& cmd, const Options& opts,
bool show_multi_node = false) {
std::string profile_file = io::JoinPath(testing::TmpDir(), "profile");
tf_stats_->WriteProfile(profile_file);
TFStats new_stats(profile_file, nullptr);
new_stats.BuildAllViews();
if (show_multi_node) {
new_stats.ShowMultiGraphNode(cmd, opts);
} else {
new_stats.ShowGraphNode(cmd, opts);
}
std::string dump_str;
TF_CHECK_OK(ReadFileToString(Env::Default(),
opts.output_options.at("outfile"), &dump_str));
return dump_str;
}
std::unique_ptr<TFStats> tf_stats_;
};
TEST_F(TFProfShowTest, DumpScopeMode) {
std::string dump_file = io::JoinPath(testing::TmpDir(), "dump");
Options opts(
5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, "name",
{"VariableV2"}, // accout_type_regexes
{".*"}, {""}, {".*"}, {""}, false,
{"params", "bytes", "peak_bytes", "residual_bytes", "output_bytes",
"micros", "accelerator_micros", "cpu_micros", "float_ops"},
"file", {{"outfile", dump_file}});
tf_stats_->ShowGraphNode("scope", opts);
std::string dump_str;
TF_CHECK_OK(ReadFileToString(Env::Default(), dump_file, &dump_str));
EXPECT_EQ(
"node name | # parameters | # float_ops | requested bytes | peak bytes | "
"residual bytes | output bytes | total execution time | accelerator "
"execution time | cpu execution time\n_TFProfRoot (--/451 params, --/0 "
"flops, --/2.56KB, --/2.56KB, --/2.56KB, --/2.56KB, --/13us, --/0us, "
"--/13us)\n DW (3x3x3x6, 162/162 params, 0/0 flops, 1.28KB/1.28KB, "
"1.28KB/1.28KB, 1.28KB/1.28KB, 1.28KB/1.28KB, 2us/2us, 0us/0us, "
"2us/2us)\n DW2 (2x2x6x12, 288/288 params, 0/0 flops, 1.28KB/1.28KB, "
"1.28KB/1.28KB, 1.28KB/1.28KB, 1.28KB/1.28KB, 11us/11us, 0us/0us, "
"11us/11us)\n ScalarW (1, 1/1 params, 0/0 flops, 0B/0B, 0B/0B, 0B/0B, "
"0B/0B, 0us/0us, 0us/0us, 0us/0us)\n",
CheckAndRemoveDoc(dump_str));
EXPECT_EQ(dump_str, TestToFromProto("scope", opts));
}
TEST_F(TFProfShowTest, DumpAcceleratorAndCPUMicros) {
std::string dump_file = io::JoinPath(testing::TmpDir(), "dump");
Options opts(5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, "cpu_micros",
{".*"}, // accout_type_regexes
{".*"}, {""}, {".*"}, {""}, false,
{"accelerator_micros", "cpu_micros"}, "file",
{{"outfile", dump_file}});
tf_stats_->ShowGraphNode("scope", opts);
std::string dump_str;
TF_CHECK_OK(ReadFileToString(Env::Default(), dump_file, &dump_str));
EXPECT_EQ(
"node name | accelerator execution time | cpu execution "
"time\n_TFProfRoot (--/404us, --/4.54ms)\n Conv2D (226us/226us, "
"4.07ms/4.07ms)\n Conv2D_1 (178us/178us, 419us/419us)\n "
"_retval_Conv2D_1_0_0 (0us/0us, 41us/41us)\n DW2 (0us/0us, 11us/11us)\n "
" DW2/Assign (0us/0us, 0us/0us)\n DW2/Initializer (0us/0us, "
"0us/0us)\n DW2/Initializer/random_normal (0us/0us, 0us/0us)\n "
" DW2/Initializer/random_normal/RandomStandardNormal (0us/0us, "
"0us/0us)\n DW2/Initializer/random_normal/mean (0us/0us, "
"0us/0us)\n DW2/Initializer/random_normal/mul (0us/0us, "
"0us/0us)\n DW2/Initializer/random_normal/shape (0us/0us, "
"0us/0us)\n DW2/Initializer/random_normal/stddev (0us/0us, "
"0us/0us)\n DW2/read (0us/0us, 0us/0us)\n DW (0us/0us, 2us/2us)\n "
"DW/Assign (0us/0us, 0us/0us)\n DW/Initializer (0us/0us, 0us/0us)\n "
" DW/Initializer/random_normal (0us/0us, 0us/0us)\n "
"DW/Initializer/random_normal/RandomStandardNormal (0us/0us, 0us/0us)\n "
" DW/Initializer/random_normal/mean (0us/0us, 0us/0us)\n "
"DW/Initializer/random_normal/mul (0us/0us, 0us/0us)\n "
"DW/Initializer/random_normal/shape (0us/0us, 0us/0us)\n "
"DW/Initializer/random_normal/stddev (0us/0us, 0us/0us)\n DW/read "
"(0us/0us, 0us/0us)\n zeros (0us/0us, 2us/2us)\n ScalarW (0us/0us, "
"0us/0us)\n ScalarW/Assign (0us/0us, 0us/0us)\n "
"ScalarW/Initializer (0us/0us, 0us/0us)\n "
"ScalarW/Initializer/random_normal (0us/0us, 0us/0us)\n "
"ScalarW/Initializer/random_normal/RandomStandardNormal (0us/0us, "
"0us/0us)\n ScalarW/Initializer/random_normal/mean (0us/0us, "
"0us/0us)\n ScalarW/Initializer/random_normal/mul (0us/0us, "
"0us/0us)\n ScalarW/Initializer/random_normal/shape (0us/0us, "
"0us/0us)\n ScalarW/Initializer/random_normal/stddev (0us/0us, "
"0us/0us)\n ScalarW/read (0us/0us, 0us/0us)\n init (0us/0us, "
"0us/0us)\n",
CheckAndRemoveDoc(dump_str));
EXPECT_EQ(dump_str, TestToFromProto("scope", opts));
}
TEST_F(TFProfShowTest, DumpOpMode) {
std::string dump_file = io::JoinPath(testing::TmpDir(), "dump");
Options opts(
5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, "params",
{".*"}, // accout_type_regexes
{".*"}, {""}, {".*"}, {""}, false,
{"params", "bytes", "micros", "float_ops", "occurrence", "input_shapes"},
"file", {{"outfile", dump_file}});
tf_stats_->ShowMultiGraphNode("op", opts);
std::string dump_str;
TF_CHECK_OK(ReadFileToString(Env::Default(), dump_file, &dump_str));
EXPECT_EQ(
"nodename|requestedbytes|totalexecutiontime|acceleratorexecutiontime|"
"cpuexecutiontime|#parameters|#float_ops|opoccurrence(run|defined)|"
"inputshapes\nVariableV22.56KB(100.00%,8.40%),13us(100.00%,0.26%),0us("
"100.00%,0.00%),13us(100.00%,0.29%),451params(100.00%,100.00%),0float_"
"ops(100.00%,0.00%),2|3\n\ninput_type:\t(run*2|defined*3)\texec_time:"
"13us\n\nAdd0B(0.00%,0.00%),0us(99.74%,0.00%),0us(100.00%,0.00%),0us(99."
"71%,0.00%),0params(0.00%,0.00%),0float_ops(100.00%,0.00%),0|3\n\ninput_"
"type:0:1,\t1:1\t(run*0|defined*1)\texec_time:0us\ninput_type:0:2x2x6x12,"
"\t1:1\t(run*0|defined*1)\texec_time:0us\ninput_type:0:3x3x3x6,\t1:1\t("
"run*0|defined*1)\texec_time:0us\n\nAssign0B(0.00%,0.00%),0us(99.74%,0."
"00%),0us(100.00%,0.00%),0us(99.71%,0.00%),0params(0.00%,0.00%),0float_"
"ops(100.00%,0.00%),0|3\n\ninput_type:0:1,\t1:1\t(run*0|defined*1)\texec_"
"time:0us\ninput_type:0:2x2x6x12,\t1:2x2x6x12\t(run*0|defined*1)\texec_"
"time:0us\ninput_type:0:3x3x3x6,\t1:3x3x3x6\t(run*0|defined*1)\texec_"
"time:0us\n\nConst0B(0.00%,0.00%),2us(99.74%,0.04%),0us(100.00%,0.00%),"
"2us(99.71%,0.04%),0params(0.00%,0.00%),0float_ops(100.00%,0.00%),1|"
"10\n\ninput_type:\t(run*1|defined*10)\texec_time:2us\n\nConv2D27.90KB("
"91.60%,91.60%),4.89ms(99.70%,98.87%),404us(100.00%,100.00%),4.49ms(99."
"67%,98.77%),0params(0.00%,0.00%),10.44kfloat_ops(100.00%,100.00%),2|"
"2\n\ninput_type:0:2x3x3x6,\t1:2x2x6x12\t(run*1|defined*1)\texec_time:"
"597us\ninput_type:0:2x6x6x3,\t1:3x3x3x6\t(run*1|defined*1)\texec_time:4."
"29ms\n\nIdentity0B(0.00%,0.00%),0us(0.83%,0.00%),0us(0.00%,0.00%),0us(0."
"90%,0.00%),0params(0.00%,0.00%),0float_ops(0.00%,0.00%),0|3\n\ninput_"
"type:0:1\t(run*0|defined*1)\texec_time:0us\ninput_type:0:2x2x6x12\t(run*"
"0|defined*1)\texec_time:0us\ninput_type:0:3x3x3x6\t(run*0|defined*1)"
"\texec_time:0us\n\n",
StringReplace(CheckAndRemoveDoc(dump_str), " ", ""));
EXPECT_EQ(dump_str, TestToFromProto("op", opts, true));
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_stats.cc
+379
View File
@@ -0,0 +1,379 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_stats.h"
#include <stdio.h>
#include <map>
#include <memory>
#include <utility>
#include <vector>
#include "absl/strings/numbers.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "tensorflow/core/platform/regexp.h"
#include "tensorflow/core/profiler/internal/tfprof_timeline.h"
namespace tensorflow {
namespace tfprof {
namespace {
const char* const kProfilePrefix = "Profile:\n";
bool CreateRunMetadataNode(const std::string& name, NodeDef* def) {
// TODO(xpan): Better solution than denylisting this 2 nodes. They
// actually cost some resources, maybe include them. Some nodes, such
// as _SOURCE appear in multiple devices, which breaks tfprof's assumption.
if (name == "RecvTensor" || name == "_SOURCE" ||
name.find("MEMCPY") != name.npos) {
return false;
}
def->set_name(name);
// TODO(xpan): Better operation type.
// This is because some times a node doesn't have a op type,
// so we use node name as the op type.
def->set_op(name);
return true;
}
} // namespace
TFStats::TFStats(std::unique_ptr<GraphDef> graph,
std::unique_ptr<RunMetadata> run_meta,
std::unique_ptr<OpLogProto> op_log,
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader)
: has_code_traces_(false),
miss_accelerator_stream_(false),
ckpt_reader_(std::move(ckpt_reader)) {
CHECK(graph) << "Must at least have GraphDef";
AddGraph(std::move(graph));
if (run_meta && run_meta->has_step_stats()) {
AddRunMeta(0, std::move(run_meta));
}
AddOpLogProto(std::move(op_log));
if (ckpt_reader_) {
for (const auto& v : ckpt_reader_->GetVariableToShapeMap()) {
auto node = nodes_map_.find(v.first);
if (node != nodes_map_.end()) {
node->second->AddOpType("_checkpoint_variables");
}
}
}
}
TFStats::TFStats(const std::string& filename,
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader)
: has_code_traces_(false),
miss_accelerator_stream_(false),
ckpt_reader_(std::move(ckpt_reader)) {
std::string str;
absl::Status s = ReadFileToString(Env::Default(), filename, &str);
if (!s.ok()) {
absl::FPrintF(stderr, "Failed to read profile: %s", s.ToString());
return;
}
ProfileProto profile;
if (!profile.ParseFromString(str)) {
absl::FPrintF(stderr, "Failed to parse profile\n");
return;
}
for (const auto& entry : profile.id_to_string()) {
id_to_string_[entry.first] = entry.second;
}
for (const auto& node_pb : profile.nodes()) {
std::unique_ptr<TFGraphNode> node = std::make_unique<TFGraphNode>(
node_pb.second, profile, &id_to_string_, &nodes_map_);
nodes_map_.insert(std::pair<std::string, std::unique_ptr<TFGraphNode>>(
node_pb.second.name(), std::move(node)));
}
has_code_traces_ = profile.has_trace();
for (int64_t s : profile.steps()) {
steps_.insert(s);
}
}
void TFStats::BuildView(const std::string& cmd) {
if (cmd == kCmds[0] && !scope_view_) {
scope_view_ = std::make_unique<TFScope>(ckpt_reader_.get());
for (auto it = nodes_map_.begin(); it != nodes_map_.end(); it++) {
scope_view_->AddNode(it->second.get());
}
scope_view_->Build();
}
if (cmd == kCmds[1] && !graph_view_) {
graph_view_ = std::make_unique<TFGraph>(ckpt_reader_.get());
for (auto it = nodes_map_.begin(); it != nodes_map_.end(); it++) {
graph_view_->AddNode(it->second.get());
}
graph_view_->Build();
}
if (cmd == kCmds[2] && !code_view_) {
code_view_ = std::make_unique<TFCode>();
for (auto it = nodes_map_.begin(); it != nodes_map_.end(); it++) {
code_view_->AddNode(it->second.get());
}
code_view_->Build();
}
if (cmd == kCmds[3] && !op_view_) {
op_view_ = std::make_unique<TFOp>();
for (auto it = nodes_map_.begin(); it != nodes_map_.end(); it++) {
op_view_->AddNode(it->second.get());
}
op_view_->Build();
}
}
void TFStats::BuildAllViews() {
std::vector<std::string> cmds_str(kCmds,
kCmds + sizeof(kCmds) / sizeof(*kCmds));
for (const std::string& cmd : cmds_str) {
BuildView(cmd);
}
}
const GraphNodeProto& TFStats::ShowGraphNode(const std::string& cmd,
const Options& opts) const {
if (!Validate(opts)) {
return empty_graph_node_;
}
std::string prefix = MaybeReportMissingTrace();
prefix += QueryDoc(cmd, opts) + kProfilePrefix;
if (cmd == kCmds[0]) {
return scope_view_->Show(prefix, opts);
} else if (cmd == kCmds[1]) {
if (opts.step < 0 && opts.output_type == kOutput[0]) {
for (int64_t step : steps_) {
Options nopts = opts;
nopts.step = step;
graph_view_->Show(prefix, nopts);
}
return empty_graph_node_;
}
return graph_view_->Show(prefix, opts);
} else {
absl::FPrintF(stderr, "Unknown command: %s\n", cmd);
return empty_graph_node_;
}
}
const MultiGraphNodeProto& TFStats::ShowMultiGraphNode(
const std::string& cmd, const Options& opts) const {
if (!Validate(opts)) {
return empty_multi_graph_node_;
}
std::string prefix = MaybeReportMissingTrace();
prefix += QueryDoc(cmd, opts) + kProfilePrefix;
if (cmd == kCmds[2]) {
if (!has_code_traces()) {
absl::FPrintF(stderr, "No code trace information\n");
return empty_multi_graph_node_;
}
return code_view_->Show(prefix, opts);
} else if (cmd == kCmds[3]) {
return op_view_->Show(prefix, opts);
} else {
absl::FPrintF(stderr, "Unknown command: %s\n", cmd);
return empty_multi_graph_node_;
}
}
void TFStats::AddGraph(std::unique_ptr<GraphDef> graph) {
std::map<std::string, const NodeDef*> node_defs;
bool node_added = false;
for (const NodeDef& node : graph->node()) {
if (nodes_map_.find(node.name()) != nodes_map_.end()) {
continue;
}
node_added = true;
size_t num_nodes = nodes_map_.size();
nodes_map_[node.name()] =
std::make_unique<TFGraphNode>(&node, num_nodes, &nodes_map_);
node_defs[node.name()] = &node;
}
for (auto it = node_defs.begin(); it != node_defs.end(); it++) {
TFGraphNode* node = nodes_map_.at(it->first).get();
for (int i = 0; i < it->second->input_size(); ++i) {
std::string node_input = it->second->input(i);
int output_idx = 0;
// input name format can be: "^node:src_output"
// if not :src_output, then it's the first one (further verify?)
auto prefix_pos = node_input.find(':');
if (prefix_pos != node_input.npos) {
std::vector<std::string> input_parts = absl::StrSplit(node_input, ':');
DCHECK(input_parts.size() == 2)
<< "Unknown NodeDef.input format: " << node_input;
node_input = input_parts[0];
DCHECK(absl::SimpleAtoi(input_parts[1], &output_idx))
<< "Failed to parse integer: " << output_idx;
}
if (node_input.substr(0, 1) == "^") {
node_input = node_input.substr(1);
}
// Delay input TFGraphNode retrieval as late as possible.
// In long run, when we have TensorFlow runtime graph, the
// graph connection should be dynamic and per-step.
node->AddInput(node_input, output_idx, i);
}
}
if (node_added) {
graph_view_.reset(nullptr);
scope_view_.reset(nullptr);
op_view_.reset(nullptr);
code_view_.reset(nullptr);
}
}
void TFStats::AddOpLogProto(std::unique_ptr<OpLogProto> op_log) {
if (!op_log) {
return;
}
for (const auto& entry : op_log->id_to_string()) {
if (id_to_string_.find(entry.first) == id_to_string_.end()) {
id_to_string_[entry.first] = entry.second;
}
}
for (const OpLogEntry& entry : op_log->log_entries()) {
auto node = nodes_map_.find(entry.name());
if (node == nodes_map_.end()) continue;
for (const std::string& type : entry.types()) {
node->second->AddOpType(type);
}
if (entry.float_ops()) {
node->second->AddFloatOps(entry.float_ops());
}
if (entry.has_code_def()) {
has_code_traces_ = true;
node->second->AddCode(entry.code_def(), &id_to_string_);
}
}
}
void TFStats::AddRunMeta(int64_t step, std::unique_ptr<RunMetadata> run_meta) {
if (!run_meta || !run_meta->has_step_stats()) {
absl::FPrintF(stderr, "Invalid RunMetadata for step %d\n", step);
return;
}
if (steps_.find(step) == steps_.end()) {
steps_.insert(step);
}
steps_.insert(step);
bool has_gpu_scheduling = false;
bool has_gpu_stream = false;
for (const auto& dev_stat : run_meta->step_stats().dev_stats()) {
std::string dev = absl::AsciiStrToLower(dev_stat.device());
if (IsPlacedOnAccelerator(dev)) {
has_gpu_scheduling = true;
if (CountAsAcceleratorTime(dev)) {
has_gpu_stream = true;
}
}
for (const NodeExecStats& node_stat : dev_stat.node_stats()) {
std::string name = node_stat.node_name();
// Sometimes the node_name is suffixed with unnecessary information.
auto split_pos = node_stat.node_name().find(':');
if (split_pos != node_stat.node_name().npos) {
name = node_stat.node_name().substr(0, split_pos);
}
auto node = nodes_map_.find(name);
if (node == nodes_map_.end()) {
NodeDef def;
if (CreateRunMetadataNode(name, &def)) {
size_t num_nodes = nodes_map_.size();
nodes_map_[name] =
std::make_unique<TFGraphNode>(&def, num_nodes, &nodes_map_);
nodes_map_.at(name)->AddStepStat(step, dev_stat.device(), node_stat);
}
} else {
covered_nodes_.insert(node->second->id());
node->second->AddStepStat(step, dev_stat.device(), node_stat);
}
}
}
if (has_gpu_scheduling && !has_gpu_stream) {
miss_accelerator_stream_ = true;
}
}
std::string TFStats::MaybeReportMissingTrace() const {
std::string report = "";
if (miss_accelerator_stream_) {
report +=
"\n\nFound accelerator operation but misses accelerator "
"stream stats!\n\n"
"It's likely a gpu tracing issue rather than tf-profiler issue.\n"
"If you found your operation missing accelerator time, "
"consider to post to discuss@tensorflow.org!\n\n";
}
return report;
}
void TFStats::SerializeToString(std::string* content) {
ProfileProto profile;
for (const auto& entry : id_to_string_) {
(*profile.mutable_id_to_string())[entry.first] = entry.second;
}
for (auto it = nodes_map_.begin(); it != nodes_map_.end(); it++) {
if (it->second->id() < 0) {
continue;
}
(*profile.mutable_nodes())[it->second->id()].MergeFrom(
it->second->ToProto(nodes_map_));
}
profile.set_has_trace(has_code_traces_);
profile.set_miss_accelerator_stream(miss_accelerator_stream_);
for (int64_t s : steps_) {
profile.add_steps(s);
}
*content = profile.SerializeAsString();
}
void TFStats::WriteProfile(const std::string& filename) {
std::string content;
SerializeToString(&content);
absl::Status s = WriteStringToFile(Env::Default(), filename, content);
if (!s.ok()) {
absl::FPrintF(stderr, "%s\n", s.ToString());
}
}
bool TFStats::Validate(const Options& opts) const {
if (opts.step >= 0 && steps_.find(opts.step) == steps_.end()) {
absl::FPrintF(stderr,
"Options -step=%d not found.\nAvailable steps: ", opts.step);
for (int64_t s : steps_) {
absl::FPrintF(stderr, "%d ", s);
}
absl::FPrintF(stderr, "\n");
return false;
}
return true;
}
void TFStats::AddNodeForTest(int64_t step, std::unique_ptr<TFGraphNode> node) {
steps_.insert(step);
nodes_map_[node->name()] = std::move(node);
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_stats.h
+127
View File
@@ -0,0 +1,127 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Core API of tfprof.
// 1. Load protos generated from a tensorflow model.
// 2. Build in-memory representations of the tensorflow model, annotate the
// representation with various stats, such as params,times,memory,etc.
// 3. Accept command and options to selectively aggregate stats for analysis
// and print out the results.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_STATS_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_STATS_H_
#include <map>
#include <memory>
#include <set>
#include <string>
#include "tensorflow/c/checkpoint_reader.h"
#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/profiler/internal/tfprof_code.h"
#include "tensorflow/core/profiler/internal/tfprof_graph.h"
#include "tensorflow/core/profiler/internal/tfprof_node.h"
#include "tensorflow/core/profiler/internal/tfprof_op.h"
#include "tensorflow/core/profiler/internal/tfprof_scope.h"
#include "tensorflow/core/profiler/internal/tfprof_show.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_log.pb.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace tensorflow {
namespace tfprof {
class TFStats {
public:
TFStats(std::unique_ptr<GraphDef> graph,
std::unique_ptr<RunMetadata> run_meta,
std::unique_ptr<OpLogProto> op_log,
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader);
TFStats(const std::string& filename,
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader);
~TFStats() = default;
const std::map<std::string, std::unique_ptr<TFGraphNode>>& nodes() const {
return nodes_map_;
}
const std::set<int64_t>& steps() const { return steps_; }
bool has_code_traces() const { return has_code_traces_; }
double run_coverage() const {
return covered_nodes_.size() / (nodes_map_.size() + 1e-10);
}
void BuildView(const std::string& cmd);
void BuildAllViews();
// Note: Must first BuildView(view_foo) before ShowXXX(view_foo) methods.
//
// Organize the TensorFlow model as different types of views, and generate
// outputs for profiling.
// TODO(xpan): Should it return reference here?
const GraphNodeProto& ShowGraphNode(const std::string& cmd,
const Options& opts) const;
const MultiGraphNodeProto& ShowMultiGraphNode(const std::string& cmd,
const Options& opts) const;
// Add a (partial) graph to existing graph.
void AddGraph(std::unique_ptr<GraphDef> graph);
// Add a step of run time meta data.
void AddRunMeta(int64_t step, std::unique_ptr<RunMetadata> run_meta);
// Add tfprof operation meta data, such as customized op type, float_ops,
// and code traces.
void AddOpLogProto(std::unique_ptr<OpLogProto> op_log);
void SerializeToString(std::string* content);
void WriteProfile(const std::string& filename);
// For test purpose only.
void AddNodeForTest(int64_t step, std::unique_ptr<TFGraphNode> node);
private:
bool Validate(const Options& opts) const;
std::string MaybeReportMissingTrace() const;
std::set<int64_t> steps_;
bool has_code_traces_;
bool miss_accelerator_stream_;
std::unique_ptr<TFScope> scope_view_;
std::unique_ptr<TFGraph> graph_view_;
std::unique_ptr<TFCode> code_view_;
std::unique_ptr<TFOp> op_view_;
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader_;
// TODO(xpan): Store TFGraphNode instead of TFGraphNode* to avoid large
// number of dynamic alloc.
// Maps from graph node name to TFGraphNode.
std::map<std::string, std::unique_ptr<TFGraphNode>> nodes_map_;
GraphNodeProto empty_graph_node_;
MultiGraphNodeProto empty_multi_graph_node_;
std::map<int64_t, std::string> id_to_string_;
// Graph nodes covered by RunMetadata, that is traced with run time stats.
std::set<int64_t> covered_nodes_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_STATS_H_
tensorflow/core/profiler/internal/tfprof_stats_test.cc
+378
View File
@@ -0,0 +1,378 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_stats.h"
#include <memory>
#include <utility>
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_log.pb.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class TFProfStatsTest : public ::testing::Test {
protected:
TFProfStatsTest() {
std::string graph_path =
io::JoinPath(testing::TensorFlowSrcRoot(),
"core/profiler/internal/testdata/graph.pbtxt");
std::unique_ptr<tensorflow::GraphDef> graph_pb =
std::make_unique<tensorflow::GraphDef>();
TF_CHECK_OK(
ReadProtoFile(Env::Default(), graph_path, graph_pb.get(), false));
std::unique_ptr<tensorflow::RunMetadata> run_meta_pb =
std::make_unique<tensorflow::RunMetadata>();
std::string run_meta_path =
io::JoinPath(testing::TensorFlowSrcRoot(),
"core/profiler/internal/testdata/run_meta");
TF_CHECK_OK(
ReadProtoFile(Env::Default(), run_meta_path, run_meta_pb.get(), true));
std::unique_ptr<OpLogProto> op_log_pb = std::make_unique<OpLogProto>();
std::string op_log_path =
io::JoinPath(testing::TensorFlowSrcRoot(),
"core/profiler/internal/testdata/tfprof_log");
TF_CHECK_OK(ReadBinaryProto(Env::Default(), op_log_path, op_log_pb.get()));
std::string ckpt_path = io::JoinPath(
testing::TensorFlowSrcRoot(), "core/profiler/internal/testdata/ckpt");
TF_Status* status = TF_NewStatus();
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader =
std::make_unique<checkpoint::CheckpointReader>(ckpt_path, status);
CHECK(TF_GetCode(status) == TF_OK);
TF_DeleteStatus(status);
tf_stats_ =
std::make_unique<TFStats>(std::move(graph_pb), std::move(run_meta_pb),
std::move(op_log_pb), std::move(ckpt_reader));
tf_stats_->BuildAllViews();
}
std::string TestToFromProto(const std::string& cmd, const Options& opts) {
std::string profile_file = io::JoinPath(testing::TmpDir(), "profile");
tf_stats_->WriteProfile(profile_file);
TFStats new_stats(profile_file, nullptr);
new_stats.BuildAllViews();
return new_stats.ShowGraphNode(cmd, opts).DebugString();
}
std::unique_ptr<TFStats> tf_stats_;
};
TEST_F(TFProfStatsTest, CustomOpType) {
Options opts(3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, "name",
{kTrainableVarType}, // accout_type_regexes
{".*"}, {""}, {".*"}, {""}, false,
{"params", "bytes", "micros", "float_ops"}, "", {});
const GraphNodeProto& root = tf_stats_->ShowGraphNode("scope", opts);
GraphNodeProto expected;
CHECK(protobuf::TextFormat::ParseFromString(
"name: \"_TFProfRoot\"\ntotal_exec_micros: 13\ntotal_requested_bytes: "
"2560\ntotal_parameters: 451\nchildren {\n name: \"DW\"\n exec_micros: "
"2\n requested_bytes: 1280\n parameters: 162\n total_exec_micros: 2\n "
" total_requested_bytes: 1280\n total_parameters: 162\n devices: "
"\"/job:localhost/replica:0/task:0/gpu:0\"\n cpu_exec_micros: 2\n "
"total_cpu_exec_micros: 2\n run_count: 1\n total_run_count: 1\n "
"total_definition_count: 1\n peak_bytes: 1280\n residual_bytes: 1280\n "
" output_bytes: 1280\n total_peak_bytes: 1280\n total_residual_bytes: "
"1280\n total_output_bytes: 1280\n}\nchildren {\n name: \"DW2\"\n "
"exec_micros: 11\n requested_bytes: 1280\n parameters: 288\n "
"total_exec_micros: 11\n total_requested_bytes: 1280\n "
"total_parameters: 288\n devices: "
"\"/job:localhost/replica:0/task:0/gpu:0\"\n cpu_exec_micros: 11\n "
"total_cpu_exec_micros: 11\n run_count: 1\n total_run_count: 1\n "
"total_definition_count: 1\n peak_bytes: 1280\n residual_bytes: 1280\n "
" output_bytes: 1280\n total_peak_bytes: 1280\n total_residual_bytes: "
"1280\n total_output_bytes: 1280\n}\nchildren {\n name: \"ScalarW\"\n "
"parameters: 1\n total_parameters: 1\n total_definition_count: "
"1\n}\ntotal_cpu_exec_micros: 13\ntotal_run_count: "
"2\ntotal_definition_count: 3\ntotal_peak_bytes: "
"2560\ntotal_residual_bytes: 2560\ntotal_output_bytes: 2560\n",
&expected));
EXPECT_EQ(expected.DebugString(), root.DebugString());
EXPECT_EQ(root.DebugString(), TestToFromProto("scope", opts));
}
TEST_F(TFProfStatsTest, CheckPointOpType) {
Options opts(3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, "name",
{kCkptVarType}, // accout_type_regexes
{".*"}, {""}, {".*"}, {""}, false,
{"params", "bytes", "micros", "float_ops"}, "", {});
const GraphNodeProto& root = tf_stats_->ShowGraphNode("scope", opts);
GraphNodeProto expected;
CHECK(protobuf::TextFormat::ParseFromString(
"name: \"_TFProfRoot\"\ntotal_exec_micros: 13\ntotal_requested_bytes: "
"2560\ntotal_parameters: 451\nchildren {\n name: \"DW\"\n exec_micros: "
"2\n requested_bytes: 1280\n parameters: 162\n total_exec_micros: 2\n "
" total_requested_bytes: 1280\n total_parameters: 162\n devices: "
"\"/job:localhost/replica:0/task:0/gpu:0\"\n cpu_exec_micros: 2\n "
"total_cpu_exec_micros: 2\n run_count: 1\n total_run_count: 1\n "
"total_definition_count: 1\n peak_bytes: 1280\n residual_bytes: 1280\n "
" output_bytes: 1280\n total_peak_bytes: 1280\n total_residual_bytes: "
"1280\n total_output_bytes: 1280\n}\nchildren {\n name: \"DW2\"\n "
"exec_micros: 11\n requested_bytes: 1280\n parameters: 288\n "
"total_exec_micros: 11\n total_requested_bytes: 1280\n "
"total_parameters: 288\n devices: "
"\"/job:localhost/replica:0/task:0/gpu:0\"\n cpu_exec_micros: 11\n "
"total_cpu_exec_micros: 11\n run_count: 1\n total_run_count: 1\n "
"total_definition_count: 1\n peak_bytes: 1280\n residual_bytes: 1280\n "
" output_bytes: 1280\n total_peak_bytes: 1280\n total_residual_bytes: "
"1280\n total_output_bytes: 1280\n}\nchildren {\n name: \"ScalarW\"\n "
"parameters: 1\n total_parameters: 1\n total_definition_count: "
"1\n}\ntotal_cpu_exec_micros: 13\ntotal_run_count: "
"2\ntotal_definition_count: 3\ntotal_peak_bytes: "
"2560\ntotal_residual_bytes: 2560\ntotal_output_bytes: 2560\n",
&expected));
EXPECT_EQ(expected.DebugString(), root.DebugString());
EXPECT_EQ(root.DebugString(), TestToFromProto("scope", opts));
}
TEST_F(TFProfStatsTest, TestGraph) {
Options opts(100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, "name", {".*"},
{"DW/Initializer/random_normal/mul"}, // start_name_regexes
{""}, {".*"}, {""}, false,
{"params", "bytes", "micros", "float_ops"}, "", {});
const GraphNodeProto& root = tf_stats_->ShowGraphNode("graph", opts);
GraphNodeProto expected;
CHECK(protobuf::TextFormat::ParseFromString(
"name: \"_TFProfRoot\"\ntotal_exec_micros: 4945\ntotal_requested_bytes: "
"30464\ntotal_parameters: 451\nchildren {\n name: "
"\"DW/Initializer/random_normal/mul\"\n children {\n name: "
"\"DW/Initializer/random_normal/RandomStandardNormal\"\n children {\n "
" name: \"DW/Initializer/random_normal/shape\"\n "
"total_definition_count: 1\n }\n input_shapes {\n key: 0\n "
" value {\n dim {\n size: 4\n }\n }\n "
"}\n total_definition_count: 2\n }\n children {\n name: "
"\"DW/Initializer/random_normal/stddev\"\n total_definition_count: "
"1\n }\n input_shapes {\n key: 0\n value {\n dim {\n "
"size: 3\n }\n dim {\n size: 3\n }\n dim {\n "
" size: 3\n }\n dim {\n size: 6\n }\n }\n "
"}\n input_shapes {\n key: 1\n value {\n dim {\n "
"size: 1\n }\n }\n }\n total_definition_count: "
"4\n}\ntotal_float_ops: 10440\ntotal_accelerator_exec_micros: "
"404\ntotal_cpu_exec_micros: 4541\ntotal_run_count: "
"6\ntotal_definition_count: 32\ntotal_peak_bytes: "
"25856\ntotal_residual_bytes: 3840\ntotal_output_bytes: 4864\n",
&expected));
EXPECT_EQ(expected.DebugString(), root.DebugString());
EXPECT_EQ(root.DebugString(), TestToFromProto("graph", opts));
}
TEST_F(TFProfStatsTest, TestFloatOps) {
Options opts(10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, -1, "name", {".*"}, {".*"},
{""}, {".*"}, {""}, false, {"float_ops"}, "", {});
const GraphNodeProto& root = tf_stats_->ShowGraphNode("scope", opts);
GraphNodeProto expected;
CHECK(protobuf::TextFormat::ParseFromString(
"name: \"_TFProfRoot\"\ntotal_exec_micros: 4945\ntotal_requested_bytes: "
"30464\ntotal_parameters: 451\nchildren {\n name: \"Conv2D\"\n "
"exec_micros: 4292\n requested_bytes: 18176\n total_exec_micros: "
"4292\n total_requested_bytes: 18176\n devices: "
"\"/job:localhost/replica:0/task:0/gpu:0\"\n float_ops: 5832\n "
"total_float_ops: 5832\n input_shapes {\n key: 0\n value {\n "
"dim {\n size: 2\n }\n dim {\n size: 6\n "
"}\n dim {\n size: 6\n }\n dim {\n size: "
"3\n }\n }\n }\n input_shapes {\n key: 1\n value {\n "
" dim {\n size: 3\n }\n dim {\n size: 3\n "
"}\n dim {\n size: 3\n }\n dim {\n size: "
"6\n }\n }\n }\n accelerator_exec_micros: 226\n "
"cpu_exec_micros: 4066\n total_accelerator_exec_micros: 226\n "
"total_cpu_exec_micros: 4066\n run_count: 1\n total_run_count: 1\n "
"total_definition_count: 1\n peak_bytes: 14592\n residual_bytes: 768\n "
" output_bytes: 768\n total_peak_bytes: 14592\n total_residual_bytes: "
"768\n total_output_bytes: 768\n}\nchildren {\n name: \"Conv2D_1\"\n "
"exec_micros: 597\n requested_bytes: 9728\n total_exec_micros: 597\n "
"total_requested_bytes: 9728\n devices: "
"\"/job:localhost/replica:0/task:0/gpu:0\"\n float_ops: 4608\n "
"total_float_ops: 4608\n input_shapes {\n key: 0\n value {\n "
"dim {\n size: 2\n }\n dim {\n size: 3\n "
"}\n dim {\n size: 3\n }\n dim {\n size: "
"6\n }\n }\n }\n input_shapes {\n key: 1\n value {\n "
" dim {\n size: 2\n }\n dim {\n size: 2\n "
"}\n dim {\n size: 6\n }\n dim {\n size: "
"12\n }\n }\n }\n accelerator_exec_micros: 178\n "
"cpu_exec_micros: 419\n total_accelerator_exec_micros: 178\n "
"total_cpu_exec_micros: 419\n run_count: 1\n total_run_count: 1\n "
"total_definition_count: 1\n peak_bytes: 8704\n residual_bytes: 512\n "
"output_bytes: 512\n total_peak_bytes: 8704\n total_residual_bytes: "
"512\n total_output_bytes: 512\n}\ntotal_float_ops: "
"10440\ntotal_accelerator_exec_micros: 404\ntotal_cpu_exec_micros: "
"4541\ntotal_run_count: 6\ntotal_definition_count: 35\ntotal_peak_bytes: "
"25856\ntotal_residual_bytes: 3840\ntotal_output_bytes: 4864\n",
&expected));
EXPECT_EQ(expected.DebugString(), root.DebugString());
EXPECT_EQ(root.DebugString(), TestToFromProto("scope", opts));
}
TEST_F(TFProfStatsTest, TestAccountShownNameOnly) {
Options opts(100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, "name", {".*"}, {".*"},
{""}, {"Conv2D_1"}, // show_name_regexes.
{""}, true, // account_displayed_op_only.
{"params"}, "", {});
const GraphNodeProto& root = tf_stats_->ShowGraphNode("scope", opts);
GraphNodeProto expected;
CHECK(protobuf::TextFormat::ParseFromString(
"name: \"_TFProfRoot\"\ntotal_exec_micros: 597\ntotal_requested_bytes: "
"9728\nchildren {\n name: \"Conv2D_1\"\n exec_micros: 597\n "
"requested_bytes: 9728\n total_exec_micros: 597\n "
"total_requested_bytes: 9728\n devices: "
"\"/job:localhost/replica:0/task:0/gpu:0\"\n float_ops: 4608\n "
"total_float_ops: 4608\n input_shapes {\n key: 0\n value {\n "
"dim {\n size: 2\n }\n dim {\n size: 3\n "
"}\n dim {\n size: 3\n }\n dim {\n size: "
"6\n }\n }\n }\n input_shapes {\n key: 1\n value {\n "
" dim {\n size: 2\n }\n dim {\n size: 2\n "
"}\n dim {\n size: 6\n }\n dim {\n size: "
"12\n }\n }\n }\n accelerator_exec_micros: 178\n "
"cpu_exec_micros: 419\n total_accelerator_exec_micros: 178\n "
"total_cpu_exec_micros: 419\n run_count: 1\n total_run_count: 1\n "
"total_definition_count: 1\n peak_bytes: 8704\n residual_bytes: 512\n "
"output_bytes: 512\n total_peak_bytes: 8704\n total_residual_bytes: "
"512\n total_output_bytes: 512\n}\ntotal_float_ops: "
"4608\ntotal_accelerator_exec_micros: 178\ntotal_cpu_exec_micros: "
"419\ntotal_run_count: 1\ntotal_definition_count: 2\ntotal_peak_bytes: "
"8704\ntotal_residual_bytes: 512\ntotal_output_bytes: 512\n",
&expected));
EXPECT_EQ(expected.DebugString(), root.DebugString());
EXPECT_EQ(root.DebugString(), TestToFromProto("scope", opts));
}
TEST_F(TFProfStatsTest, TestShowTensorValue) {
Options opts(10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, "name", {".*"}, {".*"},
{""}, {"DW"}, {""}, false,
{"tensor_value"}, // Show tensor value from checkpoint.
"", {});
const GraphNodeProto& root = tf_stats_->ShowGraphNode("scope", opts);
GraphNodeProto expected;
CHECK(protobuf::TextFormat::ParseFromString(
"name: \"_TFProfRoot\"\ntotal_exec_micros: 4945\ntotal_requested_bytes: "
"30464\ntotal_parameters: 451\nchildren {\n name: \"DW\"\n "
"exec_micros: 2\n requested_bytes: 1280\n parameters: 162\n "
"total_exec_micros: 2\n total_requested_bytes: 1280\n "
"total_parameters: 162\n devices: "
"\"/job:localhost/replica:0/task:0/gpu:0\"\n tensor_value {\n dtype: "
"DT_FLOAT\n value_double: -0.000534315\n value_double: "
"-0.00089602\n value_double: -0.000417239\n value_double: "
"0.00041444\n value_double: 0.000780691\n value_double: "
"-0.000559057\n value_double: -0.000234623\n value_double: "
"0.00013393\n value_double: -0.00187574\n value_double: "
"0.000785666\n value_double: 0.000673294\n value_double: "
"0.000653368\n value_double: 0.000924489\n value_double: "
"-0.000318373\n value_double: -0.000385202\n value_double: "
"-7.92661e-05\n value_double: 2.70287e-05\n value_double: "
"0.00152302\n value_double: 8.04435e-05\n value_double: "
"-0.00058102\n value_double: 0.000244291\n value_double: "
"-0.000438045\n value_double: -0.000110199\n value_double: "
"0.000731663\n value_double: -0.0012326\n value_double: "
"0.00064065\n value_double: -0.00135203\n value_double: "
"-6.42784e-05\n value_double: -0.0011857\n value_double: "
"-0.000487383\n value_double: 3.41493e-05\n value_double: "
"-0.00158447\n value_double: 0.00168448\n value_double: "
"0.00160946\n value_double: -0.000600483\n value_double: "
"0.000650259\n value_double: -0.00109938\n value_double: "
"-0.000842166\n value_double: -0.0022673\n value_double: "
"-0.00101941\n value_double: -0.0011169\n value_double: "
"-0.0013557\n value_double: -1.46354e-05\n value_double: "
"-1.05487e-05\n value_double: -0.00092014\n value_double: "
"0.00272874\n value_double: 5.13942e-05\n value_double: "
"-0.00223472\n value_double: -0.000250875\n value_double: "
"-0.00180747\n value_double: -0.00234714\n value_double: "
"-0.00113523\n value_double: -0.00112635\n value_double: "
"-0.000843118\n value_double: -6.84256e-05\n value_double: "
"0.000243336\n value_double: 0.00119151\n value_double: "
"0.00131022\n value_double: 0.000768038\n value_double: "
"-8.90095e-05\n value_double: -0.000626427\n value_double: "
"-7.0617e-05\n value_double: -0.0021988\n value_double: "
"-0.00221544\n value_double: -0.000393118\n value_double: "
"0.000159464\n value_double: -0.000874746\n value_double: "
"-0.00131239\n value_double: -0.00135747\n value_double: "
"-0.00179753\n value_double: -0.00101005\n value_double: "
"-0.000107518\n value_double: -0.000616882\n value_double: "
"-0.000360923\n value_double: -0.00026896\n value_double: "
"-0.000142548\n value_double: 0.000577227\n value_double: "
"0.000536027\n value_double: 0.00126907\n value_double: "
"-0.00122712\n value_double: -3.60499e-05\n value_double: "
"0.000151026\n value_double: 0.00107658\n value_double: "
"0.00116475\n value_double: -0.00145312\n value_double: "
"0.000233326\n value_double: -0.00020198\n value_double: "
"0.00179029\n value_double: 0.00150048\n value_double: "
"-0.000884775\n value_double: 0.000409188\n value_double: "
"2.97176e-05\n value_double: -0.000506118\n value_double: "
"-2.33992e-05\n value_double: -0.00037212\n value_double: "
"0.000862773\n value_double: 0.00174046\n value_double: "
"-0.000240207\n value_double: 0.000663976\n value_double: "
"-0.00134747\n value_double: 0.00115585\n value_double: "
"0.000555869\n value_double: 0.00176722\n value_double: "
"-0.000518409\n value_double: 0.00101051\n value_double: "
"0.000129399\n value_double: -0.000916389\n value_double: "
"-0.00137693\n value_double: -0.00152412\n value_double: "
"7.32515e-05\n value_double: -0.000190811\n value_double: "
"-0.000158692\n value_double: -5.7791e-05\n value_double: "
"0.000671785\n value_double: -0.00152924\n value_double: "
"0.00117314\n value_double: -0.000384202\n value_double: "
"0.00176709\n value_double: -0.000181703\n value_double: "
"-0.000460994\n value_double: 0.000643716\n value_double: "
"4.76719e-05\n value_double: -0.00101037\n value_double: "
"0.00159621\n value_double: 0.00186758\n value_double: "
"0.00100001\n value_double: -0.00121831\n value_double: "
"0.00132231\n value_double: 0.0013511\n value_double: 0.00106659\n "
" value_double: 0.00018091\n value_double: 0.00155925\n "
"value_double: 4.26087e-05\n value_double: 0.000243264\n "
"value_double: -0.0017202\n value_double: -0.000218897\n "
"value_double: 0.00118693\n value_double: 0.00258909\n "
"value_double: 0.000641913\n value_double: -0.0013211\n "
"value_double: -0.00171943\n value_double: 0.00089151\n "
"value_double: -0.00114969\n value_double: -0.000196331\n "
"value_double: 0.00109994\n value_double: 0.000302616\n "
"value_double: 0.000675812\n value_double: 0.00112222\n "
"value_double: 0.000516456\n value_double: 0.00133357\n "
"value_double: 0.000298491\n value_double: 0.00145934\n "
"value_double: -0.00159102\n value_double: -0.000819061\n "
"value_double: 0.000120583\n value_double: 0.0006108\n "
"value_double: 0.00124132\n value_double: 0.000764859\n "
"value_double: 0.000374641\n value_double: -0.00149603\n "
"value_double: -0.000317367\n value_double: -0.000417829\n }\n "
"cpu_exec_micros: 2\n total_cpu_exec_micros: 2\n run_count: 1\n "
"total_run_count: 1\n total_definition_count: 10\n peak_bytes: 1280\n "
"residual_bytes: 1280\n output_bytes: 1280\n total_peak_bytes: 1280\n "
"total_residual_bytes: 1280\n total_output_bytes: "
"1280\n}\ntotal_float_ops: 10440\ntotal_accelerator_exec_micros: "
"404\ntotal_cpu_exec_micros: 4541\ntotal_run_count: "
"6\ntotal_definition_count: 35\ntotal_peak_bytes: "
"25856\ntotal_residual_bytes: 3840\ntotal_output_bytes: 4864\n",
&expected));
EXPECT_EQ(expected.DebugString(), root.DebugString());
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_tensor.cc
+83
View File
@@ -0,0 +1,83 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_tensor.h"
#include <vector>
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
namespace tensorflow {
namespace tfprof {
void TFProfTensor::Display(std::string* formatted_str,
TFProfTensorProto* tfprof_tensor_pb) {
if (formatted_str) {
if (formatted_str_.length() >= kTFProfTenosrMaxDisplayLen) {
*formatted_str =
absl::StrCat(formatted_str_, "...omitted from display\n\n");
} else {
*formatted_str = formatted_str_;
}
}
if (tfprof_tensor_pb) {
tfprof_tensor_pb->MergeFrom(tfprof_tensor_pb_);
}
}
void TFProfTensor::Build() {
tfprof_tensor_pb_.set_dtype(tensor_->dtype());
switch (tensor_->dtype()) {
// Double for all floats.
case DataType::DT_FLOAT:
case DataType::DT_DOUBLE: {
std::vector<double> values_vec;
if (tensor_->dtype() == DataType::DT_FLOAT) {
GetValueVec<float, double>(&values_vec);
} else if (tensor_->dtype() == DataType::DT_DOUBLE) {
GetValueVec<double, double>(&values_vec);
}
BuildOutput<double>(0, 0, values_vec, &tfprof_tensor_pb_);
break;
}
// Int64 for all integers.
case DataType::DT_INT32:
case DataType::DT_INT64: {
std::vector<int64_t> values_vec;
if (tensor_->dtype() == DataType::DT_INT32) {
GetValueVec<int32_t, int64_t>(&values_vec);
} else if (tensor_->dtype() == DataType::DT_INT64) {
GetValueVec<int64_t, int64_t>(&values_vec);
}
BuildOutput<int64_t>(0, 0, values_vec, &tfprof_tensor_pb_);
break;
}
case DataType::DT_STRING: {
// Not supported by TensorFlow.
std::vector<tstring> values_vec;
GetValueVec<tstring, tstring>(&values_vec);
BuildOutput<tstring>(0, 0, values_vec, &tfprof_tensor_pb_);
break;
}
default: {
absl::FPrintF(stderr, "Not Supported type %d\n", tensor_->dtype());
break;
}
}
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_tensor.h
+175
View File
@@ -0,0 +1,175 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// TFProf representation of a Tensor's value.
// 1. Multi-dimension tensor is flattened in row major, and stored in proto.
// 2. integer are up-casted to int64. floats are up-casted to double. string
// is not supported by TensorFlow CheckPointReader library, though it is
// supported in current code.
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_TENSOR_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_TENSOR_H_
#include <memory>
#include <sstream>
#include <typeinfo>
#include <utility>
#include "absl/strings/numbers.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class TFProfTensor {
public:
explicit TFProfTensor(std::unique_ptr<Tensor> tensor)
: tensor_(std::move(tensor)) {
Build();
}
// If pointers are provided, they are filled by the method.
void Display(std::string* formatted_str, TFProfTensorProto* tfprof_tensor_pb);
private:
// Max length of tensor value displayed to CLI.
const int64_t kTFProfTenosrMaxDisplayLen = 10000;
// Max length after which a latency warning will be printed.
const int64_t kTFProfTensorMaxWarnLen = 100000;
void Build();
template <typename T>
bool AddValue(const T& value, TFProfTensorProto* dim) {
std::ostringstream sstream;
sstream << value;
if (typeid(value) == typeid(double)) {
double double_val = 0.0;
CHECK(absl::SimpleAtod(sstream.str(), &double_val)); // Crash OK
dim->add_value_double(double_val);
absl::StrAppendFormat(&formatted_str_, "%.2f ",
dim->value_double(dim->value_double_size() - 1));
} else if (typeid(value) == typeid(int64_t)) {
int64_t int64_val = 0;
CHECK(absl::SimpleAtoi(sstream.str(), &int64_val)); // Crash OK
dim->add_value_int64(int64_val);
absl::StrAppendFormat(&formatted_str_, "%d ",
dim->value_int64(dim->value_int64_size() - 1));
} else if (typeid(value) == typeid(std::string)) {
dim->add_value_str(sstream.str());
absl::StrAppend(&formatted_str_, "'",
dim->value_str(dim->value_str_size() - 1), "' ");
} else {
CHECK(false) << "Unsupported type: " << typeid(value).name();
}
}
// It assumes the flatten values are stored in row-major, which is mentioned
// indirectly at various places:
// TODO(xpan): Further verifying it.
template <typename T>
int64_t BuildOutput(int64_t start, int depth, const std::vector<T>& values,
TFProfTensorProto* dim) {
formatted_str_ += "[";
int64_t nstart = start;
if (tensor_->dims() == 0 && values.size() == 1) {
std::ostringstream sstream;
sstream << values[nstart];
if (typeid(values[nstart]) == typeid(double)) {
double double_val = 0.0;
CHECK(absl::SimpleAtod(sstream.str(), &double_val)); // Crash OK
dim->add_value_double(double_val);
absl::StrAppendFormat(&formatted_str_, "%.2f ",
dim->value_double(dim->value_double_size() - 1));
} else if (typeid(values[nstart]) == typeid(int64_t)) {
int64_t int64_val = 0;
CHECK(absl::SimpleAtoi(sstream.str(), &int64_val)); // Crash OK
dim->add_value_int64(int64_val);
absl::StrAppendFormat(&formatted_str_, "%d ",
dim->value_int64(dim->value_int64_size() - 1));
} else if (typeid(values[nstart]) == typeid(std::string)) {
dim->add_value_str(sstream.str());
absl::StrAppend(&formatted_str_, "'",
dim->value_str(dim->value_str_size() - 1), "' ");
} else {
CHECK(false) << "Unsupported type: " << typeid(values[nstart]).name();
}
} else {
for (int i = 0; i < tensor_->dim_size(depth); i++) {
// Last dimension, pull the values.
if (depth == tensor_->dims() - 1) {
std::ostringstream sstream;
sstream << values[nstart];
if (typeid(values[nstart]) == typeid(double)) {
double double_val = 0.0;
CHECK(absl::SimpleAtod(sstream.str(), &double_val)); // Crash OK
dim->add_value_double(double_val);
absl::StrAppendFormat(
&formatted_str_, "%.2f ",
dim->value_double(dim->value_double_size() - 1));
} else if (typeid(values[nstart]) == typeid(int64_t)) {
int64_t int64_val = 0;
CHECK(absl::SimpleAtoi(sstream.str(), &int64_val)); // Crash OK
dim->add_value_int64(int64_val);
absl::StrAppendFormat(
&formatted_str_, "%d ",
dim->value_int64(dim->value_int64_size() - 1));
} else if (typeid(values[nstart]) == typeid(std::string)) {
dim->add_value_str(sstream.str());
absl::StrAppend(&formatted_str_, "'",
dim->value_str(dim->value_str_size() - 1), "' ");
} else {
CHECK(false) << "Unsupported type: "
<< typeid(values[nstart]).name();
}
++nstart;
} else {
// Not-last dimension. Drill deeper.
nstart = BuildOutput<T>(nstart, depth + 1, values, dim);
}
}
}
if (formatted_str_.length() > kTFProfTenosrMaxDisplayLen) {
formatted_str_ = formatted_str_.substr(0, kTFProfTenosrMaxDisplayLen);
}
formatted_str_ += "],\n";
return nstart;
}
template <typename T, typename U>
void GetValueVec(std::vector<U>* value_vec) {
// TODO(xpan): Address the huge tensor problem.
if (tensor_->NumElements() > kTFProfTensorMaxWarnLen) {
absl::FPrintF(stderr, "Showing huge tensor, the tool might halt...\n");
}
auto values = tensor_->flat<T>();
for (int64_t i = 0; i < tensor_->NumElements(); i++) {
value_vec->push_back(static_cast<U>(values(i)));
}
}
TFProfTensorProto tfprof_tensor_pb_;
std::unique_ptr<Tensor> tensor_;
std::string formatted_str_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_TENSOR_H_
tensorflow/core/profiler/internal/tfprof_tensor_test.cc
+77
View File
@@ -0,0 +1,77 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <memory>
#include <utility>
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/profiler/internal/tfprof_stats.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_log.pb.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class TFProfTensorTest : public ::testing::Test {
protected:
TFProfTensorTest() {
std::string graph_path =
io::JoinPath(testing::TensorFlowSrcRoot(),
"core/profiler/internal/testdata/graph.pbtxt");
std::unique_ptr<tensorflow::GraphDef> graph_pb =
std::make_unique<tensorflow::GraphDef>();
TF_CHECK_OK(
ReadProtoFile(Env::Default(), graph_path, graph_pb.get(), false));
std::unique_ptr<tensorflow::RunMetadata> run_meta_pb;
std::unique_ptr<OpLogProto> op_log_pb;
std::string ckpt_path = io::JoinPath(
testing::TensorFlowSrcRoot(), "core/profiler/internal/testdata/ckpt");
TF_Status* status = TF_NewStatus();
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader =
std::make_unique<checkpoint::CheckpointReader>(ckpt_path, status);
CHECK(TF_GetCode(status) == TF_OK);
TF_DeleteStatus(status);
tf_stats_ =
std::make_unique<TFStats>(std::move(graph_pb), std::move(run_meta_pb),
std::move(op_log_pb), std::move(ckpt_reader));
tf_stats_->BuildAllViews();
}
std::unique_ptr<TFStats> tf_stats_;
};
TEST_F(TFProfTensorTest, Basics) {
Options opts(3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, "name", {"VariableV2"},
{".*"}, {""}, {".*"}, {""}, false,
{"tensor_value"}, // show the tensor value.
"", {});
const GraphNodeProto& root = tf_stats_->ShowGraphNode("scope", opts);
EXPECT_EQ(root.children(0).name(), "DW");
EXPECT_GT(root.children(0).tensor_value().value_double_size(), 10);
EXPECT_EQ(root.children(1).name(), "DW2");
EXPECT_GT(root.children(1).tensor_value().value_double_size(), 10);
EXPECT_EQ(root.children(2).name(), "ScalarW");
EXPECT_EQ(root.children(2).tensor_value().value_double_size(), 1);
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_timeline.cc
+399
View File
@@ -0,0 +1,399 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_timeline.h"
#include <algorithm>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
namespace tensorflow {
namespace tfprof {
namespace {
int kMaxDisplayedMemNode = 10;
std::string GetTimeDevName(const std::string& dev) {
if (dev.find("stream") != dev.npos) {
return absl::StrCat("Op execution threads: ", dev);
} else {
return absl::StrCat("Op scheduling threads: ", dev);
}
}
std::string GetMemoryLaneName(const std::string& dev) {
return absl::StrCat("mem usage on:", dev);
}
} // namespace
Json::Value ChromeTraceFormatter::CreateEvent(const std::string& ph,
const std::string& category,
const std::string& name,
int64_t pid, int64_t tid,
int64_t ts) {
Json::Value event(Json::objectValue);
event["ph"] = Json::Value(ph);
event["cat"] = Json::Value(category);
event["name"] = Json::Value(name);
event["pid"] = Json::Int64(pid);
event["tid"] = Json::Int64(tid);
event["ts"] = Json::Int64(ts);
return event;
}
void ChromeTraceFormatter::EmitPID(const std::string& name, int64_t pid) {
Json::Value event(Json::objectValue);
event["name"] = Json::Value("process_name");
event["ph"] = Json::Value("M");
event["pid"] = Json::Int64(pid);
Json::Value args(Json::objectValue);
args["name"] = Json::Value(name);
event["args"] = args;
metadata_.push_back(event);
}
void ChromeTraceFormatter::EmitRegion(int64_t ts, int64_t duration, int64_t pid,
int64_t tid, const std::string& category,
const std::string& name,
Json::Value args) {
Json::Value event = CreateEvent("X", category, name, pid, tid, ts);
event["dur"] = Json::Int64(duration);
event["args"] = std::move(args);
metadata_.push_back(event);
}
void ChromeTraceFormatter::EmitFlowStart(const std::string& name, int64_t ts,
int64_t pid, int64_t tid,
int64_t flow_id) {
Json::Value event = CreateEvent("s", "DataFlow", name, pid, tid, ts);
event["id"] = Json::Int64(flow_id);
events_.push_back(event);
}
void ChromeTraceFormatter::EmitFlowEnd(const std::string& name, int64_t ts,
int64_t pid, int64_t tid,
int64_t flow_id) {
Json::Value event = CreateEvent("t", "DataFlow", name, pid, tid, ts);
event["id"] = Json::Int64(flow_id);
events_.push_back(event);
}
void ChromeTraceFormatter::EmitCounter(
const std::string& category, const std::string& name, int64_t pid,
int64_t ts, const std::string& device, int64_t bytes,
const std::map<int64_t, std::vector<std::string>>& tensor_mem) {
Json::Value event = CreateEvent("C", category, "Allocated Bytes", pid, 0, ts);
Json::Value args(Json::objectValue);
args["Allocator Bytes in Use"] = Json::Int64(bytes);
event["args"] = args;
events_.push_back(event);
// TODO(xpan): chrome://tracing is not ideal visualization for memory.
// It would be great to have a customized UI for it.
Json::Value event2 =
CreateEvent("C", category, "Top Allocations", pid + 1, 0, ts);
Json::Value args2(Json::objectValue);
// Need to reserve the same args for all locations.
for (int i = 1; i < kMaxDisplayedMemNode; ++i) {
args2[absl::StrFormat("Top Allocation %02d", i)] = Json::Value("N/A");
}
int count = 0;
for (auto it = tensor_mem.rbegin(); it != tensor_mem.rend(); ++it) {
for (const std::string& t : it->second) {
if (bytes < it->first || count >= kMaxDisplayedMemNode) {
break;
}
args2[absl::StrFormat("Top Allocation %02d", count)] =
Json::Value(absl::StrCat(it->first / 1000000.0, " MB from ", t));
++count;
bytes -= it->first;
}
}
args2["Not Displayed"] =
Json::Value(absl::StrFormat("%.2f MB", bytes / 1000000.0));
event2["args"] = args2;
events_.push_back(event2);
}
std::string ChromeTraceFormatter::Format() {
Json::Value trace;
trace["traceEvents"] = Json::Value(Json::arrayValue);
for (const Json::Value& v : metadata_) {
trace["traceEvents"].append(v);
}
for (const Json::Value& v : events_) {
trace["traceEvents"].append(v);
}
Json::FastWriter writer;
std::string trace_str = writer.write(trace);
if (trace_str.length() > 200 * 1024 * 1024) {
absl::FPrintF(stderr,
"Trace file is over 200MB. Chrome might not be able to "
"display it. Consider to use filters (e.g. -min_micros "
"> 1000 or -op_type .*gpu:0.* to reduce the size.\n");
}
return trace_str;
}
void MemoryTracker::TrackNode(int64_t step, const GraphNode* node) {
if (!node->Trackable(step)) {
return;
}
Device& dev = devices_[node->node->canonical_device()];
std::map<int64_t, int64_t> allocs;
for (const auto& alloc : node->node->allocations(step)) {
allocs[alloc.alloc_micros()] += alloc.alloc_bytes();
dev.tracked_allocations[alloc.alloc_micros()] += alloc.alloc_bytes();
}
dev.tracked_allocations[0] += node->node->accelerator_persistent_bytes();
allocs[0] += node->node->accelerator_persistent_bytes();
int64_t last = 0;
std::map<int64_t, int64_t>& aggregate_allocs =
dev.tensor_allocs[node->name()];
for (auto it = allocs.begin(); it != allocs.end(); ++it) {
last += it->second;
aggregate_allocs[it->first] = last;
}
for (const auto& bytes_in_use : node->node->allocator_bytes_in_use(step)) {
if (bytes_in_use.first <= 0) continue;
dev.allocations[bytes_in_use.first] = bytes_in_use.second;
}
}
void Timeline::AllocateTimeNodes(GraphNode* gnode) {
if (gnode->Trackable(step_)) {
TrackNode(gnode);
const TFGraphNode* node = gnode->node;
for (const auto& kernel_execs : node->op_execs(step_)) {
const std::string& device = kernel_execs.first;
if (process_.find(device) == process_.end()) {
int64_t pid = AllocatePID();
process_[device] = std::make_unique<Process>(device, pid);
chrome_formatter_.EmitPID(GetTimeDevName(device), pid);
}
Process* p = process_[device].get();
for (const auto& exec : kernel_execs.second) {
int64_t start_micros = exec.first;
int64_t exec_micros = exec.second;
// TODO(xpan): There might be start time duplication here.
if (tnodes_[device].find(start_micros) == tnodes_[device].end()) {
// TODO(xpan): Give each kernel call a unique_name.
tnodes_[device][start_micros] =
std::make_unique<TimeNode>(p, gnode, start_micros, exec_micros);
}
}
}
}
for (GraphNode* n : gnode->show_children) {
AllocateTimeNodes(n);
}
}
void Timeline::GenerateGraphTimeline(const std::vector<GraphNode*>& gnodes) {
for (GraphNode* gnode : gnodes) {
AllocateTimeNodes(gnode);
}
// To save memory, we only track cross-device (canonical device) flows.
for (auto& process : tnodes_) {
if (!IsCanonicalDevice(process.first)) continue;
for (auto& tn : process.second) {
TimeNode* tnode = tn.second.get();
for (GraphNode* inp : tnode->node->children) {
if (!inp->account || !inp->Trackable(step_)) {
continue;
}
for (const auto& execs : inp->node->cpu_execs(step_)) {
if (!IsCanonicalDevice(execs.first)) continue;
if (process.first == execs.first) {
// Not interested in flow within the same device.
continue;
}
for (const auto& exec : execs.second) {
int64_t start_micros = exec.first;
auto cprocess = tnodes_.find(execs.first);
if (cprocess == tnodes_.end()) continue;
auto ctn = cprocess->second.find(start_micros);
if (ctn == cprocess->second.end()) continue;
ctn->second->next_tnodes.push_back(tnode);
}
}
}
}
}
AllocateLanes();
absl::FPrintF(stdout, "generating trace file.\n");
int64_t flow_id = 1;
for (const auto& process : alloc_nodes_) {
for (const auto& lane : process.second) {
for (const auto& node : lane.second) {
TimeNode* tnode = node.second;
Json::Value args(Json::objectValue);
args["name"] = Json::Value(tnode->name());
chrome_formatter_.EmitRegion(node.first, tnode->exec_micros,
process.first, lane.first, "Op",
tnode->name(), args);
// Flow is a directed arrow pointing from src to dst.
// TODO(xpan): Disable flow to reduce json file size for now. Need
// to think of a better way to make flow interpretable.
for (TimeNode* next_tnode : node.second->next_tnodes) {
chrome_formatter_.EmitFlowStart(
tnode->name() + "_flow", tnode->start_micros + tnode->exec_micros,
process.first, lane.first, flow_id);
chrome_formatter_.EmitFlowEnd(
tnode->name() + "_flow", next_tnode->start_micros,
next_tnode->process->pid, next_tnode->tid, flow_id);
flow_id += 1;
}
}
}
}
for (const auto& dev : mem_tracker_.devices()) {
if (IsPlacedOnCPU(dev.first)) {
// TODO(xpan): Maybe also support CPU allocator memory tracking.
continue;
}
int64_t pid = AllocatePID();
chrome_formatter_.EmitPID(GetMemoryLaneName(dev.first), pid);
int64_t pid2 = AllocatePID();
chrome_formatter_.EmitPID(GetMemoryLaneName(dev.first) + " allocations",
pid2);
const MemoryTracker::Device& device = dev.second;
int64_t max_bytes_in_use = 0;
int64_t cur_bytes_in_use = 0;
int64_t last_point = 0;
for (const auto& alloc : device.allocations) {
cur_bytes_in_use = alloc.second;
max_bytes_in_use = std::max(max_bytes_in_use, cur_bytes_in_use);
// Do not plot too dense to reduce file size.
int64_t ts = alloc.first;
if (ts - last_point < 100) continue;
last_point = ts;
std::map<int64_t, std::vector<std::string>> tensor_mem;
for (const auto& tensor_alloc_it : dev.second.tensor_allocs) {
const auto& tensor_alloc = tensor_alloc_it.second;
auto it = tensor_alloc.lower_bound(ts);
if (it != tensor_alloc.begin()) {
--it;
}
if (it->second > 0) {
tensor_mem[it->second].push_back(tensor_alloc_it.first);
}
}
chrome_formatter_.EmitCounter("Memory", "Memory Series", pid, ts,
dev.first, cur_bytes_in_use, tensor_mem);
}
if (IsPlacedOnAccelerator(dev.first)) {
absl::FPrintF(stdout, "%s peak memory: %.2f MB\n", dev.first,
max_bytes_in_use / 1000000.0);
}
}
OutputTimeline();
}
void Timeline::GenerateScopeTimeline(const ScopeNode* node) {
std::set<int64_t> visited_depth;
EmitTreeNode(node, 0, node->proto().total_exec_micros(), 0, &visited_depth);
OutputTimeline();
}
void Timeline::GenerateCodeTimeline(const CodeNode* node) {
std::set<int64_t> visited_depth;
EmitTreeNode(node, 0, node->proto().total_exec_micros(), 0, &visited_depth);
OutputTimeline();
}
void Timeline::OutputTimeline() {
std::string outfile = absl::StrFormat("%s_%d", outfile_, step());
absl::Status s =
WriteStringToFile(Env::Default(), outfile, chrome_formatter_.Format());
if (!s.ok()) {
absl::FPrintF(stderr, "Failed to write timeline file: %s\nError: %s\n",
outfile, s.ToString());
return;
}
absl::FPrintF(stdout,
"\n******************************************************\n");
absl::FPrintF(stdout,
"Timeline file is written to %s.\n"
"Open a Chrome browser, enter URL chrome://tracing and "
"load the timeline file.",
outfile);
absl::FPrintF(stdout,
"\n******************************************************\n");
fflush(stdout);
}
void Timeline::AllocateLanes() {
for (auto& process : tnodes_) {
Process* p = process_[process.first].get();
for (auto& tnode : process.second) {
int64_t start_time = tnode.second->start_micros;
int64_t end_time = tnode.second->start_micros + tnode.second->exec_micros;
int64_t l = -1;
for (int64_t i = 0, end = p->lanes.size(); i < end; ++i) {
const auto& lane = p->lanes[i];
l = i;
for (auto cur_it = lane.rbegin(); cur_it != lane.rend(); ++cur_it) {
if (cur_it->second > start_time) {
l = -1;
break;
}
if (start_time > cur_it->second) {
break;
}
}
if (l >= 0) {
break;
}
}
if (l < 0) {
l = p->lanes.size();
std::map<int64_t, int64_t> nlane;
nlane[start_time] = end_time;
p->lanes.push_back(nlane);
} else {
p->lanes[l][start_time] = end_time;
}
tnode.second->tid = l;
alloc_nodes_[p->pid][l][start_time] = tnode.second.get();
}
}
}
int64_t Timeline::AllocatePID() {
int64_t cur_pid = next_pid_;
next_pid_ += 1;
return cur_pid;
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_timeline.h
+199
View File
@@ -0,0 +1,199 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_TIMELINE_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_TIMELINE_H_
#include <map>
#include <memory>
#include <set>
#include <vector>
#include "absl/strings/str_cat.h"
#include "json/json.h"
#include "tensorflow/core/profiler/internal/tfprof_node_show.h"
namespace tensorflow {
namespace tfprof {
typedef std::map<std::string, std::string> Event;
// Class for generating timeline json output.
class ChromeTraceFormatter {
public:
ChromeTraceFormatter() = default;
// The following methods creates timeline nodes. See chrome tracing format
// document for details.
Json::Value CreateEvent(const std::string& ph, const std::string& category,
const std::string& name, int64_t pid, int64_t tid,
int64_t ts);
void EmitPID(const std::string& name, int64_t pid);
void EmitRegion(int64_t ts, int64_t duration, int64_t pid, int64_t tid,
const std::string& category, const std::string& name,
Json::Value args);
void EmitFlowStart(const std::string& name, int64_t ts, int64_t pid,
int64_t tid, int64_t flow_id);
void EmitFlowEnd(const std::string& name, int64_t ts, int64_t pid,
int64_t tid, int64_t flow_id);
void EmitCounter(
const std::string& category, const std::string& name, int64_t pid,
int64_t ts, const std::string& device, int64_t bytes,
const std::map<int64_t, std::vector<std::string>>& tensor_mem);
std::string Format();
private:
// A event is a visualization unit in timeline.
std::vector<Json::Value> events_;
std::vector<Json::Value> metadata_;
};
// A process (time series of events) in the timeline.
class Process {
public:
Process(const std::string& device, int64_t pid) : device(device), pid(pid) {}
// Each lane is a map from start_time to end_time.
std::vector<std::map<int64_t, int64_t>> lanes;
// device for the time series.
std::string device;
// unique id for the time series.
int64_t pid;
};
class TimeNode {
public:
TimeNode(Process* process, GraphNode* node, int64_t start_micros,
int64_t exec_micros)
: process(process),
node(node),
start_micros(start_micros),
exec_micros(exec_micros),
tid(-1) {}
virtual ~TimeNode() = default;
const std::string& name() { return node->name(); }
Process* process;
GraphNode* node;
int64_t start_micros;
int64_t exec_micros;
int64_t tid;
std::vector<TimeNode*> next_tnodes;
};
// Tracking the memory based on the op input/output, temporary bytes and
// persistent bytes.
// Currently, we calculate a "predicted" memory, but do not use it for display.
// The displayed memory timeline is directly from the TensorFlow allocator,
// which is the groundtruth.
class MemoryTracker {
public:
class Device {
public:
// map from tensor name to a pair of <alloc time, bytes_in_use>.
std::map<std::string, std::map<int64_t, int64_t>> tensor_allocs;
// ground truth memory stats. time->bytes.
std::map<int64_t, int64_t> allocations;
// tracked allocations, might miss some bytes.
std::map<int64_t, int64_t> tracked_allocations;
};
void TrackNode(int64_t step, const GraphNode* node);
const std::map<std::string, Device>& devices() const { return devices_; }
private:
std::map<std::string, Device> devices_;
};
class Timeline {
public:
Timeline(int64_t step, const std::string& outfile)
: step_(step), outfile_(outfile) {}
~Timeline() = default;
int64_t step() const { return step_; }
void SetStep(int64_t step) { step_ = step; }
void GenerateGraphTimeline(const std::vector<GraphNode*>& gnodes);
void GenerateScopeTimeline(const ScopeNode* node);
void GenerateCodeTimeline(const CodeNode* node);
private:
void TrackNode(const GraphNode* node) { mem_tracker_.TrackNode(step_, node); }
void OutputTimeline();
template <typename Node>
void EmitTreeNode(const Node* node, int64_t start_time, int64_t duration,
int64_t depth, std::set<int64_t>* visited_depth) {
if (visited_depth->find(depth) == visited_depth->end()) {
chrome_formatter_.EmitPID(absl::StrCat("Scope:", depth), depth);
visited_depth->insert(depth);
}
Json::Value args(Json::objectValue);
args["name"] = Json::Value(node->name());
args["op"] = Json::Value(node->name());
chrome_formatter_.EmitRegion(start_time, duration, depth, 0, "Op",
node->name(), args);
int64_t total_micros = 0;
int64_t c_start_time = start_time;
for (const Node* child : node->show_children) {
int64_t total_exec_micros = child->proto().total_exec_micros();
if (total_exec_micros <= 0) {
continue;
}
EmitTreeNode(child, c_start_time, total_exec_micros, depth + 1,
visited_depth);
c_start_time += total_exec_micros;
total_micros += total_exec_micros;
}
CHECK(total_micros <= duration) << node->name() << " parent:" << duration
<< " children:" << total_micros;
}
void AllocateTimeNodes(GraphNode* gnode);
void AllocateLanes();
int64_t AllocatePID();
int64_t step_;
const std::string outfile_;
int64_t next_pid_ = 0;
MemoryTracker mem_tracker_;
ChromeTraceFormatter chrome_formatter_;
std::map<std::string, int64_t> device_pids_;
std::map<std::string, std::unique_ptr<Process>> process_;
std::map<int64_t, std::map<int64_t, std::map<int64_t, TimeNode*>>>
alloc_nodes_;
std::map<std::string, std::map<int64_t, std::unique_ptr<TimeNode>>> tnodes_;
};
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_TIMELINE_H_
tensorflow/core/profiler/internal/tfprof_timeline_test.cc
+93
View File
@@ -0,0 +1,93 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <memory>
#include <utility>
#include "tensorflow/core/lib/hash/hash.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/profiler/internal/tfprof_constants.h"
#include "tensorflow/core/profiler/internal/tfprof_stats.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_log.pb.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/profiler/tfprof_output.pb.h"
namespace tensorflow {
namespace tfprof {
class TFProfTimelineTest : public ::testing::Test {
protected:
TFProfTimelineTest() {
std::string graph_path =
io::JoinPath(testing::TensorFlowSrcRoot(),
"core/profiler/internal/testdata/graph.pbtxt");
std::unique_ptr<tensorflow::GraphDef> graph_pb =
std::make_unique<tensorflow::GraphDef>();
TF_CHECK_OK(
ReadProtoFile(Env::Default(), graph_path, graph_pb.get(), false));
std::unique_ptr<tensorflow::RunMetadata> run_meta_pb =
std::make_unique<tensorflow::RunMetadata>();
std::string run_meta_path =
io::JoinPath(testing::TensorFlowSrcRoot(),
"core/profiler/internal/testdata/run_meta");
TF_CHECK_OK(
ReadProtoFile(Env::Default(), run_meta_path, run_meta_pb.get(), true));
tf_stats_ = std::make_unique<TFStats>(
std::move(graph_pb), std::move(run_meta_pb), nullptr, nullptr);
tf_stats_->BuildAllViews();
}
std::unique_ptr<TFStats> tf_stats_;
};
// Before adding test, first dump the json file and
// manually check it's correct
TEST_F(TFProfTimelineTest, GraphView) {
std::string dump_file = io::JoinPath(testing::TmpDir(), "dump");
Options opts(10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "name",
{".*"}, // accout_type_regexes
{".*"}, {""}, {".*"}, {""}, false,
{"params", "bytes", "micros", "float_ops"}, "timeline",
{{"outfile", dump_file}});
tf_stats_->ShowGraphNode("graph", opts);
std::string dump_str;
TF_CHECK_OK(ReadFileToString(Env::Default(), dump_file + "_0", &dump_str));
EXPECT_EQ(16556121177519539380ull, Hash64(dump_str));
}
TEST_F(TFProfTimelineTest, ScopeView) {
std::string dump_file = io::JoinPath(testing::TmpDir(), "dump");
Options opts(5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "name",
{".*"}, // accout_type_regexes
{".*"}, {""}, {".*"}, {""}, false,
{"params", "bytes", "micros", "float_ops"}, "timeline",
{{"outfile", dump_file}});
tf_stats_->ShowGraphNode("scope", opts);
std::string dump_str;
TF_CHECK_OK(ReadFileToString(Env::Default(), dump_file + "_0", &dump_str));
EXPECT_EQ(17545174915963890413ull, Hash64(dump_str));
}
// TODO(xpan): tfprof_log is too large to include in testdata when adding
// code traces.
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_utils.cc
+433
View File
@@ -0,0 +1,433 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include <stdio.h>
#include <algorithm>
#include <memory>
#include <set>
#include <vector>
#include "absl/strings/match.h"
#include "absl/strings/numbers.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/regexp.h"
namespace tensorflow {
namespace tfprof {
std::string FormatNumber(int64_t n) {
if (n < 1000) {
return absl::StrFormat("%d", n);
} else if (n < 1000000) {
return absl::StrFormat("%.2fk", n / 1000.0);
} else if (n < 1000000000) {
return absl::StrFormat("%.2fm", n / 1000000.0);
} else {
return absl::StrFormat("%.2fb", n / 1000000000.0);
}
}
std::string FormatTime(int64_t micros) {
if (micros < 1000) {
return absl::StrFormat("%dus", micros);
} else if (micros < 1000000) {
return absl::StrFormat("%.2fms", micros / 1000.0);
} else {
return absl::StrFormat("%.2fsec", micros / 1000000.0);
}
}
std::string FormatMemory(int64_t bytes) {
if (bytes < 1000) {
return absl::StrFormat("%dB", bytes);
} else if (bytes < 1000000) {
return absl::StrFormat("%.2fKB", bytes / 1000.0);
} else {
return absl::StrFormat("%.2fMB", bytes / 1000000.0);
}
}
std::string FormatShapes(const std::vector<int64_t>& shape) {
return absl::StrJoin(shape, "x");
}
std::string StringReplace(const std::string& str, const std::string& oldsub,
const std::string& newsub) {
std::string out = str;
RE2::GlobalReplace(&out, oldsub, newsub);
return out;
}
namespace {
std::string StripQuote(const std::string& s) {
int start = s.find_first_not_of("\"\'");
int end = s.find_last_not_of("\"\'");
if (start == s.npos || end == s.npos) return "";
return s.substr(start, end - start + 1);
}
absl::Status ReturnError(const std::vector<std::string>& pieces, int idx) {
std::string val;
if (pieces.size() > idx + 1) {
val = pieces[idx + 1];
}
return absl::Status(
absl::StatusCode::kInvalidArgument,
absl::StrCat("Invalid option '", pieces[idx], "' value: '", val, "'"));
}
bool CaseEqual(absl::string_view s1, absl::string_view s2) {
if (s1.size() != s2.size()) return false;
return absl::AsciiStrToLower(s1) == absl::AsciiStrToLower(s2);
}
bool StringToBool(absl::string_view str, bool* value) {
CHECK(value != nullptr) << "NULL output boolean given.";
if (CaseEqual(str, "true") || CaseEqual(str, "t") || CaseEqual(str, "yes") ||
CaseEqual(str, "y") || CaseEqual(str, "1")) {
*value = true;
return true;
}
if (CaseEqual(str, "false") || CaseEqual(str, "f") || CaseEqual(str, "no") ||
CaseEqual(str, "n") || CaseEqual(str, "0")) {
*value = false;
return true;
}
return false;
}
} // namespace
absl::Status ParseCmdLine(const std::string& line, std::string* cmd,
tensorflow::tfprof::Options* opts) {
std::vector<std::string> pieces =
absl::StrSplit(line, ' ', absl::SkipEmpty());
std::vector<std::string> cmds_str(kCmds,
kCmds + sizeof(kCmds) / sizeof(*kCmds));
if (std::find(cmds_str.begin(), cmds_str.end(), pieces[0]) ==
cmds_str.end()) {
return absl::Status(absl::StatusCode::kInvalidArgument,
"First string must be a valid command.");
}
*cmd = pieces[0];
for (int i = 1; i < pieces.size(); ++i) {
if (pieces[i] == std::string(tensorflow::tfprof::kOptions[0])) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->max_depth)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[1]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_bytes)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[2]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_peak_bytes)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[3]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_residual_bytes)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[4]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_output_bytes)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[5]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_micros)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[6]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_accelerator_micros)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[7]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_cpu_micros)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[8]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_params)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[9]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_float_ops)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[10]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->min_occurrence)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[11]) {
if (pieces.size() <= i + 1 ||
!absl::SimpleAtoi(pieces[i + 1], &opts->step)) {
return ReturnError(pieces, i);
}
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[12]) {
if (pieces.size() <= i + 1) {
return ReturnError(pieces, i);
}
std::set<std::string> order_by_set(
kOrderBy, kOrderBy + sizeof(kOrderBy) / sizeof(*kOrderBy));
auto order_by = order_by_set.find(pieces[i + 1]);
if (order_by == order_by_set.end()) {
return ReturnError(pieces, i);
}
opts->order_by = *order_by;
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[13]) {
if (pieces.size() <= i + 1) {
return ReturnError(pieces, i);
}
opts->account_type_regexes =
absl::StrSplit(StripQuote(pieces[i + 1]), ',', absl::SkipEmpty());
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[14]) {
if (pieces.size() <= i + 1) {
return ReturnError(pieces, i);
}
opts->start_name_regexes =
absl::StrSplit(StripQuote(pieces[i + 1]), ',', absl::SkipEmpty());
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[15]) {
if (pieces.size() <= i + 1) {
return ReturnError(pieces, i);
}
opts->trim_name_regexes =
absl::StrSplit(StripQuote(pieces[i + 1]), ',', absl::SkipEmpty());
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[16]) {
if (pieces.size() <= i + 1) {
return ReturnError(pieces, i);
}
opts->show_name_regexes =
absl::StrSplit(StripQuote(pieces[i + 1]), ',', absl::SkipEmpty());
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[17]) {
if (pieces.size() <= i + 1) {
return ReturnError(pieces, i);
}
opts->hide_name_regexes =
absl::StrSplit(StripQuote(pieces[i + 1]), ',', absl::SkipEmpty());
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[18]) {
if ((pieces.size() > i + 1 && absl::StartsWith(pieces[i + 1], "-")) ||
pieces.size() == i + 1) {
opts->account_displayed_op_only = true;
} else if (!StringToBool(pieces[i + 1],
&opts->account_displayed_op_only)) {
return ReturnError(pieces, i);
} else {
++i;
}
} else if (pieces[i] == tensorflow::tfprof::kOptions[19]) {
if (pieces.size() <= i + 1) {
return ReturnError(pieces, i);
}
std::set<std::string> shown_set(
kShown, kShown + sizeof(kShown) / sizeof(*kShown));
std::vector<std::string> requested_vector =
absl::StrSplit(StripQuote(pieces[i + 1]), ',', absl::SkipEmpty());
std::set<std::string> requested_set(requested_vector.begin(),
requested_vector.end());
for (const std::string& requested : requested_set) {
if (shown_set.find(requested) == shown_set.end()) {
return ReturnError(pieces, i);
}
}
opts->select = requested_set;
++i;
} else if (pieces[i] == tensorflow::tfprof::kOptions[20]) {
if (pieces.size() <= i + 1) {
return ReturnError(pieces, i);
}
absl::Status s =
ParseOutput(pieces[i + 1], &opts->output_type, &opts->output_options);
if (!s.ok()) return s;
++i;
} else {
return ReturnError(pieces, i);
}
}
return absl::OkStatus();
}
void PrintHelp() {
absl::PrintF(
"See https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/profiler/"
"README.md for profiler tutorial.\n");
absl::PrintF(
"See https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/profiler/"
"g3doc/command_line.md for command line tool tutorial.\n");
absl::PrintF(
"profiler --profile_path=<ProfileProto binary file> # required\n"
"\nOr:\n\n"
"profiler --graph_path=<GraphDef proto file> "
"# Contains model graph info (no needed for eager execution)\n"
" --run_meta_path=<RunMetadata proto file> "
"# Contains runtime info. Optional.\n"
" --run_log_path=<OpLogProto proto file> "
"# Contains extra source code, flops, custom type info. Optional\n\n");
absl::PrintF(
"\nTo skip interactive mode, append one of the following commands:\n"
" scope: Organize profiles based on name scopes.\n"
" graph: Organize profiles based on graph node input/output.\n"
" op: Organize profiles based on operation type.\n"
" code: Organize profiles based on python codes (need op_log_path).\n"
" advise: Auto-profile and advise. (experimental)\n"
" set: Set options that will be default for follow up commands.\n"
" help: Show helps.\n");
fflush(stdout);
}
static const char* const kTotalMicrosHelp =
"total execution time: Sum of accelerator execution time and cpu execution "
"time.";
static const char* const kAccMicrosHelp =
"accelerator execution time: Time spent executing on the accelerator. "
"This is normally measured by the actual hardware library.";
static const char* const kCPUHelp =
"cpu execution time: The time from the start to the end of the operation. "
"It's the sum of actual cpu run time plus the time that it spends waiting "
"if part of computation is launched asynchronously.";
static const char* const kBytes =
"requested bytes: The memory requested by the operation, accumulatively.";
static const char* const kPeakBytes =
"peak bytes: The peak amount of memory that the operation is holding at "
"some point.";
static const char* const kResidualBytes =
"residual bytes: The memory not de-allocated after the operation finishes.";
static const char* const kOutputBytes =
"output bytes: The memory that is output from the operation (not "
"necessarily allocated by the operation)";
static const char* const kOccurrence =
"occurrence: The number of times it occurs";
static const char* const kInputShapes =
"input shape: The shape of input tensors";
static const char* const kDevice = "device: which device is placed on.";
static const char* const kFloatOps =
"flops: Number of float operations. Note: Please read the implementation "
"for the math behind it.";
static const char* const kParams =
"param: Number of parameters (in the Variable).";
static const char* const kTensorValue = "tensor_value: Not supported now.";
static const char* const kOpTypes =
"op_types: The attributes of the operation, includes the Kernel name "
"device placed on and user-defined strings.";
static const char* const kScope =
"scope: The nodes in the model graph are organized by their names, which "
"is hierarchical like filesystem.";
static const char* const kCode =
"code: When python trace is available, the nodes are python lines and "
"their are organized by the python call stack.";
static const char* const kOp =
"op: The nodes are operation kernel type, such as MatMul, Conv2D. Graph "
"nodes belonging to the same type are aggregated together.";
static const char* const kAdvise =
"advise: Automatically profile and discover issues. (Experimental)";
static const char* const kSet =
"set: Set a value for an option for future use.";
static const char* const kHelp = "help: Print helping messages.";
std::string QueryDoc(const std::string& cmd, const Options& opts) {
std::string cmd_help = "";
if (cmd == kCmds[0]) {
cmd_help = kScope;
} else if (cmd == kCmds[1]) {
cmd_help = kScope;
} else if (cmd == kCmds[2]) {
cmd_help = kCode;
} else if (cmd == kCmds[3]) {
cmd_help = kOp;
} else if (cmd == kCmds[4]) {
cmd_help = kAdvise;
} else if (cmd == kCmds[5]) {
cmd_help = kSet;
} else if (cmd == kCmds[6]) {
cmd_help = kHelp;
} else {
cmd_help = "Unknown command: " + cmd;
}
std::vector<std::string> helps;
for (const std::string& s : opts.select) {
if (s == kShown[0]) {
helps.push_back(kBytes);
} else if (s == kShown[1]) {
helps.push_back(
absl::StrCat(kTotalMicrosHelp, "\n", kCPUHelp, "\n", kAccMicrosHelp));
} else if (s == kShown[2]) {
helps.push_back(kParams);
} else if (s == kShown[3]) {
helps.push_back(kFloatOps);
} else if (s == kShown[4]) {
helps.push_back(kTensorValue);
} else if (s == kShown[5]) {
helps.push_back(kDevice);
} else if (s == kShown[6]) {
helps.push_back(kOpTypes);
} else if (s == kShown[7]) {
helps.push_back(kOccurrence);
} else if (s == kShown[8]) {
helps.push_back(kInputShapes);
} else if (s == kShown[9]) {
helps.push_back(kAccMicrosHelp);
} else if (s == kShown[10]) {
helps.push_back(kCPUHelp);
} else if (s == kShown[11]) {
helps.push_back(kPeakBytes);
} else if (s == kShown[12]) {
helps.push_back(kResidualBytes);
} else if (s == kShown[13]) {
helps.push_back(kOutputBytes);
} else {
helps.push_back("Unknown select: " + s);
}
}
return absl::StrCat("\nDoc:\n", cmd_help, "\n", absl::StrJoin(helps, "\n"),
"\n\n");
}
} // namespace tfprof
} // namespace tensorflow
tensorflow/core/profiler/internal/tfprof_utils.h
+73
View File
@@ -0,0 +1,73 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_UTILS_H_
#define TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_UTILS_H_
#include <string>
#include <vector>
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/profiler/tfprof_options.h"
namespace tensorflow {
namespace tfprof {
std::string FormatNumber(int64_t n);
std::string FormatTime(int64_t micros);
std::string FormatMemory(int64_t bytes);
std::string FormatShapes(const std::vector<int64_t>& shapes);
absl::Status ParseCmdLine(const std::string& line, std::string* cmd,
tensorflow::tfprof::Options* opts);
std::string StringReplace(const std::string& str, const std::string& oldsub,
const std::string& newsub);
template <typename T>
absl::Status ReadProtoFile(Env* env, const std::string& fname, T* proto,
bool binary_first) {
std::string out;
absl::Status s = ReadFileToString(env, fname, &out);
if (!s.ok()) return s;
if (binary_first) {
if (ReadBinaryProto(tensorflow::Env::Default(), fname, proto).ok()) {
return absl::Status();
} else if (protobuf::TextFormat::ParseFromString(out, proto)) {
return absl::Status();
}
} else {
if (protobuf::TextFormat::ParseFromString(out, proto)) {
return absl::Status();
} else if (ReadBinaryProto(tensorflow::Env::Default(), fname, proto).ok()) {
return absl::Status();
}
}
return absl::InvalidArgumentError("Cannot parse proto file.");
}
void PrintHelp();
// Generate helper message based on the command and options.
std::string QueryDoc(const std::string& cmd, const Options& opts);
} // namespace tfprof
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_INTERNAL_TFPROF_UTILS_H_
tensorflow/core/profiler/lib/BUILD
+273
View File
@@ -0,0 +1,273 @@
load("//tensorflow:tensorflow.bzl", "if_cuda_or_rocm", "if_not_android", "tf_cc_test", "tf_cuda_library")
load("//tensorflow:tensorflow.default.bzl", "filegroup")
load("//tensorflow/core/platform:build_config_root.bzl", "if_static")
load("//tensorflow/core/platform:rules_cc.bzl", "cc_library")
load(
"//tensorflow/core/profiler/builds:build_config.bzl",
"tf_profiler_copts",
"tf_profiler_pybind_cc_library_wrapper",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:internal"],
licenses = ["notice"],
)
cc_library(
name = "device_profiler_session",
hdrs = ["device_profiler_session.h"],
visibility = ["//tensorflow:internal"],
deps = [
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/platform",
"@tsl//tsl/profiler/protobuf:profiler_options_proto_cc",
] + if_not_android([
":profiler_session",
"//tensorflow/core/profiler/convert:xplane_to_step_stats",
]),
)
tf_profiler_pybind_cc_library_wrapper(
name = "profiler_session_for_pybind",
actual = ":profiler_session",
visibility = ["//tensorflow/python/profiler/internal:__pkg__"],
)
cc_library(
name = "profiler_session",
hdrs = ["profiler_session.h"],
visibility = ["//tensorflow:internal"],
deps = [
"//tensorflow/core:lib",
"//tensorflow/core/platform",
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:profiler_session",
"@tsl//tsl/profiler/protobuf:xplane_proto_cc",
] + if_not_android([
":profiler_interface",
":profiler_lock",
]) + if_static([
"@tsl//tsl/profiler/lib:profiler_session_impl",
]),
)
alias(
name = "profiler_session_impl",
actual = "@tsl//tsl/profiler/lib:profiler_session_impl",
)
cc_library(
name = "profiler_factory",
hdrs = ["profiler_factory.h"],
visibility = ["//tensorflow/c/experimental/pluggable_profiler:__pkg__"],
deps = [
":profiler_interface",
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:profiler_factory",
"@tsl//tsl/profiler/protobuf:profiler_options_proto_cc",
] + if_static([
":profiler_factory_impl",
]),
)
cc_library(
name = "profiler_factory_impl",
srcs = [
"profiler_factory.h",
],
copts = tf_profiler_copts(),
visibility = [
"//tensorflow/core/profiler:__pkg__",
],
deps = [
":profiler_interface",
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:profiler_factory",
"@tsl//tsl/profiler/lib:profiler_factory_impl",
"@tsl//tsl/profiler/protobuf:profiler_options_proto_cc",
],
alwayslink = True,
)
cc_library(
name = "profiler_interface",
hdrs = ["profiler_interface.h"],
copts = tf_profiler_copts(),
visibility = [
"//tensorflow:internal",
"//tensorflow/core/profiler:internal",
],
deps = [
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:profiler_interface",
"@tsl//tsl/profiler/protobuf:xplane_proto_cc",
],
)
cc_library(
name = "profiler_controller",
hdrs = ["profiler_controller.h"],
deps = [
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:profiler_controller",
"@tsl//tsl/profiler/protobuf:xplane_proto_cc",
],
)
tf_cuda_library(
name = "profiler_backends",
visibility = ["//tensorflow:internal"],
deps = [
"@xla//xla/backends/profiler:profiler_backends",
] + if_cuda_or_rocm([
"@xla//xla/backends/profiler/gpu:device_tracer",
]),
alwayslink = True,
)
cc_library(
name = "traceme",
hdrs = ["traceme.h"],
visibility = ["//visibility:public"],
deps = [
":traceme_encode",
"//tensorflow/core:lib",
"//tensorflow/core/platform",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/strings",
"@tsl//tsl/profiler/lib:traceme",
"@tsl//tsl/profiler/lib:traceme_encode",
] + if_not_android([
"@xla//xla/tsl/profiler/backends/cpu:traceme_recorder",
"@xla//xla/tsl/profiler/utils:time_utils",
]),
)
cc_library(
name = "traceme_encode",
hdrs = ["traceme_encode.h"],
visibility = ["//visibility:public"],
deps = [
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/strings",
"@tsl//tsl/profiler/lib:traceme_encode",
],
)
cc_library(
name = "annotated_traceme",
hdrs = ["annotated_traceme.h"],
visibility = ["//visibility:public"],
deps = [
":scoped_annotation",
":traceme",
"//tensorflow/core:lib",
],
)
cc_library(
name = "connected_traceme",
hdrs = ["connected_traceme.h"],
visibility = ["//visibility:public"],
deps = [
":context_types_hdrs",
":traceme_encode",
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:connected_traceme",
],
)
cc_library(
name = "context_types_hdrs",
hdrs = ["context_types.h"],
visibility = ["//visibility:public"],
deps = [
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:context_types_hdrs",
],
)
cc_library(
name = "context_types",
hdrs = ["context_types.h"],
visibility = ["//visibility:public"],
deps = [
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:context_types",
],
)
cc_library(
name = "scoped_memory_debug_annotation",
hdrs = ["scoped_memory_debug_annotation.h"],
visibility = ["//visibility:public"],
deps = [
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:scoped_memory_debug_annotation",
],
)
# TODO(csigg): Remove this forwarding target.
cc_library(
name = "scoped_annotation",
hdrs = ["scoped_annotation.h"],
visibility = ["//visibility:public"],
deps = [
"//tensorflow/core:lib",
"//tensorflow/core/platform",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:string_view",
"@tsl//tsl/profiler/lib:scoped_annotation",
] + if_not_android([
"@xla//xla/tsl/profiler/backends/cpu:annotation_stack",
]),
)
cc_library(
name = "profiler_lock",
hdrs = ["profiler_lock.h"],
copts = tf_profiler_copts(),
visibility = ["//tensorflow/core/profiler:internal"],
deps = [
"@com_google_absl//absl/base:core_headers",
"@tsl//tsl/profiler/lib:profiler_lock",
],
)
tf_cc_test(
name = "profiler_disabled_test",
srcs = ["profiler_disabled_test.cc"],
deps = [
"//tensorflow/core:lib",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"@com_google_absl//absl/status:statusor",
"@tsl//tsl/profiler/lib:profiler_lock",
],
)
filegroup(
name = "mobile_hdrs_no_runtime",
srcs = [
"scoped_annotation.h",
"scoped_memory_debug_annotation.h",
"traceme.h",
"traceme_encode.h",
],
visibility = ["//visibility:public"],
)
filegroup(
name = "mobile_hdrs_only_runtime",
srcs = [
"annotated_traceme.h",
"connected_traceme.h",
"context_types.h",
"device_profiler_session.h",
"profiler_interface.h",
],
visibility = ["//visibility:public"],
)
tensorflow/core/profiler/lib/annotated_traceme.h
+59
View File
@@ -0,0 +1,59 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_ANNOTATED_TRACEME_H_
#define TENSORFLOW_CORE_PROFILER_LIB_ANNOTATED_TRACEME_H_
#include <optional>
#include <string>
#include <utility>
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/profiler/lib/scoped_annotation.h"
#include "tensorflow/core/profiler/lib/traceme.h"
namespace tensorflow {
namespace profiler {
// Combination of TraceMe and ScopedAnnotation which share the same label.
// Optimization are done to ensure the label generation are done once.
class AnnotatedTraceMe {
public:
template <typename NameGeneratorT>
explicit AnnotatedTraceMe(NameGeneratorT&& name_generator, int level = 1) {
DCHECK_GE(level, 1);
bool annotation_enabled = tsl::profiler::ScopedAnnotation::IsEnabled();
bool traceme_enabled = tsl::profiler::TraceMe::Active(level);
if (TF_PREDICT_TRUE(!annotation_enabled && !traceme_enabled)) {
return;
}
std::string name = name_generator();
if (annotation_enabled) {
scoped_annotation_.emplace(name);
}
if (TF_PREDICT_TRUE(traceme_enabled)) {
trace_me_.emplace([&name] { return std::move(name); }, level);
}
}
private:
std::optional<tsl::profiler::TraceMe> trace_me_;
std::optional<tsl::profiler::ScopedAnnotation> scoped_annotation_;
};
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_ANNOTATED_TRACEME_H_
tensorflow/core/profiler/lib/connected_traceme.h
+34
View File
@@ -0,0 +1,34 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_CONNECTED_TRACEME_H_
#define TENSORFLOW_CORE_PROFILER_LIB_CONNECTED_TRACEME_H_
#include "absl/base/macros.h"
#include "tensorflow/core/profiler/lib/context_types.h" // IWYU pragma: keep
#include "tensorflow/core/profiler/lib/traceme_encode.h" // IWYU pragma: keep
#include "tsl/profiler/lib/connected_traceme.h"
namespace tensorflow {
namespace profiler {
using TraceMeConsumer ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::TraceMeConsumer; // NOLINT
using TraceMeProducer ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::TraceMeProducer; // NOLINT
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_CONNECTED_TRACEME_H_
tensorflow/core/profiler/lib/context_types.h
+43
View File
@@ -0,0 +1,43 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_CONTEXT_TYPES_H_
#define TENSORFLOW_CORE_PROFILER_LIB_CONTEXT_TYPES_H_
#include <cstdint>
#include "absl/base/macros.h"
#include "tsl/profiler/lib/context_types.h"
namespace tensorflow {
namespace profiler {
using ContextType ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::ContextType; // NOLINT
ABSL_DEPRECATE_AND_INLINE()
inline const char* GetContextTypeString(
tsl::profiler::ContextType context_type) {
return tsl::profiler::GetContextTypeString(context_type);
}
ABSL_DEPRECATE_AND_INLINE()
inline tsl::profiler::ContextType GetSafeContextType(uint32_t context_type) {
return tsl::profiler::GetSafeContextType(context_type);
}
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_CONTEXT_TYPES_H_
tensorflow/core/profiler/lib/device_profiler_session.h
+83
View File
@@ -0,0 +1,83 @@
/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_DEVICE_PROFILER_SESSION_H_
#define TENSORFLOW_CORE_PROFILER_LIB_DEVICE_PROFILER_SESSION_H_
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/platform.h"
#include "tensorflow/core/platform/status.h"
#if !defined(IS_MOBILE_PLATFORM)
#include "tensorflow/core/profiler/convert/xplane_to_step_stats.h"
#include "tensorflow/core/profiler/lib/profiler_session.h"
#include "tsl/profiler/protobuf/xplane.pb.h"
#endif
#include "tsl/profiler/protobuf/profiler_options.pb.h"
namespace tensorflow {
// Wraps a ProfilerSession configured to collect only device traces.
// Returns data in StepStats format.
class DeviceProfilerSession {
public:
// Creates a DeviceProfilerSession and starts tracing.
// Traces GPU devices if present.
// Does not trace TPU devices (not supported).
static std::unique_ptr<DeviceProfilerSession> Create() {
#if !defined(IS_MOBILE_PLATFORM)
ProfileOptions options = tsl::ProfilerSession::DefaultOptions();
options.set_host_tracer_level(0);
options.set_device_type(ProfileOptions::GPU);
return absl::WrapUnique(new DeviceProfilerSession(options));
#else
return nullptr;
#endif
}
// Stops tracing and converts the data to StepStats format.
// Should be called at most once.
absl::Status CollectData(StepStats* step_stats) {
#if defined(IS_MOBILE_PLATFORM)
return errors::Unimplemented("Profiling not supported on mobile platform.");
#else
profiler::XSpace space;
TF_RETURN_IF_ERROR(profiler_session_->CollectData(&space));
profiler::ConvertGpuXSpaceToStepStats(space, step_stats);
return absl::OkStatus();
#endif
}
private:
// Constructs an instance of the class and starts profiling
explicit DeviceProfilerSession(const ProfileOptions& options)
#if !defined(IS_MOBILE_PLATFORM)
: profiler_session_(tsl::ProfilerSession::Create(options))
#endif
{
}
// DeviceProfilerSession is neither copyable nor movable.
DeviceProfilerSession(const DeviceProfilerSession&) = delete;
DeviceProfilerSession& operator=(const DeviceProfilerSession&) = delete;
#if !defined(IS_MOBILE_PLATFORM)
// TODO(b/256013238)
std::unique_ptr<tsl::ProfilerSession> profiler_session_;
#endif
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_DEVICE_PROFILER_SESSION_H_
tensorflow/core/profiler/lib/profiler_controller.h
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_PROFILER_CONTROLLER_H_
#define TENSORFLOW_CORE_PROFILER_LIB_PROFILER_CONTROLLER_H_
#include "absl/base/macros.h"
#include "tsl/profiler/lib/profiler_controller.h"
#include "tsl/profiler/protobuf/xplane.pb.h"
namespace tensorflow {
namespace profiler {
using ProfilerController ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::ProfilerController; // NOLINT
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_PROFILER_CONTROLLER_H_
tensorflow/core/profiler/lib/profiler_disabled_test.cc
+34
View File
@@ -0,0 +1,34 @@
/* Copyright 2022 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "absl/status/statusor.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/test.h"
#include "tsl/profiler/lib/profiler_lock.h"
namespace tensorflow {
namespace profiler {
namespace {
TEST(ProfilerDisabledTest, ProfilerDisabledTest) {
setenv("TF_DISABLE_PROFILING", "1", /*overwrite=*/1);
absl::StatusOr<tsl::profiler::ProfilerLock> profiler_lock =
tsl::profiler::ProfilerLock::Acquire();
EXPECT_FALSE(profiler_lock.ok());
}
} // namespace
} // namespace profiler
} // namespace tensorflow
tensorflow/core/profiler/lib/profiler_factory.h
+58
View File
@@ -0,0 +1,58 @@
/* Copyright 2019 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_PROFILER_FACTORY_H_
#define TENSORFLOW_CORE_PROFILER_LIB_PROFILER_FACTORY_H_
#include <memory>
#include <utility>
#include <vector>
#include "absl/base/macros.h"
#include "tensorflow/core/profiler/lib/profiler_interface.h" // IWYU pragma: keep
#include "tsl/profiler/lib/profiler_factory.h"
#include "tsl/profiler/protobuf/profiler_options.pb.h"
namespace tensorflow {
namespace profiler {
// A ProfilerFactory returns an instance of ProfilerInterface if ProfileOptions
// require it. Otherwise, it might return nullptr.
using ProfilerFactor ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::ProfilerFactory; // NOLINT
// Registers a profiler factory. Should be invoked at most once per factory.
ABSL_DEPRECATE_AND_INLINE()
inline void RegisterProfilerFactory(tsl::profiler::ProfilerFactory factory) {
tsl::profiler::RegisterProfilerFactory(std::move(factory));
}
// Invokes all registered profiler factories with the given options, and
// returns the instantiated (non-null) profiler interfaces.
ABSL_DEPRECATE_AND_INLINE()
inline std::vector<std::unique_ptr<tsl::profiler::ProfilerInterface>>
CreateProfilers(const tensorflow::ProfileOptions& options) {
return tsl::profiler::CreateProfilers(options);
}
// For testing only.
ABSL_DEPRECATE_AND_INLINE()
inline void ClearRegisteredProfilersForTest() {
tsl::profiler::ClearRegisteredProfilersForTest();
}
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_PROFILER_FACTORY_H_
tensorflow/core/profiler/lib/profiler_interface.h
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_PROFILER_INTERFACE_H_
#define TENSORFLOW_CORE_PROFILER_LIB_PROFILER_INTERFACE_H_
#include "absl/base/macros.h"
#include "tsl/profiler/lib/profiler_interface.h"
#include "tsl/profiler/protobuf/xplane.pb.h"
namespace tensorflow {
namespace profiler {
using ProfilerInterface ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::ProfilerInterface; // NOLINT
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_PROFILER_INTERFACE_H_
tensorflow/core/profiler/lib/profiler_lock.h
+30
View File
@@ -0,0 +1,30 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_PROFILER_LOCK_H_
#define TENSORFLOW_CORE_PROFILER_LIB_PROFILER_LOCK_H_
#include "absl/base/macros.h"
#include "tsl/profiler/lib/profiler_lock.h"
namespace tensorflow {
namespace profiler {
using ProfilerLock ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::ProfilerLock; // NOLINT
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_PROFILER_LOCK_H_
tensorflow/core/profiler/lib/profiler_session.h
+27
View File
@@ -0,0 +1,27 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_PROFILER_SESSION_H_
#define TENSORFLOW_CORE_PROFILER_LIB_PROFILER_SESSION_H_
#include "absl/base/macros.h"
#include "tsl/profiler/lib/profiler_session.h"
namespace tensorflow {
using ProfilerSession ABSL_DEPRECATE_AND_INLINE() =
tsl::ProfilerSession; // NOLINT
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_PROFILER_SESSION_H_
tensorflow/core/profiler/lib/scoped_annotation.h
+30
View File
@@ -0,0 +1,30 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_SCOPED_ANNOTATION_H_
#define TENSORFLOW_CORE_PROFILER_LIB_SCOPED_ANNOTATION_H_
#include "absl/base/macros.h"
#include "tsl/profiler/lib/scoped_annotation.h"
namespace tensorflow {
namespace profiler {
using ScopedAnnotation ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::ScopedAnnotation; // NOLINT
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_SCOPED_ANNOTATION_H_
tensorflow/core/profiler/lib/scoped_memory_debug_annotation.h
+32
View File
@@ -0,0 +1,32 @@
/* Copyright 2021 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_SCOPED_MEMORY_DEBUG_ANNOTATION_H_
#define TENSORFLOW_CORE_PROFILER_LIB_SCOPED_MEMORY_DEBUG_ANNOTATION_H_
#include "absl/base/macros.h"
#include "tsl/profiler/lib/scoped_memory_debug_annotation.h"
namespace tensorflow {
namespace profiler {
using MemoryDebugAnnotation ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::MemoryDebugAnnotation; // NOLINT
using ScopedMemoryDebugAnnotation ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::ScopedMemoryDebugAnnotation; // NOLINT
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_SCOPED_MEMORY_DEBUG_ANNOTATION_H_
tensorflow/core/profiler/lib/traceme.h
+41
View File
@@ -0,0 +1,41 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_TRACEME_H_
#define TENSORFLOW_CORE_PROFILER_LIB_TRACEME_H_
#include "absl/base/macros.h"
#include "tensorflow/core/profiler/lib/traceme_encode.h" // IWYU pragma: export
#include "tsl/profiler/lib/traceme.h"
namespace tensorflow {
namespace profiler {
using tsl::profiler::kInfo; // NOLINT
using TraceMe ABSL_DEPRECATE_AND_INLINE() = tsl::profiler::TraceMe; // NOLINT
using TraceMeLevel ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::TraceMeLevel; // NOLINT
ABSL_DEPRECATE_AND_INLINE()
inline int GetTFTraceMeLevel(bool is_expensive) {
return tsl::profiler::GetTFTraceMeLevel(is_expensive);
}
ABSL_DEPRECATE_AND_INLINE()
inline bool TfOpDetailsEnabled() { return tsl::profiler::TfOpDetailsEnabled(); }
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_TRACEME_H_
tensorflow/core/profiler/lib/traceme_encode.h
+89
View File
@@ -0,0 +1,89 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_LIB_TRACEME_ENCODE_H_
#define TENSORFLOW_CORE_PROFILER_LIB_TRACEME_ENCODE_H_
#include <initializer_list>
#include <string>
#include <utility>
#include "absl/base/macros.h"
#include "absl/strings/string_view.h"
#include "tsl/profiler/lib/traceme_encode.h"
namespace tensorflow {
namespace profiler {
using TraceMeArg ABSL_DEPRECATE_AND_INLINE() =
tsl::profiler::TraceMeArg; // NOLINT
ABSL_DEPRECATE_AND_INLINE()
inline std::string TraceMeEncode(
std::string name, std::initializer_list<tsl::profiler::TraceMeArg> args) {
return tsl::profiler::TraceMeEncode(std::move(name), args);
}
ABSL_DEPRECATE_AND_INLINE()
inline std::string TraceMeEncode(
absl::string_view name,
std::initializer_list<tsl::profiler::TraceMeArg> args) {
return tsl::profiler::TraceMeEncode(name, args);
}
ABSL_DEPRECATE_AND_INLINE()
inline std::string TraceMeEncode(
const char* name, std::initializer_list<tsl::profiler::TraceMeArg> args) {
return tsl::profiler::TraceMeEncode(name, args);
}
ABSL_DEPRECATE_AND_INLINE()
inline std::string TraceMeEncode(
std::initializer_list<tsl::profiler::TraceMeArg> args) {
return tsl::profiler::TraceMeEncode(args);
}
ABSL_DEPRECATE_AND_INLINE()
// Concatenates op_name and op_type.
inline std::string TraceMeOp(absl::string_view op_name,
absl::string_view op_type) {
return tsl::profiler::TraceMeOp(op_name, op_type);
}
ABSL_DEPRECATE_AND_INLINE()
inline std::string TraceMeOp(const char* op_name, const char* op_type) {
return tsl::profiler::TraceMeOp(op_name, op_type);
}
ABSL_DEPRECATE_AND_INLINE()
inline std::string TraceMeOp(std::string&& op_name, absl::string_view op_type) {
return tsl::profiler::TraceMeOp(op_name, op_type);
}
ABSL_DEPRECATE_AND_INLINE()
// Concatenates op_name and op_type.
inline std::string TraceMeOpOverride(absl::string_view op_name,
absl::string_view op_type) {
return tsl::profiler::TraceMeOpOverride(op_name, op_type);
}
ABSL_DEPRECATE_AND_INLINE()
inline std::string TraceMeOpOverride(const char* op_name, const char* op_type) {
return tsl::profiler::TraceMeOpOverride(op_name, op_type);
}
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_LIB_TRACEME_ENCODE_H_
tensorflow/core/profiler/profile.proto
+21
View File
@@ -0,0 +1,21 @@
// Copyright 2026 The TensorFlow Authors All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ==============================================================================
// This proto intends to match format expected by pprof tool.
syntax = "proto3";
package tensorflow.tfprof.pprof.dummy;
import public "tsl/profiler/protobuf/profile.proto";
tensorflow/core/profiler/profiler.cc
+333
View File
@@ -0,0 +1,333 @@
/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <stdio.h>
#include <stdlib.h>
#include <cstdint>
#include <limits>
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/log/check.h"
#include "absl/strings/match.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "linenoise.h"
#include "tensorflow/c/checkpoint_reader.h"
#include "tensorflow/c/tf_status.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/profiler/internal/advisor/tfprof_advisor.h"
#include "tensorflow/core/profiler/internal/tfprof_stats.h"
#include "tensorflow/core/profiler/internal/tfprof_utils.h"
#include "tensorflow/core/profiler/tfprof_log.pb.h"
#include "tensorflow/core/profiler/tfprof_options.h"
#include "tensorflow/core/protobuf/config.pb.h"
#include "tensorflow/core/util/command_line_flags.h"
namespace tensorflow {
namespace tfprof {
void completion(const char* buf, linenoiseCompletions* lc) {
std::string buf_str = buf;
if (buf_str.find(' ') == buf_str.npos) {
for (const char* opt : kCmds) {
if (absl::StartsWith(opt, buf_str)) {
linenoiseAddCompletion(lc, opt);
}
}
return;
}
std::string prefix;
int last_dash = buf_str.find_last_of(' ');
if (last_dash != std::string::npos) {
prefix = buf_str.substr(0, last_dash + 1);
buf_str =
buf_str.substr(last_dash + 1, std::numeric_limits<int32_t>::max());
}
for (const char* opt : kOptions) {
if (absl::StartsWith(opt, buf_str)) {
linenoiseAddCompletion(lc, (prefix + opt).c_str());
}
}
}
int Run(int argc, char** argv) {
std::string FLAGS_profile_path = "";
std::string FLAGS_graph_path = "";
std::string FLAGS_run_meta_path = "";
std::string FLAGS_op_log_path = "";
std::string FLAGS_checkpoint_path = "";
int32_t FLAGS_max_depth = 10;
int64_t FLAGS_min_bytes = 0;
int64_t FLAGS_min_peak_bytes = 0;
int64_t FLAGS_min_residual_bytes = 0;
int64_t FLAGS_min_output_bytes = 0;
int64_t FLAGS_min_micros = 0;
int64_t FLAGS_min_accelerator_micros = 0;
int64_t FLAGS_min_cpu_micros = 0;
int64_t FLAGS_min_params = 0;
int64_t FLAGS_min_float_ops = 0;
int64_t FLAGS_min_occurrence = 0;
int64_t FLAGS_step = -1;
std::string FLAGS_order_by = "name";
std::string FLAGS_account_type_regexes = ".*";
std::string FLAGS_start_name_regexes = ".*";
std::string FLAGS_trim_name_regexes = "";
std::string FLAGS_show_name_regexes = ".*";
std::string FLAGS_hide_name_regexes;
bool FLAGS_account_displayed_op_only = false;
std::string FLAGS_select = "micros";
std::string FLAGS_output = "";
for (int i = 0; i < argc; i++) {
absl::FPrintF(stderr, "%s\n", argv[i]);
}
std::vector<Flag> flag_list = {
Flag("profile_path", &FLAGS_profile_path, "Profile binary file name"),
Flag("graph_path", &FLAGS_graph_path, "GraphDef proto text file name"),
Flag("run_meta_path", &FLAGS_run_meta_path,
"Comma-separated list of RunMetadata proto binary "
"files. Each file is given step number 0,1,2,etc"),
Flag("op_log_path", &FLAGS_op_log_path,
"tensorflow::tfprof::OpLogProto proto binary file name"),
Flag("checkpoint_path", &FLAGS_checkpoint_path,
"TensorFlow Checkpoint file name"),
Flag("max_depth", &FLAGS_max_depth, "max depth"),
Flag("min_bytes", &FLAGS_min_bytes, "min_bytes"),
Flag("min_peak_bytes", &FLAGS_min_peak_bytes, "min_peak_bytes"),
Flag("min_residual_bytes", &FLAGS_min_residual_bytes,
"min_residual_bytes"),
Flag("min_output_bytes", &FLAGS_min_output_bytes, "min_output_bytes"),
Flag("min_micros", &FLAGS_min_micros, "min micros"),
Flag("min_accelerator_micros", &FLAGS_min_accelerator_micros,
"min accelerator_micros"),
Flag("min_cpu_micros", &FLAGS_min_cpu_micros, "min_cpu_micros"),
Flag("min_params", &FLAGS_min_params, "min params"),
Flag("min_float_ops", &FLAGS_min_float_ops, "min float ops"),
Flag("min_occurrence", &FLAGS_min_occurrence, "min occurrence"),
Flag("step", &FLAGS_step,
"The stats of which step to use. By default average"),
Flag("order_by", &FLAGS_order_by, "order by"),
Flag("account_type_regexes", &FLAGS_start_name_regexes,
"start name regexes"),
Flag("trim_name_regexes", &FLAGS_trim_name_regexes, "trim name regexes"),
Flag("show_name_regexes", &FLAGS_show_name_regexes, "show name regexes"),
Flag("hide_name_regexes", &FLAGS_hide_name_regexes, "hide name regexes"),
Flag("account_displayed_op_only", &FLAGS_account_displayed_op_only,
"account displayed op only"),
Flag("select", &FLAGS_select, "select"),
Flag("output", &FLAGS_output, "output"),
};
std::string usage = Flags::Usage(argv[0], flag_list);
bool parse_ok = Flags::Parse(&argc, argv, flag_list);
if (!parse_ok) {
absl::PrintF("%s", usage);
return (2);
}
port::InitMain(argv[0], &argc, &argv);
if (!FLAGS_profile_path.empty() &&
(!FLAGS_graph_path.empty() || !FLAGS_run_meta_path.empty())) {
absl::FPrintF(stderr,
"--profile_path is set, do not set --graph_path or "
"--run_meta_path\n");
return 1;
}
std::vector<std::string> account_type_regexes =
absl::StrSplit(FLAGS_account_type_regexes, ',', absl::SkipEmpty());
std::vector<std::string> start_name_regexes =
absl::StrSplit(FLAGS_start_name_regexes, ',', absl::SkipEmpty());
std::vector<std::string> trim_name_regexes =
absl::StrSplit(FLAGS_trim_name_regexes, ',', absl::SkipEmpty());
std::vector<std::string> show_name_regexes =
absl::StrSplit(FLAGS_show_name_regexes, ',', absl::SkipEmpty());
std::vector<std::string> hide_name_regexes =
absl::StrSplit(FLAGS_hide_name_regexes, ',', absl::SkipEmpty());
std::vector<std::string> select =
absl::StrSplit(FLAGS_select, ',', absl::SkipEmpty());
std::string output_type;
std::map<std::string, std::string> output_options;
absl::Status s = ParseOutput(FLAGS_output, &output_type, &output_options);
CHECK(s.ok()) << s;
std::string cmd = "";
if (argc == 1 && FLAGS_graph_path.empty() && FLAGS_profile_path.empty() &&
FLAGS_run_meta_path.empty()) {
PrintHelp();
return 0;
} else if (argc > 1) {
if (std::string(argv[1]) == kCmds[6]) {
PrintHelp();
return 0;
}
if (std::string(argv[1]) == kCmds[0] || std::string(argv[1]) == kCmds[1] ||
std::string(argv[1]) == kCmds[2] || std::string(argv[1]) == kCmds[3] ||
std::string(argv[1]) == kCmds[4]) {
cmd = argv[1];
}
}
absl::PrintF("Reading Files...\n");
std::unique_ptr<checkpoint::CheckpointReader> ckpt_reader;
TF_Status* status = TF_NewStatus();
if (!FLAGS_checkpoint_path.empty()) {
ckpt_reader = std::make_unique<checkpoint::CheckpointReader>(
FLAGS_checkpoint_path, status);
if (TF_GetCode(status) != TF_OK) {
absl::FPrintF(stderr, "%s\n", TF_Message(status));
TF_DeleteStatus(status);
return 1;
}
TF_DeleteStatus(status);
}
std::unique_ptr<TFStats> tf_stat;
if (!FLAGS_profile_path.empty()) {
tf_stat =
std::make_unique<TFStats>(FLAGS_profile_path, std::move(ckpt_reader));
} else {
absl::PrintF(
"Try to use a single --profile_path instead of "
"graph_path,op_log_path,run_meta_path\n");
std::unique_ptr<GraphDef> graph = std::make_unique<GraphDef>();
if (!FLAGS_graph_path.empty()) {
s = ReadProtoFile(Env::Default(), FLAGS_graph_path, graph.get(), false);
if (!s.ok()) {
absl::FPrintF(stderr, "Failed to read graph_path: %s\n", s.ToString());
return 1;
}
}
std::unique_ptr<OpLogProto> op_log = std::make_unique<OpLogProto>();
if (!FLAGS_op_log_path.empty()) {
std::string op_log_str;
s = ReadFileToString(Env::Default(), FLAGS_op_log_path, &op_log_str);
if (!s.ok()) {
absl::FPrintF(stderr, "Failed to read op_log_path: %s\n", s.ToString());
return 1;
}
if (!ParseProtoUnlimited(op_log.get(), op_log_str)) {
absl::FPrintF(stderr, "Failed to parse op_log_path\n");
return 1;
}
}
tf_stat = std::make_unique<TFStats>(
std::move(graph), nullptr, std::move(op_log), std::move(ckpt_reader));
std::vector<std::string> run_meta_files =
absl::StrSplit(FLAGS_run_meta_path, ',', absl::SkipEmpty());
for (int i = 0; i < run_meta_files.size(); ++i) {
std::unique_ptr<RunMetadata> run_meta = std::make_unique<RunMetadata>();
s = ReadProtoFile(Env::Default(), run_meta_files[i], run_meta.get(),
true);
if (!s.ok()) {
absl::FPrintF(stderr, "Failed to read run_meta_path %s. Status: %s\n",
run_meta_files[i], s.ToString());
return 1;
}
tf_stat->AddRunMeta(i, std::move(run_meta));
absl::FPrintF(stdout, "run graph coverage: %.2f\n",
tf_stat->run_coverage());
}
}
if (cmd == kCmds[4]) {
tf_stat->BuildAllViews();
Advisor(tf_stat.get()).Advise(Advisor::DefaultOptions());
return 0;
}
Options opts(
FLAGS_max_depth, FLAGS_min_bytes, FLAGS_min_peak_bytes,
FLAGS_min_residual_bytes, FLAGS_min_output_bytes, FLAGS_min_micros,
FLAGS_min_accelerator_micros, FLAGS_min_cpu_micros, FLAGS_min_params,
FLAGS_min_float_ops, FLAGS_min_occurrence, FLAGS_step, FLAGS_order_by,
account_type_regexes, start_name_regexes, trim_name_regexes,
show_name_regexes, hide_name_regexes, FLAGS_account_displayed_op_only,
select, output_type, output_options);
if (cmd == kCmds[2] || cmd == kCmds[3]) {
tf_stat->BuildView(cmd);
tf_stat->ShowMultiGraphNode(cmd, opts);
return 0;
} else if (cmd == kCmds[0] || cmd == kCmds[1]) {
tf_stat->BuildView(cmd);
tf_stat->ShowGraphNode(cmd, opts);
return 0;
}
linenoiseSetCompletionCallback(completion);
linenoiseHistoryLoad(".tfprof_history.txt");
bool looped = false;
while (true) {
char* line = linenoise("tfprof> ");
if (line == nullptr) {
if (!looped) {
absl::FPrintF(stderr,
"Cannot start interactive shell, "
"use 'bazel-bin' instead of 'bazel run'.\n");
}
break;
}
looped = true;
std::string line_s = line;
free(line);
if (line_s.empty()) {
absl::PrintF("%s", opts.ToString());
continue;
}
linenoiseHistoryAdd(line_s.c_str());
linenoiseHistorySave(".tfprof_history.txt");
Options new_opts = opts;
absl::Status s = ParseCmdLine(line_s, &cmd, &new_opts);
if (!s.ok()) {
absl::FPrintF(stderr, "E: %s\n", s.ToString());
continue;
}
if (cmd == kCmds[5]) {
opts = new_opts;
} else if (cmd == kCmds[6]) {
PrintHelp();
} else if (cmd == kCmds[2] || cmd == kCmds[3]) {
tf_stat->BuildView(cmd);
tf_stat->ShowMultiGraphNode(cmd, new_opts);
} else if (cmd == kCmds[0] || cmd == kCmds[1]) {
tf_stat->BuildView(cmd);
tf_stat->ShowGraphNode(cmd, new_opts);
} else if (cmd == kCmds[4]) {
tf_stat->BuildAllViews();
Advisor(tf_stat.get()).Advise(Advisor::DefaultOptions());
}
}
return 0;
}
} // namespace tfprof
} // namespace tensorflow
int main(int argc, char** argv) { return tensorflow::tfprof::Run(argc, argv); }
tensorflow/core/profiler/profiler_analysis.proto
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2026 The TensorFlow Authors All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ==============================================================================
syntax = "proto3";
package tensorflow.dummy;
import public "tsl/profiler/protobuf/profiler_analysis.proto";
tensorflow/core/profiler/profiler_options.proto
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2026 The TensorFlow Authors All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ==============================================================================
syntax = "proto3";
package tensorflow.dummy;
import public "tsl/profiler/protobuf/profiler_options.proto";
tensorflow/core/profiler/profiler_service.proto
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2026 The TensorFlow Authors All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ==============================================================================
syntax = "proto3";
package tensorflow.dummy;
import public "tsl/profiler/protobuf/profiler_service.proto";

Some files were not shown because too many files have changed in this diff Show More