wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/core/debug/BUILD
+311
View File
@@ -0,0 +1,311 @@
# Description:
# TensorFlow Debugger (tfdbg).
#
# Public target(s):
#
# ":debug" - Depending on this target causes a concrete implementation of
# DebuggerState to be constructed at initialization time, enabling
# TensorFlow Debugger (tfdbg) support. For details, please see
# core/common_runtime/debugger_state_interface.h.
# ":debug_callback_registry" - Depending on this target exposes a global
# callback registry that will be used to record any observed tensors matching
# a watch state.
# ":debug_node_key" - Defines a struct used for tracking tensors.
load("@rules_python//python:proto.bzl", "py_proto_library")
load(
"//tensorflow:tensorflow.bzl",
"check_deps",
"if_windows",
"tf_cc_binary",
"tf_cc_test",
"tf_copts",
"tf_cuda_library",
)
load("//tensorflow:tensorflow.default.bzl", "tf_grpc_cc_dependencies")
# For platform specific build config
load(
"//tensorflow/core/platform:build_config.bzl",
"tf_additional_all_protos",
"tf_proto_library",
)
load("//tensorflow/core/platform:rules_cc.bzl", "cc_library")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:internal"],
licenses = ["notice"],
)
# Check that tensorflow/core:tensorflow does not depend on grpc.
check_deps(
name = "core_tensorflow_check_deps",
disallowed_deps = ["@com_github_grpc_grpc//:grpc++"],
deps = ["//tensorflow/core:tensorflow"],
)
tf_proto_library(
name = "debug_service_proto",
srcs = [
"debug_service.proto",
],
has_services = 1,
create_grpc_library = True,
protodeps = [
":debugger_event_metadata_proto",
"//tensorflow/core/profiler:protos_all",
] + tf_additional_all_protos(),
visibility = ["//tensorflow:__subpackages__"],
)
tf_proto_library(
name = "debugger_event_metadata_proto",
srcs = ["debugger_event_metadata.proto"],
)
cc_library(
name = "debug",
srcs = ["debug.cc"],
copts = tf_copts(),
linkstatic = 1,
visibility = ["//visibility:public"],
deps = [
":debugger_state_impl",
"//tensorflow/core:core_cpu_internal",
"//tensorflow/core:debug_ops_op_lib",
# Depends on grpc and hence should stay out of
# //third_party/tensorflow/core.
"//tensorflow/core/kernels:debug_ops",
],
alwayslink = 1,
)
tf_cuda_library(
name = "debugger_state_impl",
srcs = ["debugger_state_impl.cc"],
hdrs = ["debugger_state_impl.h"],
copts = tf_copts(),
linkstatic = 1,
deps = [
":debug_graph_utils",
":debug_io_utils",
"//tensorflow/core:core_cpu_internal",
],
alwayslink = 1,
)
tf_cuda_library(
name = "debug_graph_utils",
srcs = ["debug_graph_utils.cc"],
hdrs = ["debug_graph_utils.h"],
copts = tf_copts(),
linkstatic = 1,
deps = [
"//tensorflow/core:core_cpu_internal",
"//tensorflow/core:framework",
"//tensorflow/core:graph",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:protos_all_cc",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
)
tf_cuda_library(
name = "debug_io_utils",
srcs = ["debug_io_utils.cc"],
hdrs = ["debug_io_utils.h"],
copts = tf_copts(),
linkopts = if_windows(["-DEFAULTLIB:ws2_32.lib"]),
linkstatic = 1,
deps = [
":debug_callback_registry",
":debug_node_key",
":debug_service_cc_grpc_proto",
":debugger_event_metadata_proto_cc",
"//tensorflow/core:core_cpu_internal",
"//tensorflow/core:framework",
"//tensorflow/core:graph",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:protos_all_cc",
"@com_google_absl//absl/strings",
] + tf_grpc_cc_dependencies(),
alwayslink = 1,
)
tf_cuda_library(
name = "debug_grpc_testlib",
srcs = ["debug_grpc_testlib.cc"],
hdrs = ["debug_grpc_testlib.h"],
copts = tf_copts(),
linkstatic = 1,
deps = [
":debug_graph_utils",
":debug_io_utils",
":debug_service_cc_grpc_proto",
":debugger_event_metadata_proto_cc",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:protos_all_cc",
] + tf_grpc_cc_dependencies(),
alwayslink = 1,
)
tf_cuda_library(
name = "debug_node_key",
srcs = ["debug_node_key.cc"],
hdrs = ["debug_node_key.h"],
copts = tf_copts(),
linkstatic = 1,
visibility = ["//visibility:public"],
deps = [
"//tensorflow/core:lib",
],
)
tf_cc_test(
name = "debug_io_utils_test",
size = "small",
srcs = ["debug_io_utils_test.cc"],
tags = [
"no_oss", # TODO(b/137652456): remove when fixed
],
deps = [
":debug_callback_registry",
":debug_grpc_testlib",
":debug_io_utils",
":debug_node_key",
":debugger_event_metadata_proto_cc",
"//tensorflow/core:core_cpu_internal",
"//tensorflow/core:framework_internal",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
"@com_google_absl//absl/synchronization",
],
)
tf_cc_test(
name = "debug_graph_utils_test",
size = "small",
srcs = ["debug_graph_utils_test.cc"],
deps = [
":debug_graph_utils",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
],
)
tf_cc_test(
name = "grpc_session_debug_test",
size = "medium",
srcs = ["grpc_session_debug_test.cc"],
tags = [
"no_oss", # b/62956105: port conflicts.
"nomac", # b/38276817
],
deps = [
":debug_grpc_testlib",
":debug_io_utils",
"//tensorflow/core:array_ops_op_lib",
"//tensorflow/core:core_cpu",
"//tensorflow/core:framework",
"//tensorflow/core:functional_ops_op_lib",
"//tensorflow/core:lib",
"//tensorflow/core:math_ops_op_lib",
"//tensorflow/core:nn_ops_op_lib",
"//tensorflow/core:no_op_op_lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core:sendrecv_ops_op_lib",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
"//tensorflow/core/distributed_runtime/rpc:grpc_server_lib",
"//tensorflow/core/distributed_runtime/rpc:grpc_session",
"//tensorflow/core/distributed_runtime/rpc:grpc_testlib",
"//tensorflow/core/kernels:constant_op",
"//tensorflow/core/kernels:matmul_op",
"//tensorflow/core/protobuf:master_proto_cc",
],
)
tf_cc_test(
name = "debug_grpc_io_utils_test",
size = "small",
srcs = ["debug_grpc_io_utils_test.cc"],
tags = [
"no_oss", # b/73962011
],
deps = [
":debug_graph_utils",
":debug_grpc_testlib",
":debug_io_utils",
"//tensorflow/core:core_cpu_internal",
"//tensorflow/core:framework_internal",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"@com_google_absl//absl/synchronization",
],
)
cc_library(
name = "debug_callback_registry",
srcs = ["debug_callback_registry.cc"],
hdrs = ["debug_callback_registry.h"],
visibility = ["//visibility:public"],
deps = [
":debug_node_key",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
],
)
tf_cc_binary(
name = "bfc_dump_reader",
srcs = ["bfc_dump_reader.cc"],
deps = [
"//tensorflow/core:framework_internal",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/platform:regexp",
],
)
# copybara:uncomment_begin(google-only)
# py_proto_library(
# name = "debug_service_py_pb2",
# has_services = 1,
# deps = [":debug_service_proto"],
# )
#
# py_proto_library(
# name = "debugger_event_metadata_py_pb2",
# deps = [":debugger_event_metadata_proto"],
# )
# copybara:uncomment_end
# TODO(cais): Add the following back in when tfdbg is supported on Android.
# filegroup(
# name = "android_srcs",
# srcs = [
# "debug_graph_utils.cc",
# "debug_graph_utils.h",
# "debug_io_utils.cc",
# "debug_io_utils.h",
# ],
# visibility = ["//visibility:public"],
# )
tensorflow/core/debug/bfc_dump_reader.cc
+280
View File
@@ -0,0 +1,280 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cinttypes>
#include <string>
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/file_system.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/platform/regexp.h"
#include "tensorflow/core/protobuf/bfc_memory_map.pb.h"
#include "tensorflow/core/util/command_line_flags.h"
namespace tensorflow {
MemoryDump ReadDumpFile(const std::string& fname) {
absl::Status status;
uint64_t file_size = 0;
status = Env::Default()->GetFileSize(fname, &file_size);
if (!status.ok()) {
LOG(ERROR) << "Failed to get size of " << fname;
exit(1);
}
std::unique_ptr<RandomAccessFile> file;
status = Env::Default()->NewRandomAccessFile(fname, &file);
if (!status.ok()) {
LOG(ERROR) << "read from file " << fname << " failed " << status;
}
std::unique_ptr<char> buffer(static_cast<char*>(malloc(file_size + 1)));
DCHECK(buffer.get());
absl::string_view contents(buffer.get(), file_size);
status = file->Read(0, contents, absl::MakeSpan(buffer.get(), file_size));
if (!status.ok()) {
LOG(ERROR) << "read from file " << fname << " failed " << status;
}
MemoryDump md;
md.ParseFromString(contents);
return md;
}
MemoryDump FilterByChunkType(MemoryDump md, const char chunk_type) {
if (chunk_type == 'A' || (chunk_type != 'U' && chunk_type != 'F')) {
return md;
}
MemoryDump filtered;
filtered.set_allocator_name(md.allocator_name());
for (const auto& it : md.bin_summary()) {
*filtered.add_bin_summary() = it;
}
for (const auto& it : md.chunk()) {
if ((chunk_type == 'U' && it.in_use()) ||
(chunk_type == 'F' && !it.in_use())) {
*filtered.add_chunk() = it;
}
}
return filtered;
}
void PrintChunk(const MemChunk& mc, const uint64_t ac_offset, bool freed_at,
const int64_t total_bytes, int64_t* cumulative_bytes) {
// A size class corresponding approximately to log base 100.
int size_class = floor(0.5 * log10(static_cast<double>(mc.size())));
*cumulative_bytes += mc.size();
printf(" %c %d %p bin=%d bytes=%" PRIu64 " %3.1f%%", mc.in_use() ? 'U' : 'F',
size_class, reinterpret_cast<const void*>(mc.address()), mc.bin(),
static_cast<uint64_t>(mc.size()),
100 * (*cumulative_bytes / static_cast<float>(total_bytes)));
if (freed_at) {
printf(" freed_at=%" PRIu64, static_cast<uint64_t>(mc.freed_at_count()));
}
if (ac_offset > 0) {
printf(" age=%" PRIu64,
static_cast<uint64_t>(ac_offset - mc.action_count()));
} else {
printf(" ac=%" PRIu64, static_cast<uint64_t>(mc.action_count()));
}
// step_ids are random, so save space by showing only low 16 bits.
printf(" step=%x op=%s\n", static_cast<uint>(0xFFFF & mc.step_id()),
mc.op_name().c_str());
}
void PrintSummary(const MemoryDump& md) {
printf("MemoryMap for allocator %s\n", md.allocator_name().c_str());
for (auto& it : md.bin_summary()) {
printf(" Bin %2d total bytes=%10" PRId64 " \tin use=%10" PRId64
" \ttotal_chunks=%6" PRId64
" "
"\tin_use=%6" PRId64 "\n",
it.bin(), static_cast<int64_t>(it.total_bytes_in_bin()),
static_cast<int64_t>(it.total_bytes_in_use()),
static_cast<int64_t>(it.total_chunks_in_bin()),
static_cast<int64_t>(it.total_chunks_in_use()));
}
printf("Total num_allocs: %" PRId64 ", bytes_in_use: %" PRId64
", peak_bytes_in_use: %" PRId64
",\n"
"largest_alloc_size: %" PRId64 ", fragmentation: %f\n",
static_cast<int64_t>(md.stats().num_allocs()),
static_cast<int64_t>(md.stats().bytes_in_use()),
static_cast<int64_t>(md.stats().peak_bytes_in_use()),
static_cast<int64_t>(md.stats().largest_alloc_size()),
md.stats().fragmentation_metric());
}
void PrintSortedChunks(
const MemoryDump& md,
std::function<bool(const MemChunk*, const MemChunk*)> compare, bool by_age,
bool freed_at, bool by_addr) {
std::vector<const MemChunk*> chunks;
chunks.reserve(md.chunk_size());
int64_t total_bytes = 0;
int64_t cumulative_bytes = 0;
uint64_t max_action_count = 0;
for (auto& it : md.chunk()) {
chunks.push_back(&it);
total_bytes += it.size();
if (by_age && it.action_count() > max_action_count) {
max_action_count = it.action_count();
}
}
sort(chunks.begin(), chunks.end(), compare);
uint64_t last_end = 0;
for (int i = 0; i < chunks.size(); ++i) {
const MemChunk* c = chunks[i];
if (by_addr && i > 0 && last_end != c->address()) {
printf(" empty range from %p to %p (%" PRId64 ")\n",
reinterpret_cast<const void*>(last_end),
reinterpret_cast<const void*>(c->address()),
static_cast<int64_t>(c->address() - last_end));
}
PrintChunk(*c, max_action_count, freed_at, total_bytes, &cumulative_bytes);
last_end = c->address() + c->size();
}
}
void PrintChunksByAddress(const MemoryDump& md, bool by_age, bool freed_at) {
printf("---------------Chunks by address:--------------------------\n");
PrintSortedChunks(
md,
[](const MemChunk* a, const MemChunk* b) {
return a->address() < b->address();
},
by_age, freed_at, true /*by_addr*/);
}
void PrintChunksByActionCount(const MemoryDump& md, bool by_age,
bool freed_at) {
printf("------------Chunks by last action count:----------------------\n");
PrintSortedChunks(
md,
[](const MemChunk* a, const MemChunk* b) {
return a->action_count() < b->action_count();
},
by_age, freed_at, false /*by_addr*/);
}
void PrintChunksBySize(const MemoryDump& md, bool by_age, bool freed_at) {
printf("------------Chunks by size:----------------------\n");
PrintSortedChunks(
md,
[](const MemChunk* a, const MemChunk* b) {
return a->size() > b->size();
},
by_age, freed_at, false /*by_addr*/);
}
void PrintChunksByOpName(const MemoryDump& md, const std::string& op_name,
bool by_age, bool freed_at) {
printf("------------Chunks matching \"%s\":----------------------\n",
op_name.c_str());
MemoryDump filtered;
uint64_t total_bytes = 0;
filtered.set_allocator_name(md.allocator_name());
for (const auto& it : md.bin_summary()) {
*filtered.add_bin_summary() = it;
}
for (const auto& it : md.chunk()) {
if (RE2::PartialMatch(it.op_name(), op_name)) {
*filtered.add_chunk() = it;
total_bytes += it.size();
}
}
printf("\t%d matching Chunks of total size %" PRIu64 " bytes:\n",
filtered.chunk_size(), static_cast<uint64_t>(total_bytes));
PrintSortedChunks(
filtered,
[](const MemChunk* a, const MemChunk* b) {
return a->address() > b->address();
},
by_age, freed_at, false /*by_addr*/);
}
void PrintSizeHistory(const MemoryDump& md, bool by_age) {
printf("------------Allocated Bytes by Action Count--------\n");
printf("num snapshots: %d\n", md.snap_shot_size());
uint64_t max_action_count = 0;
if (by_age) {
for (auto& it : md.snap_shot()) {
if (it.action_count() > max_action_count) {
max_action_count = it.action_count();
}
}
}
for (auto& it : md.snap_shot()) {
if (by_age) {
printf("\tage=%" PRIu64 ", size=%" PRId64 "\n",
static_cast<uint64_t>(max_action_count - it.action_count()),
static_cast<int64_t>(it.size()));
} else {
printf("\tac=%" PRIu64 ", size=%" PRId64 "\n",
static_cast<uint64_t>(it.action_count()),
static_cast<int64_t>(it.size()));
}
}
}
} // namespace tensorflow
int main(int argc, char** argv) {
std::string path = "";
bool by_addr = false;
bool by_ac = false;
bool by_size = false;
bool by_age = true;
bool freed_at = false;
bool size_history = false;
std::string chunk_type = "A";
std::string op_name = "";
std::vector<tensorflow::Flag> flag_list = {
tensorflow::Flag("path", &path,
"Path of GPU BFCAllocator memory dump file"),
tensorflow::Flag("by_addr", &by_addr,
"Whether to print Chunks by memory address"),
tensorflow::Flag("by_ac", &by_ac,
"Whether to print Chunks by action count"),
tensorflow::Flag("by_size", &by_size,
"Whether to print Chunks by decreasing size"),
tensorflow::Flag("by_age", &by_age,
"If true, replace absolute action count with age "
"(max_action_count - action_count) in display."),
tensorflow::Flag("op_name", &op_name,
"Whether to print only chunks whose Op name matches "
"the supplied regexp."),
tensorflow::Flag("freed_at", &freed_at,
"Whether to display the freed_at value (only relevant "
"with timestamped allocator)."),
tensorflow::Flag("chunk_type", &chunk_type,
"One of {'U', 'F', 'A'} meaning in Use, Free, or All "
"(default). Displays only Chunks of this type."),
tensorflow::Flag("size_history", &size_history,
"If true, show the size history."),
};
bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
if (!parse_result || path.empty()) {
std::cerr << tensorflow::Flags::Usage(argv[0], flag_list);
return -1;
}
tensorflow::port::InitMain(argv[0], &argc, &argv);
tensorflow::MemoryDump md = tensorflow::ReadDumpFile(path);
tensorflow::PrintSummary(md);
if (chunk_type != "A") {
md = FilterByChunkType(md, chunk_type[0]);
}
if (by_addr) tensorflow::PrintChunksByAddress(md, by_age, freed_at);
if (by_ac) tensorflow::PrintChunksByActionCount(md, by_age, freed_at);
if (by_size) tensorflow::PrintChunksBySize(md, by_age, freed_at);
if (!op_name.empty()) {
tensorflow::PrintChunksByOpName(md, op_name, by_age, freed_at);
}
if (size_history) tensorflow::PrintSizeHistory(md, by_age);
}
tensorflow/core/debug/debug.cc
+48
View File
@@ -0,0 +1,48 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <memory>
#include "tensorflow/core/common_runtime/debugger_state_interface.h"
#include "tensorflow/core/debug/debugger_state_impl.h"
namespace tensorflow {
namespace {
// Registers a concrete implementation of DebuggerState for use by
// DirectSession.
class DebuggerStateRegistration {
public:
static std::unique_ptr<DebuggerStateInterface> CreateDebuggerState(
const DebugOptions& options) {
return std::unique_ptr<DebuggerStateInterface>(new DebuggerState(options));
}
static std::unique_ptr<DebugGraphDecoratorInterface>
CreateDebugGraphDecorator(const DebugOptions& options) {
return std::unique_ptr<DebugGraphDecoratorInterface>(
new DebugGraphDecorator(options));
}
DebuggerStateRegistration() {
DebuggerStateRegistry::RegisterFactory(CreateDebuggerState);
DebugGraphDecoratorRegistry::RegisterFactory(CreateDebugGraphDecorator);
}
};
static DebuggerStateRegistration register_debugger_state_implementation;
} // end namespace
} // end namespace tensorflow
tensorflow/core/debug/debug_callback_registry.cc
+49
View File
@@ -0,0 +1,49 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/debug/debug_callback_registry.h"
namespace tensorflow {
DebugCallbackRegistry::DebugCallbackRegistry() {}
/*static */ DebugCallbackRegistry* DebugCallbackRegistry::instance_ = nullptr;
DebugCallbackRegistry* DebugCallbackRegistry::singleton() {
if (instance_ == nullptr) {
instance_ = new DebugCallbackRegistry();
}
return instance_;
}
void DebugCallbackRegistry::RegisterCallback(const std::string& key,
EventCallback callback) {
mutex_lock lock(mu_);
keyed_callback_[key] = std::move(callback);
}
DebugCallbackRegistry::EventCallback* DebugCallbackRegistry::GetCallback(
const std::string& key) {
mutex_lock lock(mu_);
auto iter = keyed_callback_.find(key);
return iter == keyed_callback_.end() ? nullptr : &iter->second;
}
void DebugCallbackRegistry::UnregisterCallback(const std::string& key) {
mutex_lock lock(mu_);
keyed_callback_.erase(key);
}
} // namespace tensorflow
tensorflow/core/debug/debug_callback_registry.h
+71
View File
@@ -0,0 +1,71 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_DEBUG_DEBUG_CALLBACK_REGISTRY_H_
#define TENSORFLOW_CORE_DEBUG_DEBUG_CALLBACK_REGISTRY_H_
#include <functional>
#include <map>
#include <string>
#include <vector>
#include "tensorflow/core/debug/debug_node_key.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/platform/mutex.h"
namespace tensorflow {
// Supports exporting observed debug events to clients using registered
// callbacks. Users can register a callback for each debug_url stored using
// DebugTensorWatch. The callback key be equivalent to what follows
// "memcbk:///".
//
// All events generated for a watched node will be sent to the call back in the
// order that they are observed.
//
// This callback router should not be used in production or training steps. It
// is optimized for deep inspection of graph state rather than performance.
class DebugCallbackRegistry {
public:
using EventCallback = std::function<void(const DebugNodeKey&, const Tensor&)>;
// Provides singleton access to the in memory event store.
static DebugCallbackRegistry* singleton();
// Returns the registered callback, or nullptr, for key.
EventCallback* GetCallback(const std::string& key);
// Associates callback with key. This must be called by clients observing
// nodes to be exported by this callback router before running a session.
void RegisterCallback(const std::string& key, EventCallback callback);
// Removes the callback associated with key.
void UnregisterCallback(const std::string& key);
private:
DebugCallbackRegistry();
// Mutex to ensure that keyed events are never updated in parallel.
mutex mu_;
// Maps debug_url keys to callbacks for routing observed tensors.
std::map<std::string, EventCallback> keyed_callback_ TF_GUARDED_BY(mu_);
static DebugCallbackRegistry* instance_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_DEBUG_DEBUG_CALLBACK_REGISTRY_H_
tensorflow/core/debug/debug_graph_utils.cc
+523
View File
@@ -0,0 +1,523 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/debug/debug_graph_utils.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "tensorflow/core/common_runtime/memory_types.h"
#include "tensorflow/core/framework/kernel_def.pb.h"
#include "tensorflow/core/framework/node_def_builder.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/graph/node_builder.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/protobuf/debug.pb.h"
namespace tensorflow {
namespace {
// TODO(cais): Switch to safe_strtob when available.
absl::Status ParseBoolString(const std::string& bool_str, bool* bool_val) {
const std::string lower_bool_str = absl::AsciiStrToLower(bool_str);
if (lower_bool_str == "false" || lower_bool_str == "f" ||
lower_bool_str == "0") {
*bool_val = false;
} else if (lower_bool_str == "true" || lower_bool_str == "t" ||
lower_bool_str == "1") {
*bool_val = true;
} else {
return absl::InvalidArgumentError(
absl::StrCat("Invalid string for bool value: ", bool_str));
}
return absl::OkStatus();
}
} // namespace
// static
absl::Status DebugNodeInserter::InsertNodes(
const protobuf::RepeatedPtrField<DebugTensorWatch>& watches, Graph* graph,
Device* device) {
// TODO(cais): This method is getting too large in size.
// Refactor it with helpers.
if (watches.empty()) {
// Nothing to do: Return OK right away.
return absl::OkStatus();
}
// Debug ops and URLs for wildcard node names (if any).
std::vector<std::string> default_debug_ops;
std::vector<std::string> default_debug_urls;
// A map from tensor name (e.g., "node_a:0") to list of debug op names
// (e.g., {"DebugIdentity", "DebugNanCount"})
std::unordered_map<std::string, std::vector<std::string>> tensor_watches;
// A map from tensor name to debug_url.
std::unordered_map<std::string, std::vector<std::string>> tensor_watch_urls;
std::unordered_map<std::string, bool> tensor_tolerate_failures;
// Cache the proto content for fast lookup later
for (const DebugTensorWatch& watch : watches) {
if (watch.debug_ops().empty()) {
continue;
}
if (watch.debug_urls().empty()) {
continue;
}
if (watch.node_name() == "*") {
if (watch.output_slot() == -1) {
default_debug_ops.insert(default_debug_ops.end(),
watch.debug_ops().begin(),
watch.debug_ops().end());
default_debug_urls.insert(default_debug_urls.end(),
watch.debug_urls().begin(),
watch.debug_urls().end());
} else {
return absl::Status(
absl::StatusCode::kFailedPrecondition,
absl::StrCat("output_slot is expected to be -1 for wildcard ",
"node name (\"*\"), but got ", watch.output_slot()));
}
continue;
} else {
if (watch.output_slot() < 0) {
return absl::Status(
absl::StatusCode::kFailedPrecondition,
absl::StrCat("A negative output_slot in DebugTensorWatch is ",
"valid only for the wildcard node name (\"*\"), ",
"but got node name ", watch.node_name()));
}
}
std::string tensor_name =
absl::StrCat(watch.node_name(), ":", watch.output_slot());
std::vector<std::string> debug_ops;
for (const std::string& debug_op : watch.debug_ops()) {
debug_ops.push_back(debug_op);
}
tensor_watches[tensor_name] = debug_ops;
tensor_tolerate_failures[tensor_name] =
watch.tolerate_debug_op_creation_failures();
std::vector<std::string> urls;
for (const std::string& url : watch.debug_urls()) {
urls.push_back(url);
}
tensor_watch_urls[tensor_name] = urls;
}
if (tensor_watches.empty()) {
return absl::OkStatus();
}
DeviceType device_type = DeviceType{device->device_type()};
// Keep track of all edges to be removed.
std::vector<const Edge*> edges_to_remove;
for (Node* src_node : graph->nodes()) {
// Make a map from output slot to outgoing edges from the slot.
std::unordered_map<int, std::vector<const Edge*>> output_slot_to_edges;
for (const Edge* edge : src_node->out_edges()) {
const int src_output = edge->src_output();
if (output_slot_to_edges.find(src_output) == output_slot_to_edges.end()) {
output_slot_to_edges[src_output] = {edge};
} else {
output_slot_to_edges[src_output].push_back(edge);
}
}
// Iterate through all output slots of the node.
for (int src_output_slot = 0; src_output_slot < src_node->num_outputs();
++src_output_slot) {
const std::string tensor_name =
absl::StrCat(src_node->name(), ":", src_output_slot);
const bool explicit_tensor_match =
tensor_watches.find(tensor_name) != tensor_watches.end();
if (!explicit_tensor_match && default_debug_ops.empty()) {
continue;
}
// Now we have encountered a watched tensor. We will:
// 1) Mark this edge as to be removed, iff this is a non-Reference
// tensor
// 2) Create a Copy node for the tensor
// 3) Add a new edge, from the source tensor to the Copy node
// 4) Add a new edge, from the Copy node to the destination node, iff
// this is a non-Reference tensor.
// 5) Create all the requested debug nodes and their edges to the Copy
// node.
// 6) Add control edges from the debug nodes to the destination nodes
// to ensure that the tensors values exported by the debug nodes
// to the debug URLs reflect the values before the execution of
// the destination nodes.
const DataType src_dt = src_node->output_type(src_output_slot);
MemoryType memory_type;
TF_RETURN_IF_ERROR(MemoryTypeForOutput(device_type, graph, src_node,
src_output_slot, &memory_type));
// Create the copy node for the watched tensor.
const std::vector<std::string> debug_ops =
explicit_tensor_match ? tensor_watches[tensor_name]
: default_debug_ops;
const std::vector<std::string> debug_urls =
explicit_tensor_match ? tensor_watch_urls[tensor_name]
: default_debug_urls;
Node* copy_node;
absl::Status copy_s =
CreateCopyNode(graph, device_type, memory_type == HOST_MEMORY,
src_node->name(), src_output_slot, src_dt, tensor_name,
debug_ops, debug_urls, &copy_node);
if (!copy_s.ok()) {
return absl::Status(
absl::StatusCode::kFailedPrecondition,
absl::StrCat("Failed to create Copy/CopyHost node for tensor ",
tensor_name, ", due to: ", copy_s.message()));
}
// Add edge from watched tensor to the copy node.
graph->AddEdge(src_node, src_output_slot, copy_node, 0);
// Create all requested debug nodes and their edges to the Copy node.
std::vector<Node*> debug_nodes;
for (size_t i = 0; i < debug_ops.size(); ++i) {
const std::string& debug_op_name = debug_ops[i];
Node* debug_node;
absl::Status debug_s = CreateDebugNode(
graph, *device, copy_node->name(), src_dt, tensor_name, debug_urls,
i, debug_op_name, &debug_node);
if (debug_s.ok()) {
graph->AddEdge(copy_node, 0, debug_node, 0);
debug_nodes.push_back(debug_node);
} else {
if (tensor_tolerate_failures[tensor_name]) {
LOG(INFO) << "Tolerating failure to create debug node: "
<< "tensor name = " << tensor_name << "; "
<< "debug op name = " << debug_op_name;
} else {
return absl::Status(
absl::StatusCode::kFailedPrecondition,
strings::StrCat("Failed to create debug node ", debug_op_name,
" for tensor ", tensor_name,
", due to: ", debug_s.message()));
}
}
}
// Is the output a reference?
const bool is_ref = IsRefType(src_node->output_type(src_output_slot));
// Iterate through all outgoing edges attached to the slot.
for (const Edge* edge : output_slot_to_edges[src_output_slot]) {
// Mark the edge for removal.
if (!is_ref) {
edges_to_remove.push_back(edge);
graph->AddEdge(copy_node, 0, edge->dst(), edge->dst_input());
}
// Add control edges from the debug nodes to the destination node
// to ensure that the debug nodes are executed before the destination
// node. Skip Enter and NextIteration ops to avoid hanging.
for (Node* debug_node : debug_nodes) {
if (!src_node->IsEnter() && !src_node->IsNextIteration()) {
graph->AddEdge(debug_node, Graph::kControlSlot, edge->dst(),
Graph::kControlSlot);
}
}
}
}
}
// Remove all edges marked for removal.
for (const Edge* edge : edges_to_remove) {
graph->RemoveEdge(edge);
}
return absl::OkStatus();
}
void DebugNodeInserter::DeparallelizeWhileLoops(Graph* graph, Device* device) {
bool deparallelized_a_loop = false;
for (Node* node : graph->nodes()) {
if (node->IsEnter()) {
const AttrValue* parallel_iterations =
node->attrs().Find("parallel_iterations");
if (parallel_iterations && parallel_iterations->i() > 1) {
deparallelized_a_loop = true;
VLOG(1) << "Changing the parallel_iterations attribute of the "
<< "Enter/RefEnter node \"" << node->name() << "\" on device \""
<< device->name() << "\" from " << parallel_iterations->i()
<< " to 1.";
node->AddAttr<int64_t>("parallel_iterations", 1);
}
}
}
if (deparallelized_a_loop) {
LOG(INFO) << "For debugging, tfdbg has set the parallel_iterations "
<< "attribute of all scheduled Enter/RefEnter nodes to 1. (This "
<< "does not affect subsequent non-debug runs.)";
}
}
// static
std::string DebugNodeInserter::GetCopyNodeName(const std::string& node_name,
const int output_slot) {
// For example, if the watched node is named "node1" and the output slot
// is 0, the debug node will be called: __copy_node1_0
return absl::StrCat("__copy_", node_name, "_", output_slot);
}
// static
std::string DebugNodeInserter::GetDebugNodeName(
const std::string& tensor_name, const int debug_op_num,
const std::string& debug_op_name) {
// For example, if the watched node is named "node1" and the debug op that
// watches the output slot of node1 is of the type "DebugNanCount", the
// debug node will be called: __dbg_node1_0_0_DebugNanCount.
return strings::StrCat("__dbg_", tensor_name, "_", debug_op_num, "_",
debug_op_name);
}
// static
absl::Status DebugNodeInserter::CreateCopyNode(
Graph* graph, const DeviceType device_type, const bool is_host_memory,
const std::string& src_node_name, const int src_output,
const DataType src_dt, const std::string& tensor_name,
const std::vector<std::string>& debug_ops,
const std::vector<std::string>& debug_urls, Node** copy_node) {
const std::string kGatedGrpcAttributeKey = "gated_grpc";
NodeDef node_def;
const KernelDef* kdef;
const std::string copy_op_name = is_host_memory ? "CopyHost" : "Copy";
const std::string copy_node_name = GetCopyNodeName(src_node_name, src_output);
// Cross debug_ops and debug_urls to get the list of debug ops and watches.
std::vector<std::string> debug_ops_spec;
for (const std::string& debug_op : debug_ops) {
for (const std::string& debug_url : debug_urls) {
std::string debug_op_name_proper;
std::unordered_map<std::string, std::string> custom_attributes;
TF_RETURN_IF_ERROR(ParseDebugOpName(debug_op, &debug_op_name_proper,
&custom_attributes));
bool gated_grpc_value = false;
if (custom_attributes.find(kGatedGrpcAttributeKey) !=
custom_attributes.end()) {
TF_RETURN_IF_ERROR(ParseBoolString(
custom_attributes[kGatedGrpcAttributeKey], &gated_grpc_value));
}
debug_ops_spec.push_back(strings::StrCat(debug_op_name_proper, ";",
debug_url, ";",
gated_grpc_value ? "1" : "0"));
}
}
auto builder = NodeDefBuilder(copy_node_name, copy_op_name)
.Input(src_node_name, src_output, src_dt)
.Attr("debug_ops_spec", debug_ops_spec);
if (!builder.Finalize(&node_def).ok()) {
return absl::Status(
absl::StatusCode::kFailedPrecondition,
strings::StrCat("Failed to create node definition ", "for copy op ",
copy_node_name, " on watched tensor ", tensor_name));
}
absl::Status s = FindKernelDef(device_type, node_def, &kdef, nullptr);
if (!s.ok()) {
return absl::Status(
absl::StatusCode::kFailedPrecondition,
strings::StrCat("Failed to find kernel definition ", "for copy op ",
copy_node_name, " on watched tensor ", tensor_name));
}
if (!NodeBuilder(builder).Finalize(graph, copy_node).ok()) {
return absl::Status(
absl::StatusCode::kFailedPrecondition,
absl::StrCat("Failed to create copy node ", copy_node_name,
" on watched tensor ", tensor_name));
}
return absl::OkStatus();
}
// static
absl::Status DebugNodeInserter::ParseDebugOpName(
const std::string& debug_op_name, std::string* debug_op_name_proper,
std::unordered_map<std::string, std::string>* attributes) {
const size_t l_index = debug_op_name.find('(');
const size_t r_index = debug_op_name.find(')');
if (l_index == std::string::npos && r_index == std::string::npos) {
*debug_op_name_proper = debug_op_name;
} else {
if (l_index == std::string::npos || l_index == 0 ||
r_index != debug_op_name.size() - 1) {
return absl::InvalidArgumentError(
absl::StrCat("Malformed debug op name \"", debug_op_name, "\""));
}
*debug_op_name_proper = debug_op_name.substr(0, l_index);
std::string arguments =
debug_op_name.substr(l_index + 1, r_index - l_index - 1);
std::vector<std::string> attribute_segs = str_util::Split(arguments, ";");
for (const std::string& attribute_seg : attribute_segs) {
absl::string_view seg(attribute_seg);
str_util::RemoveWhitespaceContext(&seg);
if (seg.empty()) {
continue;
}
const size_t eq_index = seg.find('=');
if (eq_index == std::string::npos) {
return absl::InvalidArgumentError(absl::StrCat(
"Malformed attributes in debug op name \"", debug_op_name, "\""));
}
const std::string key(seg.substr(0, eq_index));
const std::string value(
seg.substr(eq_index + 1, attribute_seg.size() - eq_index - 1));
if (key.empty() || value.empty()) {
return absl::InvalidArgumentError(absl::StrCat(
"Malformed attributes in debug op name \"", debug_op_name, "\""));
}
if (attributes->find(key) == attributes->end()) {
(*attributes)[key] = value;
} else {
return absl::InvalidArgumentError(
absl::StrCat("Duplicate attribute name \"", key,
"\" found in the debug op: \"", debug_op_name, "\""));
}
}
}
return absl::OkStatus();
}
// static
absl::Status DebugNodeInserter::SetDebugNodeAttributes(
Node* debug_node,
const std::unordered_map<std::string, std::string>& attributes) {
std::unordered_set<std::string> unfulfilled_keys;
for (const auto& item : attributes) {
unfulfilled_keys.insert(item.first);
}
for (const auto& attr : debug_node->op_def().attr()) {
if (attributes.find(attr.name()) != attributes.end()) {
const std::string& attr_value = attributes.at(attr.name());
if (attr.type() == "string") {
debug_node->AddAttr<std::string>(attr.name(), attr_value);
} else if (attr.type() == "float") {
float float_value = 0.0;
if (!absl::SimpleAtof(attr_value, &float_value)) {
return absl::InvalidArgumentError(absl::StrCat(
"Invalid value string for float-type attribute ", attr.name(),
"of debug node ", debug_node->name(), ": \"", attr_value, "\""));
}
debug_node->AddAttr<float>(attr.name(), float_value);
} else if (attr.type() == "int") {
int64_t int_value = 0;
if (!absl::SimpleAtoi(attr_value, &int_value)) {
return absl::InvalidArgumentError(absl::StrCat(
"Invalid value string for int-type attribute ", attr.name(),
"of debug node ", debug_node->name(), ": \"", attr_value, "\""));
}
debug_node->AddAttr<int>(attr.name(), int_value);
} else if (attr.type() == "bool") {
bool bool_value;
if (!ParseBoolString(attr_value, &bool_value).ok()) {
return absl::InvalidArgumentError(absl::StrCat(
"Invalid value string for bool-type attribute ", attr.name(),
"of debug node ", debug_node->name(), ": \"", attr_value, "\""));
}
debug_node->AddAttr<bool>(attr.name(), bool_value);
} else {
return absl::InvalidArgumentError(
absl::StrCat("Unsupported type of custom attribute for debug ops: ",
attr.type()));
}
unfulfilled_keys.erase(attr.name());
}
}
if (unfulfilled_keys.empty()) {
return absl::OkStatus();
} else {
return absl::InvalidArgumentError(absl::StrCat(
unfulfilled_keys.size(),
" attribute key(s) were not valid for debug node ", debug_node->name(),
": ", absl::StrJoin(unfulfilled_keys, ", ")));
}
}
// static
absl::Status DebugNodeInserter::CreateDebugNode(
Graph* graph, const Device& device, const std::string& src_copy_node_name,
const DataType src_dt, const std::string& tensor_name,
const std::vector<std::string>& debug_urls, const int debug_op_num,
const std::string& debug_op_name, Node** debug_node) {
NodeDef node_def;
const KernelDef* kdef;
std::string debug_op_name_proper;
std::unordered_map<std::string, std::string> custom_attributes;
TF_RETURN_IF_ERROR(ParseDebugOpName(debug_op_name, &debug_op_name_proper,
&custom_attributes));
const std::string debug_node_name =
GetDebugNodeName(tensor_name, debug_op_num, debug_op_name_proper);
auto builder = NodeDefBuilder(debug_node_name, debug_op_name_proper)
.Input(src_copy_node_name, 0, src_dt)
.Attr("device_name", device.name())
.Attr("tensor_name", tensor_name)
.Attr("debug_urls", debug_urls);
if (!builder.Finalize(&node_def).ok()) {
return absl::FailedPreconditionError(
absl::StrCat("Failed to create node definition for debug op ",
debug_op_name_proper, " on watched tensor ", tensor_name));
}
if (!FindKernelDef(DeviceType(device.device_type()), node_def, &kdef, nullptr)
.ok()) {
return absl::FailedPreconditionError(
absl::StrCat("Failed to find kernel definition for debug op ",
debug_op_name_proper, " on watched tensor ", tensor_name));
}
if (!NodeBuilder(builder).Finalize(graph, debug_node).ok()) {
return absl::FailedPreconditionError(
absl::StrCat("Failed to create debug node ", debug_op_name_proper,
" on watched tensor ", tensor_name));
}
// Set custom attributes (if any).
if (!custom_attributes.empty()) {
TF_RETURN_IF_ERROR(SetDebugNodeAttributes(*debug_node, custom_attributes));
}
return absl::OkStatus();
}
} // namespace tensorflow
tensorflow/core/debug/debug_graph_utils.h
+125
View File
@@ -0,0 +1,125 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_DEBUG_DEBUG_GRAPH_UTILS_H_
#define TENSORFLOW_CORE_DEBUG_DEBUG_GRAPH_UTILS_H_
#include <unordered_map>
#include <vector>
#include "tensorflow/core/common_runtime/debugger_state_interface.h"
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/protobuf/debug.pb.h"
namespace tensorflow {
class DebugNodeInserter {
public:
// EXPERIMENTAL: Insert special debug ops (e.g., DebugIdentity) to graph for
// debugging. Currently, such ops need to take exactly one input and has the
// string attribute "tensor_name" to indicate what tensor it watches.
// For example, before the node insertion, the graph may look like:
//
// A:0 -----------1----------> B
// |
// ---------2-----------> C
//
// wherein the output slot 0 of node A feeds as the input to nodes B through
// edge 1 and to node C through edge 2.
// After the node insertion, assuming both B and C have non-Ref input, the
// graph becomes:
// A:0 ---3---> Copy -----------4----------> B
// |
// ---------5--------> C
// |
// ---------6--------> X
//
// If a node (e.g., B) has Ref input, the graph becomes:
//
// --------------------------------> B
// |
// A:0 ---3-----> Copy -----------4----------> C
// |
// -----------5--------> X
//
// In other words, we do not feed Refs to deep-copies to downstream nodes.
//
// Copy is the inserted deep-copy node that copies the input tensor on-device
// (e.g., CPU-to-CPU or GPU-to-GPU deep copy) that reduces the likelihood of
// racy updates during the debug watches. X is the newly created debug node
// that transforms the input (copy of the watched tensor) into a debug signal.
//
// DebugIdentity is the simplest debugging paradigm, in which the debug signal
// (i.e., X:0) equals the tensor itself. More sophisticated debug ops can be
// used to transform the tensor into other debug signals. An example is the
// DebugNanCounter op.
//
// If the nodes (A, B and C) are located on GPU and the edges from A to B or C
// is HOST_MEMORY, then the CopyHost op will be used instead of the Copy op.
static absl::Status InsertNodes(
const protobuf::RepeatedPtrField<DebugTensorWatch>& watches, Graph* graph,
Device* device);
// Set the parallel_iterations attribute of TensorFlow while loops
// (specifically the nodes for which IsEnter() returns true) to 1 to prevent
// any node from being executed multiple times concurrently and
// generating temporally-overlapping debug Tensor dumps.
static void DeparallelizeWhileLoops(Graph* graph, Device* device);
// Get canonical name of a copy node.
static std::string GetCopyNodeName(const std::string& node_name,
int output_slot);
// Get canonical name of a debug node.
static std::string GetDebugNodeName(const std::string& tensor_name,
int debug_op_num,
const std::string& debug_op_name);
private:
static absl::Status CreateCopyNode(
Graph* graph, DeviceType device_type, bool is_host_memory,
const std::string& src_node_name, int src_output, DataType src_dt,
const std::string& tensor_name, const std::vector<std::string>& debug_ops,
const std::vector<std::string>& debug_urls, Node** copy_node);
// Parse the debug_op_name string to extract proper op name and attributes.
// debug_op_name can be the proper op name only, e.g., "DebugNumericSummary".
// It can also contain customizable keys and values. Each key-value pair is
// connected with an equal sign ("="). Multiple key-value pairs are separated
// with semicolons (";"), which optional whitespace in between, e.g.,
// "DebugNumericSummary(mute_if_healthy=true, lower_bound=-100.0)".
static absl::Status ParseDebugOpName(
const std::string& debug_op_name, std::string* debug_op_name_proper,
std::unordered_map<std::string, std::string>* attributes);
static absl::Status SetDebugNodeAttributes(
Node* debug_node,
const std::unordered_map<std::string, std::string>& attributes);
static absl::Status CreateDebugNode(
Graph* graph, const Device& device, const std::string& src_copy_node_name,
DataType src_dt, const std::string& tensor_name,
const std::vector<std::string>& debug_urls, int debug_op_num,
const std::string& debug_op_name, Node** debug_node);
// TODO(cais): Cut down the number of args to this method.
friend class DebugGraphUtilsTest;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_DEBUG_DEBUG_GRAPH_UTILS_H_
tensorflow/core/debug/debug_graph_utils_test.cc
+162
View File
@@ -0,0 +1,162 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/debug/debug_graph_utils.h"
#include "tensorflow/core/framework/tensor_testutil.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status_test_util.h"
#include "tensorflow/core/lib/strings/str_util.h"
namespace tensorflow {
class DebugGraphUtilsTest : public ::testing::Test {
protected:
absl::Status ParseDebugOpName(
const std::string& debug_op_name, std::string* debug_op_name_proper,
std::unordered_map<std::string, std::string>* attributes) {
return DebugNodeInserter::ParseDebugOpName(
debug_op_name, debug_op_name_proper, attributes);
}
};
TEST_F(DebugGraphUtilsTest, TestParseNoAttributeDebugOpName) {
std::string debug_op_name_proper;
std::unordered_map<std::string, std::string> attributes;
TF_ASSERT_OK(
ParseDebugOpName("DebugIdentity", &debug_op_name_proper, &attributes));
ASSERT_EQ("DebugIdentity", debug_op_name_proper);
ASSERT_EQ(0, attributes.size());
}
TEST_F(DebugGraphUtilsTest, TestMalformedDebugOpName) {
std::string debug_op_name_proper;
std::unordered_map<std::string, std::string> attributes;
absl::Status s = ParseDebugOpName("(mute_if_healthy=true)",
&debug_op_name_proper, &attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
s = ParseDebugOpName("DebugNumericSummary(", &debug_op_name_proper,
&attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
s = ParseDebugOpName("DebugNumericSummary)", &debug_op_name_proper,
&attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
}
TEST_F(DebugGraphUtilsTest, TestDebugOpNameWithMalformedAttributes) {
std::string debug_op_name_proper;
std::unordered_map<std::string, std::string> attributes;
absl::Status s = ParseDebugOpName("DebugNumericSummary(=)",
&debug_op_name_proper, &attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
s = ParseDebugOpName("DebugNumericSummary(mute_if_healthy=)",
&debug_op_name_proper, &attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
s = ParseDebugOpName("DebugNumericSummary(=true)", &debug_op_name_proper,
&attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
s = ParseDebugOpName("DebugNumericSummary(mute_if_healthy:true)",
&debug_op_name_proper, &attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
s = ParseDebugOpName("DebugNumericSummary(mute_if_healthy=true;threshold=)",
&debug_op_name_proper, &attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
s = ParseDebugOpName(
"DebugNumericSummary(mute_if_healthy=true;threshold:300.0)",
&debug_op_name_proper, &attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
}
TEST_F(DebugGraphUtilsTest, TestValidDebugOpNameWithSingleAttribute) {
std::string debug_op_name_proper;
std::unordered_map<std::string, std::string> attributes;
TF_ASSERT_OK(ParseDebugOpName("DebugNumericSummary()", &debug_op_name_proper,
&attributes));
ASSERT_EQ("DebugNumericSummary", debug_op_name_proper);
ASSERT_EQ(0, attributes.size());
attributes.clear();
TF_ASSERT_OK(ParseDebugOpName("DebugNumericSummary(mute_if_healthy=true)",
&debug_op_name_proper, &attributes));
ASSERT_EQ("DebugNumericSummary", debug_op_name_proper);
ASSERT_EQ(1, attributes.size());
ASSERT_EQ("true", attributes["mute_if_healthy"]);
}
TEST_F(DebugGraphUtilsTest, TestValidDebugOpNameWithMoreThanOneAttributes) {
std::string debug_op_name_proper;
std::unordered_map<std::string, std::string> attributes;
TF_ASSERT_OK(ParseDebugOpName(
"DebugNumericSummary(mute_if_healthy=true; threshold=300.0)",
&debug_op_name_proper, &attributes));
ASSERT_EQ("DebugNumericSummary", debug_op_name_proper);
ASSERT_EQ(2, attributes.size());
ASSERT_EQ("true", attributes["mute_if_healthy"]);
ASSERT_EQ("300.0", attributes["threshold"]);
attributes.clear();
TF_ASSERT_OK(ParseDebugOpName(
"DebugNumericSummary(mute_if_healthy=true;threshold=300.0;first_n=100)",
&debug_op_name_proper, &attributes));
ASSERT_EQ("DebugNumericSummary", debug_op_name_proper);
ASSERT_EQ(3, attributes.size());
ASSERT_EQ("true", attributes["mute_if_healthy"]);
ASSERT_EQ("300.0", attributes["threshold"]);
ASSERT_EQ("100", attributes["first_n"]);
}
TEST_F(DebugGraphUtilsTest, TestValidDebugOpNameWithMoreDuplicateAttributes) {
std::string debug_op_name_proper;
std::unordered_map<std::string, std::string> attributes;
absl::Status s = ParseDebugOpName(
"DebugNumericSummary(mute_if_healthy=true; lower_bound=3; "
"mute_if_healthy=false;)",
&debug_op_name_proper, &attributes);
ASSERT_TRUE(absl::IsInvalidArgument(s));
}
TEST_F(DebugGraphUtilsTest, TestValidDebugOpNameWithWhitespaceInAttributes) {
std::string debug_op_name_proper;
std::unordered_map<std::string, std::string> attributes;
TF_ASSERT_OK(ParseDebugOpName(
"DebugNumericSummary( mute_if_healthy=true; threshold=300.0 )",
&debug_op_name_proper, &attributes));
ASSERT_EQ("DebugNumericSummary", debug_op_name_proper);
ASSERT_EQ(2, attributes.size());
ASSERT_EQ("true", attributes["mute_if_healthy"]);
ASSERT_EQ("300.0", attributes["threshold"]);
attributes.clear();
TF_ASSERT_OK(ParseDebugOpName(
"DebugNumericSummary(;;mute_if_healthy=true; threshold=300.0;;)",
&debug_op_name_proper, &attributes));
ASSERT_EQ("DebugNumericSummary", debug_op_name_proper);
ASSERT_EQ(2, attributes.size());
ASSERT_EQ("true", attributes["mute_if_healthy"]);
ASSERT_EQ("300.0", attributes["threshold"]);
}
} // namespace tensorflow
tensorflow/core/debug/debug_grpc_io_utils_test.cc
+488
View File
@@ -0,0 +1,488 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <memory>
#include "absl/synchronization/notification.h"
#include "tensorflow/core/debug/debug_graph_utils.h"
#include "tensorflow/core/debug/debug_grpc_testlib.h"
#include "tensorflow/core/debug/debug_io_utils.h"
#include "tensorflow/core/lib/core/notification.h"
#include "tensorflow/core/lib/core/status_test_util.h"
#include "tensorflow/core/lib/core/threadpool.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/stringprintf.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/test.h"
namespace tensorflow {
class GrpcDebugTest : public ::testing::Test {
protected:
struct ServerData {
int port;
std::string url;
std::unique_ptr<test::TestEventListenerImpl> server;
std::unique_ptr<thread::ThreadPool> thread_pool;
};
void SetUp() override {
ClearEnabledWatchKeys();
SetUpInProcessServer(&server_data_, 0);
}
void TearDown() override { TearDownInProcessServer(&server_data_); }
void SetUpInProcessServer(ServerData* server_data,
int64_t server_start_delay_micros) {
server_data->port = testing::PickUnusedPortOrDie();
server_data->url = absl::StrCat("grpc://localhost:", server_data->port);
server_data->server = std::make_unique<test::TestEventListenerImpl>();
server_data->thread_pool =
std::make_unique<thread::ThreadPool>(Env::Default(), "test_server", 1);
server_data->thread_pool->Schedule(
[server_data, server_start_delay_micros]() {
Env::Default()->SleepForMicroseconds(server_start_delay_micros);
server_data->server->RunServer(server_data->port);
});
}
void TearDownInProcessServer(ServerData* server_data) {
server_data->server->StopServer();
server_data->thread_pool.reset();
}
void ClearEnabledWatchKeys() { DebugGrpcIO::ClearEnabledWatchKeys(); }
const int64_t GetChannelConnectionTimeoutMicros() {
return DebugGrpcIO::channel_connection_timeout_micros_;
}
void SetChannelConnectionTimeoutMicros(const int64_t timeout) {
DebugGrpcIO::channel_connection_timeout_micros_ = timeout;
}
ServerData server_data_;
};
TEST_F(GrpcDebugTest, ConnectionTimeoutWorks) {
// Use a short timeout so the test won't take too long.
const int64_t kOriginalTimeoutMicros = GetChannelConnectionTimeoutMicros();
const int64_t kShortTimeoutMicros = 500 * 1000;
SetChannelConnectionTimeoutMicros(kShortTimeoutMicros);
ASSERT_EQ(kShortTimeoutMicros, GetChannelConnectionTimeoutMicros());
const std::string& kInvalidGrpcUrl =
absl::StrCat("grpc://localhost:", testing::PickUnusedPortOrDie());
Tensor tensor(DT_FLOAT, TensorShape({1, 1}));
tensor.flat<float>()(0) = 42.0;
absl::Status publish_status = DebugIO::PublishDebugTensor(
DebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", "foo_tensor", 0,
"DebugIdentity"),
tensor, Env::Default()->NowMicros(), {kInvalidGrpcUrl});
SetChannelConnectionTimeoutMicros(kOriginalTimeoutMicros);
TF_ASSERT_OK(DebugIO::CloseDebugURL(kInvalidGrpcUrl));
ASSERT_FALSE(publish_status.ok());
const std::string expected_error_msg = strings::StrCat(
"Failed to connect to gRPC channel at ", kInvalidGrpcUrl.substr(7),
" within a timeout of ", kShortTimeoutMicros / 1e6, " s");
ASSERT_NE(std::string::npos,
publish_status.message().find(expected_error_msg));
}
TEST_F(GrpcDebugTest, ConnectionToDelayedStartingServerWorks) {
ServerData server_data;
// Server start will be delayed for 1 second.
SetUpInProcessServer(&server_data, 1 * 1000 * 1000);
Tensor tensor(DT_FLOAT, TensorShape({1, 1}));
tensor.flat<float>()(0) = 42.0;
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"foo_tensor", 0, "DebugIdentity");
absl::Status publish_status = DebugIO::PublishDebugTensor(
kDebugNodeKey, tensor, Env::Default()->NowMicros(), {server_data.url});
ASSERT_TRUE(publish_status.ok());
TF_ASSERT_OK(DebugIO::CloseDebugURL(server_data.url));
ASSERT_EQ(1, server_data.server->node_names.size());
ASSERT_EQ(1, server_data.server->output_slots.size());
ASSERT_EQ(1, server_data.server->debug_ops.size());
EXPECT_EQ(kDebugNodeKey.device_name, server_data.server->device_names[0]);
EXPECT_EQ(kDebugNodeKey.node_name, server_data.server->node_names[0]);
EXPECT_EQ(kDebugNodeKey.output_slot, server_data.server->output_slots[0]);
EXPECT_EQ(kDebugNodeKey.debug_op, server_data.server->debug_ops[0]);
TearDownInProcessServer(&server_data);
}
TEST_F(GrpcDebugTest, SendSingleDebugTensorViaGrpcTest) {
Tensor tensor(DT_FLOAT, TensorShape({1, 1}));
tensor.flat<float>()(0) = 42.0;
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"foo_tensor", 0, "DebugIdentity");
TF_ASSERT_OK(DebugIO::PublishDebugTensor(
kDebugNodeKey, tensor, Env::Default()->NowMicros(), {server_data_.url}));
TF_ASSERT_OK(DebugIO::CloseDebugURL(server_data_.url));
// Verify that the expected debug tensor sending happened.
ASSERT_EQ(1, server_data_.server->node_names.size());
ASSERT_EQ(1, server_data_.server->output_slots.size());
ASSERT_EQ(1, server_data_.server->debug_ops.size());
EXPECT_EQ(kDebugNodeKey.device_name, server_data_.server->device_names[0]);
EXPECT_EQ(kDebugNodeKey.node_name, server_data_.server->node_names[0]);
EXPECT_EQ(kDebugNodeKey.output_slot, server_data_.server->output_slots[0]);
EXPECT_EQ(kDebugNodeKey.debug_op, server_data_.server->debug_ops[0]);
}
TEST_F(GrpcDebugTest, SendDebugTensorWithLargeStringAtIndex0ViaGrpcTest) {
Tensor tensor(DT_STRING, TensorShape({1, 1}));
tensor.flat<tstring>()(0) = std::string(5000 * 1024, 'A');
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"foo_tensor", 0, "DebugIdentity");
const absl::Status status = DebugIO::PublishDebugTensor(
kDebugNodeKey, tensor, Env::Default()->NowMicros(), {server_data_.url});
ASSERT_FALSE(status.ok());
ASSERT_NE(status.message().find("string value at index 0 from debug "
"node foo_tensor:0:DebugIdentity does "
"not fit gRPC message size limit"),
std::string::npos);
TF_ASSERT_OK(DebugIO::CloseDebugURL(server_data_.url));
}
TEST_F(GrpcDebugTest, SendDebugTensorWithLargeStringAtIndex1ViaGrpcTest) {
Tensor tensor(DT_STRING, TensorShape({1, 2}));
tensor.flat<tstring>()(0) = "A";
tensor.flat<tstring>()(1) = std::string(5000 * 1024, 'A');
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"foo_tensor", 0, "DebugIdentity");
const absl::Status status = DebugIO::PublishDebugTensor(
kDebugNodeKey, tensor, Env::Default()->NowMicros(), {server_data_.url});
ASSERT_FALSE(status.ok());
ASSERT_NE(status.message().find("string value at index 1 from debug "
"node foo_tensor:0:DebugIdentity does "
"not fit gRPC message size limit"),
std::string::npos);
TF_ASSERT_OK(DebugIO::CloseDebugURL(server_data_.url));
}
TEST_F(GrpcDebugTest, SendMultipleDebugTensorsSynchronizedViaGrpcTest) {
const int32_t kSends = 4;
// Prepare the tensors to sent.
std::vector<Tensor> tensors;
for (int i = 0; i < kSends; ++i) {
Tensor tensor(DT_INT32, TensorShape({1, 1}));
tensor.flat<int>()(0) = i * i;
tensors.push_back(tensor);
}
thread::ThreadPool* tp =
new thread::ThreadPool(Env::Default(), "grpc_debug_test", kSends);
mutex mu;
absl::Notification all_done;
int tensor_count TF_GUARDED_BY(mu) = 0;
std::vector<absl::Status> statuses TF_GUARDED_BY(mu);
const std::vector<std::string> urls({server_data_.url});
// Set up the concurrent tasks of sending Tensors via an Event stream to the
// server.
auto fn = [this, &mu, &tensor_count, &tensors, &statuses, &all_done,
&urls]() {
int this_count;
{
mutex_lock l(mu);
this_count = tensor_count++;
}
// Different concurrent tasks will send different tensors.
const uint64_t wall_time = Env::Default()->NowMicros();
absl::Status publish_status = DebugIO::PublishDebugTensor(
DebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
absl::StrCat("synchronized_node_", this_count), 0,
"DebugIdentity"),
tensors[this_count], wall_time, urls);
{
mutex_lock l(mu);
statuses.push_back(publish_status);
if (this_count == kSends - 1 && !all_done.HasBeenNotified()) {
all_done.Notify();
}
}
};
// Schedule the concurrent tasks.
for (int i = 0; i < kSends; ++i) {
tp->Schedule(fn);
}
// Wait for all client tasks to finish.
all_done.WaitForNotification();
delete tp;
// Close the debug gRPC stream.
absl::Status close_status = DebugIO::CloseDebugURL(server_data_.url);
ASSERT_TRUE(close_status.ok());
// Check all statuses from the PublishDebugTensor calls().
for (const absl::Status& status : statuses) {
TF_ASSERT_OK(status);
}
// One prep tensor plus kSends concurrent tensors are expected.
ASSERT_EQ(kSends, server_data_.server->node_names.size());
for (size_t i = 0; i < server_data_.server->node_names.size(); ++i) {
std::vector<std::string> items =
str_util::Split(server_data_.server->node_names[i], '_');
int tensor_index;
strings::safe_strto32(items[2], &tensor_index);
ASSERT_EQ(TensorShape({1, 1}),
server_data_.server->debug_tensors[i].shape());
ASSERT_EQ(tensor_index * tensor_index,
server_data_.server->debug_tensors[i].flat<int>()(0));
}
}
TEST_F(GrpcDebugTest, SendDebugTensorsThroughMultipleRoundsUsingGrpcGating) {
// Prepare the tensor to send.
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"test_namescope/test_node", 0,
"DebugIdentity");
Tensor tensor(DT_INT32, TensorShape({1, 1}));
tensor.flat<int>()(0) = 42;
const std::vector<std::string> urls({server_data_.url});
for (int i = 0; i < 3; ++i) {
server_data_.server->ClearReceivedDebugData();
const uint64_t wall_time = Env::Default()->NowMicros();
// On the 1st send (i == 0), gating is disabled, so data should be sent.
// On the 2nd send (i == 1), gating is enabled, and the server has enabled
// the watch key in the previous send, so data should be sent.
// On the 3rd send (i == 2), gating is enabled, but the server has disabled
// the watch key in the previous send, so data should not be sent.
const bool enable_gated_grpc = (i != 0);
TF_ASSERT_OK(DebugIO::PublishDebugTensor(kDebugNodeKey, tensor, wall_time,
urls, enable_gated_grpc));
server_data_.server->RequestDebugOpStateChangeAtNextStream(
i == 0 ? EventReply::DebugOpStateChange::READ_ONLY
: EventReply::DebugOpStateChange::DISABLED,
kDebugNodeKey);
// Close the debug gRPC stream.
absl::Status close_status = DebugIO::CloseDebugURL(server_data_.url);
ASSERT_TRUE(close_status.ok());
// Check dumped files according to the expected gating results.
if (i < 2) {
ASSERT_EQ(1, server_data_.server->node_names.size());
ASSERT_EQ(1, server_data_.server->output_slots.size());
ASSERT_EQ(1, server_data_.server->debug_ops.size());
EXPECT_EQ(kDebugNodeKey.device_name,
server_data_.server->device_names[0]);
EXPECT_EQ(kDebugNodeKey.node_name, server_data_.server->node_names[0]);
EXPECT_EQ(kDebugNodeKey.output_slot,
server_data_.server->output_slots[0]);
EXPECT_EQ(kDebugNodeKey.debug_op, server_data_.server->debug_ops[0]);
} else {
ASSERT_EQ(0, server_data_.server->node_names.size());
}
}
}
TEST_F(GrpcDebugTest, SendDebugTensorsThroughMultipleRoundsUnderReadWriteMode) {
// Prepare the tensor to send.
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"test_namescope/test_node", 0,
"DebugIdentity");
Tensor tensor(DT_INT32, TensorShape({1, 1}));
tensor.flat<int>()(0) = 42;
const std::vector<std::string> urls({server_data_.url});
for (int i = 0; i < 3; ++i) {
server_data_.server->ClearReceivedDebugData();
const uint64_t wall_time = Env::Default()->NowMicros();
// On the 1st send (i == 0), gating is disabled, so data should be sent.
// On the 2nd send (i == 1), gating is enabled, and the server has enabled
// the watch key in the previous send (READ_WRITE), so data should be
// sent. In this iteration, the server response with a EventReply proto to
// unblock the debug node.
// On the 3rd send (i == 2), gating is enabled, but the server has disabled
// the watch key in the previous send, so data should not be sent.
const bool enable_gated_grpc = (i != 0);
TF_ASSERT_OK(DebugIO::PublishDebugTensor(kDebugNodeKey, tensor, wall_time,
urls, enable_gated_grpc));
server_data_.server->RequestDebugOpStateChangeAtNextStream(
i == 0 ? EventReply::DebugOpStateChange::READ_WRITE
: EventReply::DebugOpStateChange::DISABLED,
kDebugNodeKey);
// Close the debug gRPC stream.
absl::Status close_status = DebugIO::CloseDebugURL(server_data_.url);
ASSERT_TRUE(close_status.ok());
// Check dumped files according to the expected gating results.
if (i < 2) {
ASSERT_EQ(1, server_data_.server->node_names.size());
ASSERT_EQ(1, server_data_.server->output_slots.size());
ASSERT_EQ(1, server_data_.server->debug_ops.size());
EXPECT_EQ(kDebugNodeKey.device_name,
server_data_.server->device_names[0]);
EXPECT_EQ(kDebugNodeKey.node_name, server_data_.server->node_names[0]);
EXPECT_EQ(kDebugNodeKey.output_slot,
server_data_.server->output_slots[0]);
EXPECT_EQ(kDebugNodeKey.debug_op, server_data_.server->debug_ops[0]);
} else {
ASSERT_EQ(0, server_data_.server->node_names.size());
}
}
}
TEST_F(GrpcDebugTest, TestGateDebugNodeOnEmptyEnabledSet) {
ASSERT_FALSE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity",
{"grpc://localhost:3333"}));
// file:// debug URLs are not subject to grpc gating.
ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen(
"foo:0:DebugIdentity", {"grpc://localhost:3333", "file:///tmp/tfdbg_1"}));
}
TEST_F(GrpcDebugTest, TestGateDebugNodeOnNonEmptyEnabledSet) {
const std::string kGrpcUrl1 = "grpc://localhost:3333";
const std::string kGrpcUrl2 = "grpc://localhost:3334";
DebugGrpcIO::SetDebugNodeKeyGrpcState(
kGrpcUrl1, "foo:0:DebugIdentity",
EventReply::DebugOpStateChange::READ_ONLY);
DebugGrpcIO::SetDebugNodeKeyGrpcState(
kGrpcUrl1, "bar:0:DebugIdentity",
EventReply::DebugOpStateChange::READ_ONLY);
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("foo:1:DebugIdentity", {kGrpcUrl1}));
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("foo:1:DebugNumericSummary", {kGrpcUrl1}));
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("qux:0:DebugIdentity", {kGrpcUrl1}));
ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl1}));
ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl1}));
// Wrong grpc:// debug URLs.
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl2}));
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl2}));
// file:// debug URLs are not subject to grpc gating.
ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("qux:0:DebugIdentity",
{"file:///tmp/tfdbg_1", kGrpcUrl1}));
}
TEST_F(GrpcDebugTest, TestGateDebugNodeOnMultipleEmptyEnabledSets) {
const std::string kGrpcUrl1 = "grpc://localhost:3333";
const std::string kGrpcUrl2 = "grpc://localhost:3334";
const std::string kGrpcUrl3 = "grpc://localhost:3335";
DebugGrpcIO::SetDebugNodeKeyGrpcState(
kGrpcUrl1, "foo:0:DebugIdentity",
EventReply::DebugOpStateChange::READ_ONLY);
DebugGrpcIO::SetDebugNodeKeyGrpcState(
kGrpcUrl2, "bar:0:DebugIdentity",
EventReply::DebugOpStateChange::READ_ONLY);
ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl1}));
ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl2}));
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl2}));
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl1}));
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl3}));
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl3}));
ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity",
{kGrpcUrl1, kGrpcUrl2}));
ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity",
{kGrpcUrl1, kGrpcUrl2}));
ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity",
{kGrpcUrl1, kGrpcUrl3}));
ASSERT_FALSE(DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity",
{kGrpcUrl1, kGrpcUrl3}));
}
TEST_F(GrpcDebugTest, TestGateDebugNodeOnNonEmptyEnabledSetAndEmptyURLs) {
DebugGrpcIO::SetDebugNodeKeyGrpcState(
"grpc://localhost:3333", "foo:0:DebugIdentity",
EventReply::DebugOpStateChange::READ_ONLY);
std::vector<std::string> debug_urls_1;
ASSERT_FALSE(
DebugIO::IsDebugNodeGateOpen("foo:1:DebugIdentity", debug_urls_1));
}
TEST_F(GrpcDebugTest, TestGateCopyNodeOnEmptyEnabledSet) {
const std::string kGrpcUrl1 = "grpc://localhost:3333";
const std::string kWatch1 = "foo:0:DebugIdentity";
ASSERT_FALSE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, true)}));
ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, false)}));
// file:// debug URLs are not subject to grpc gating.
ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec("foo:0:DebugIdentity", kGrpcUrl1, true),
DebugWatchAndURLSpec("foo:0:DebugIdentity", "file:///tmp/tfdbg_1",
false)}));
}
TEST_F(GrpcDebugTest, TestGateCopyNodeOnNonEmptyEnabledSet) {
const std::string kGrpcUrl1 = "grpc://localhost:3333";
const std::string kGrpcUrl2 = "grpc://localhost:3334";
const std::string kWatch1 = "foo:0:DebugIdentity";
const std::string kWatch2 = "foo:1:DebugIdentity";
DebugGrpcIO::SetDebugNodeKeyGrpcState(
kGrpcUrl1, kWatch1, EventReply::DebugOpStateChange::READ_ONLY);
ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, true)}));
ASSERT_FALSE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec(kWatch1, kGrpcUrl2, true)}));
ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec(kWatch1, kGrpcUrl2, false)}));
ASSERT_FALSE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec(kWatch2, kGrpcUrl1, true)}));
ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec(kWatch2, kGrpcUrl1, false)}));
ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, true),
DebugWatchAndURLSpec(kWatch1, kGrpcUrl2, true)}));
ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen(
{DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, true),
DebugWatchAndURLSpec(kWatch2, kGrpcUrl2, true)}));
}
} // namespace tensorflow
tensorflow/core/debug/debug_grpc_testlib.cc
+191
View File
@@ -0,0 +1,191 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/debug/debug_grpc_testlib.h"
#include "tensorflow/core/debug/debug_graph_utils.h"
#include "tensorflow/core/debug/debugger_event_metadata.pb.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/framework/tensor.pb.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/protobuf.h"
namespace tensorflow {
namespace test {
::grpc::Status TestEventListenerImpl::SendEvents(
::grpc::ServerContext* context,
::grpc::ServerReaderWriter<::tensorflow::EventReply, ::tensorflow::Event>*
stream) {
Event event;
while (stream->Read(&event)) {
if (event.has_log_message()) {
debug_metadata_strings.push_back(event.log_message().message());
stream->Write(EventReply());
} else if (!event.graph_def().empty()) {
encoded_graph_defs.push_back(event.graph_def());
stream->Write(EventReply());
} else if (event.has_summary()) {
const Summary::Value& val = event.summary().value(0);
std::vector<std::string> name_items =
tensorflow::str_util::Split(val.node_name(), ':');
const std::string node_name = name_items[0];
const std::string debug_op = name_items[2];
const TensorProto& tensor_proto = val.tensor();
Tensor tensor(tensor_proto.dtype());
if (!tensor.FromProto(tensor_proto)) {
return ::grpc::Status::CANCELLED;
}
// Obtain the device name, which is encoded in JSON.
third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
if (val.metadata().plugin_data().plugin_name() != "debugger") {
// This plugin data was meant for another plugin.
continue;
}
auto status = tensorflow::protobuf::util::JsonStringToMessage(
val.metadata().plugin_data().content(), &metadata);
if (!status.ok()) {
// The device name could not be determined.
continue;
}
device_names.push_back(metadata.device());
node_names.push_back(node_name);
output_slots.push_back(metadata.output_slot());
debug_ops.push_back(debug_op);
debug_tensors.push_back(tensor);
// If the debug node is currently in the READ_WRITE mode, send an
// EventReply to 1) unblock the execution and 2) optionally modify the
// value.
const DebugNodeKey debug_node_key(metadata.device(), node_name,
metadata.output_slot(), debug_op);
if (write_enabled_debug_node_keys_.find(debug_node_key) !=
write_enabled_debug_node_keys_.end()) {
stream->Write(EventReply());
}
}
}
{
mutex_lock l(states_mu_);
for (size_t i = 0; i < new_states_.size(); ++i) {
EventReply event_reply;
EventReply::DebugOpStateChange* change =
event_reply.add_debug_op_state_changes();
// State changes will take effect in the next stream, i.e., next debugged
// Session.run() call.
change->set_state(new_states_[i]);
const DebugNodeKey& debug_node_key = debug_node_keys_[i];
change->set_node_name(debug_node_key.node_name);
change->set_output_slot(debug_node_key.output_slot);
change->set_debug_op(debug_node_key.debug_op);
stream->Write(event_reply);
if (new_states_[i] == EventReply::DebugOpStateChange::READ_WRITE) {
write_enabled_debug_node_keys_.insert(debug_node_key);
} else {
write_enabled_debug_node_keys_.erase(debug_node_key);
}
}
debug_node_keys_.clear();
new_states_.clear();
}
return ::grpc::Status::OK;
}
void TestEventListenerImpl::ClearReceivedDebugData() {
debug_metadata_strings.clear();
encoded_graph_defs.clear();
device_names.clear();
node_names.clear();
output_slots.clear();
debug_ops.clear();
debug_tensors.clear();
}
void TestEventListenerImpl::RequestDebugOpStateChangeAtNextStream(
const EventReply::DebugOpStateChange::State new_state,
const DebugNodeKey& debug_node_key) {
mutex_lock l(states_mu_);
debug_node_keys_.push_back(debug_node_key);
new_states_.push_back(new_state);
}
void TestEventListenerImpl::RunServer(const int server_port) {
::grpc::ServerBuilder builder;
builder.AddListeningPort(absl::StrCat("localhost:", server_port),
::grpc::InsecureServerCredentials());
builder.RegisterService(this);
std::unique_ptr<::grpc::Server> server = builder.BuildAndStart();
while (!stop_requested_.load()) {
Env::Default()->SleepForMicroseconds(200 * 1000);
}
server->Shutdown();
stopped_.store(true);
}
void TestEventListenerImpl::StopServer() {
stop_requested_.store(true);
while (!stopped_.load()) {
}
}
bool PollTillFirstRequestSucceeds(const std::string& server_url,
const size_t max_attempts) {
const int kSleepDurationMicros = 100 * 1000;
size_t n_attempts = 0;
bool success = false;
// Try a number of times to send the Event proto to the server, as it may
// take the server a few seconds to start up and become responsive.
Tensor prep_tensor(DT_FLOAT, TensorShape({1, 1}));
prep_tensor.flat<float>()(0) = 42.0f;
while (n_attempts++ < max_attempts) {
const uint64_t wall_time = Env::Default()->NowMicros();
absl::Status publish_s = DebugIO::PublishDebugTensor(
DebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", "prep_node", 0,
"DebugIdentity"),
prep_tensor, wall_time, {server_url});
absl::Status close_s = DebugIO::CloseDebugURL(server_url);
if (publish_s.ok() && close_s.ok()) {
success = true;
break;
} else {
Env::Default()->SleepForMicroseconds(kSleepDurationMicros);
}
}
return success;
}
} // namespace test
} // namespace tensorflow
tensorflow/core/debug/debug_grpc_testlib.h
+87
View File
@@ -0,0 +1,87 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_DEBUG_DEBUG_GRPC_TESTLIB_H_
#define TENSORFLOW_CORE_DEBUG_DEBUG_GRPC_TESTLIB_H_
#include <atomic>
#include <unordered_set>
#include "grpcpp/grpcpp.h"
#include "tensorflow/core/debug/debug_io_utils.h"
#include "tensorflow/core/debug/debug_service.grpc.pb.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/platform/mutex.h"
namespace tensorflow {
namespace test {
class TestEventListenerImpl final : public grpc::EventListener::Service {
public:
TestEventListenerImpl() : stop_requested_(false), stopped_(false) {}
void RunServer(const int server_port);
void StopServer();
::grpc::Status SendEvents(
::grpc::ServerContext* context,
::grpc::ServerReaderWriter< ::tensorflow::EventReply,
::tensorflow::Event>* stream) override;
// Clear debug data (e.g., Tensors) received so far.
void ClearReceivedDebugData();
void RequestDebugOpStateChangeAtNextStream(
const EventReply::DebugOpStateChange::State new_state,
const DebugNodeKey& debug_node_key);
std::vector<std::string> debug_metadata_strings;
std::vector<std::string> encoded_graph_defs;
std::vector<std::string> device_names;
std::vector<std::string> node_names;
std::vector<int32_t> output_slots;
std::vector<std::string> debug_ops;
std::vector<Tensor> debug_tensors;
private:
std::atomic_bool stop_requested_;
std::atomic_bool stopped_;
std::vector<DebugNodeKey> debug_node_keys_ TF_GUARDED_BY(states_mu_);
std::vector<EventReply::DebugOpStateChange::State> new_states_
TF_GUARDED_BY(states_mu_);
std::unordered_set<DebugNodeKey> write_enabled_debug_node_keys_;
mutex states_mu_;
};
// Poll a gRPC debug server by sending a small tensor repeatedly till success.
//
// Args:
// server_url: gRPC URL of the server to poll, e.g., "grpc://foo:3333".
// max_attempts: Maximum number of attempts.
//
// Returns:
// Whether the polling succeeded within max_attempts.
bool PollTillFirstRequestSucceeds(const std::string& server_url,
const size_t max_attempts);
} // namespace test
} // namespace tensorflow
#endif // TENSORFLOW_CORE_DEBUG_DEBUG_GRPC_TESTLIB_H_
tensorflow/core/debug/debug_io_utils.cc
+1014
View File
File diff suppressed because it is too large Load Diff
tensorflow/core/debug/debug_io_utils.h
+450
View File
@@ -0,0 +1,450 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_DEBUG_DEBUG_IO_UTILS_H_
#define TENSORFLOW_CORE_DEBUG_DEBUG_IO_UTILS_H_
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "tensorflow/core/debug/debug_node_key.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/gtl/array_slice.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/util/event.pb.h"
namespace tensorflow {
absl::Status ReadEventFromFile(const std::string& dump_file_path, Event* event);
struct DebugWatchAndURLSpec {
DebugWatchAndURLSpec(const std::string& watch_key, const std::string& url,
const bool gated_grpc)
: watch_key(watch_key), url(url), gated_grpc(gated_grpc) {}
const std::string watch_key;
const std::string url;
const bool gated_grpc;
};
// TODO(cais): Put static functions and members in a namespace, not a class.
class DebugIO {
public:
static const char* const kDebuggerPluginName;
static const char* const kCoreMetadataTag;
static const char* const kGraphTag;
static const char* const kHashTag;
static const char* const kFileURLScheme;
static const char* const kGrpcURLScheme;
static const char* const kMemoryURLScheme;
static absl::Status PublishDebugMetadata(
const int64_t global_step, const int64_t session_run_index,
const int64_t executor_step_index,
const std::vector<std::string>& input_names,
const std::vector<std::string>& output_names,
const std::vector<std::string>& target_nodes,
const std::unordered_set<std::string>& debug_urls);
// Publishes a tensor to a debug target URL.
//
// Args:
// debug_node_key: A DebugNodeKey identifying the debug node. If
// `debug_node_key.io_of_node` is non-empty, publish for node
// inputs/outputs dumping feature.
// tensor: The Tensor object being published.
// wall_time_us: Time stamp for the Tensor. Unit: microseconds (us).
// debug_urls: An array of debug target URLs, e.g.,
// "file:///foo/tfdbg_dump", "grpc://localhost:11011"
// gated_grpc: Whether this call is subject to gRPC gating.
// step_id: Step ID associated with the tensor.
static absl::Status PublishDebugTensor(
const DebugNodeKey& debug_node_key, const Tensor& tensor,
const uint64_t wall_time_us,
const absl::Span<const std::string> debug_urls, bool gated_grpc,
int64_t step_id = -1);
// Convenience overload of the method above for no gated_grpc by default.
static absl::Status PublishDebugTensor(
const DebugNodeKey& debug_node_key, const Tensor& tensor,
const uint64_t wall_time_us,
const absl::Span<const std::string> debug_urls);
// Publishes a graph to a set of debug URLs.
//
// Args:
// graph: The graph to be published.
// debug_urls: The set of debug URLs to publish the graph to.
static absl::Status PublishGraph(
const Graph& graph, const std::string& device_name,
const std::unordered_set<std::string>& debug_urls);
// Determines whether a copy node needs to perform deep-copy of input tensor.
//
// The input arguments contain sufficient information about the attached
// downstream debug ops for this method to determine whether all the said
// ops are disabled given the current status of the gRPC gating.
//
// Args:
// specs: A vector of DebugWatchAndURLSpec carrying information about the
// debug ops attached to the Copy node, their debug URLs and whether
// they have the attribute value gated_grpc == True.
//
// Returns:
// Whether any of the attached downstream debug ops is enabled given the
// current status of the gRPC gating.
static bool IsCopyNodeGateOpen(
const std::vector<DebugWatchAndURLSpec>& specs);
// Determines whether a debug node needs to proceed given the current gRPC
// gating status.
//
// Args:
// watch_key: debug tensor watch key, in the format of
// tensor_name:debug_op, e.g., "Weights:0:DebugIdentity".
// debug_urls: the debug URLs of the debug node.
//
// Returns:
// Whether this debug op should proceed.
static bool IsDebugNodeGateOpen(const std::string& watch_key,
const std::vector<std::string>& debug_urls);
// Determines whether debug information should be sent through a grpc://
// debug URL given the current gRPC gating status.
//
// Args:
// watch_key: debug tensor watch key, in the format of
// tensor_name:debug_op, e.g., "Weights:0:DebugIdentity".
// debug_url: the debug URL, e.g., "grpc://localhost:3333",
// "file:///tmp/tfdbg_1".
//
// Returns:
// Whether the sending of debug data to the debug_url should
// proceed.
static bool IsDebugURLGateOpen(const std::string& watch_key,
const std::string& debug_url);
static absl::Status CloseDebugURL(const std::string& debug_url);
};
// Helper class for debug ops.
class DebugFileIO {
public:
// Encapsulates the Tensor in an Event protobuf and write it to a directory.
// The actual path of the dump file will be a contactenation of
// dump_root_dir, tensor_name, along with the wall_time.
//
// For example:
// let dump_root_dir = "/tmp/tfdbg_dump",
// node_name = "foo/bar",
// output_slot = 0,
// debug_op = DebugIdentity,
// and wall_time_us = 1467891234512345,
// the dump file will be generated at path:
// /tmp/tfdbg_dump/foo/bar_0_DebugIdentity_1467891234512345.
//
// Args:
// debug_node_key: A DebugNodeKey identifying the debug node.
// wall_time_us: Wall time at which the Tensor is generated during graph
// execution. Unit: microseconds (us).
// dump_root_dir: Root directory for dumping the tensor.
// dump_file_path: The actual dump file path (passed as reference).
static absl::Status DumpTensorToDir(const DebugNodeKey& debug_node_key,
const Tensor& tensor,
const uint64_t wall_time_us,
const std::string& dump_root_dir,
std::string* dump_file_path);
// Similar to the above, but for node inputs/outputs dumping feature.
static absl::Status DumpTensorToDirForNodeDumping(
const DebugNodeKey& debug_node_key, const Tensor& tensor,
uint64_t wall_time_us, const std::string& dump_root_dir,
std::string* dump_file_path, int64_t step_id);
// Get the full path to the dump file.
//
// Args:
// dump_root_dir: The dump root directory, e.g., /tmp/tfdbg_dump
// node_name: Name of the node from which the dumped tensor is generated,
// e.g., foo/bar/node_a
// output_slot: Output slot index of the said node, e.g., 0.
// debug_op: Name of the debug op, e.g., DebugIdentity.
// wall_time_us: Time stamp of the dumped tensor, in microseconds (us).
static std::string GetDumpFilePath(const std::string& dump_root_dir,
const DebugNodeKey& debug_node_key,
const uint64_t wall_time_us);
// Similar to the above, but for node inputs/outputs dumping feature.
static std::string GetDumpFilePathForNodeDumping(
const std::string& dump_root_dir, const DebugNodeKey& debug_node_key,
uint64_t wall_time_us, int64_t step_id);
// Dumps an Event proto to a file.
//
// Args:
// event_prot: The Event proto to be dumped.
// dir_name: Directory path.
// file_name: Base file name.
static absl::Status DumpEventProtoToFile(const Event& event_proto,
const std::string& dir_name,
const std::string& file_name);
// Request additional bytes to be dumped to the file system.
//
// Does not actually dump the bytes, but instead just performs the
// bookkeeping necessary to prevent the total dumped amount of data from
// exceeding the limit (default 100 GBytes or set customly through the
// environment variable TFDBG_DISK_BYTES_LIMIT).
//
// Args:
// bytes: Number of bytes to request.
//
// Returns:
// Whether the request is approved given the total dumping
// limit.
static bool requestDiskByteUsage(uint64_t bytes);
// Reset the disk byte usage to zero.
static void resetDiskByteUsage();
static uint64_t global_disk_bytes_limit_;
private:
// Encapsulates the Tensor in an Event protobuf and write it to file.
static absl::Status DumpTensorToEventFile(const DebugNodeKey& debug_node_key,
const Tensor& tensor,
const uint64_t wall_time_us,
const std::string& file_path);
// Implemented ad hoc here for now.
// TODO(cais): Replace with shared implementation once http://b/30497715 is
// fixed.
static absl::Status RecursiveCreateDir(Env* env, const std::string& dir);
// Tracks how much disk has been used so far.
static uint64_t disk_bytes_used_;
// Mutex for thread-safe access to disk_bytes_used_.
static mutex bytes_mu_;
// Default limit for the disk space.
static const uint64_t kDefaultGlobalDiskBytesLimit;
friend class DiskUsageLimitTest;
};
} // namespace tensorflow
namespace std {
template <>
struct hash<::tensorflow::DebugNodeKey> {
size_t operator()(const ::tensorflow::DebugNodeKey& k) const {
return ::tensorflow::Hash64(
::tensorflow::strings::StrCat(k.device_name, ":", k.node_name, ":",
k.output_slot, ":", k.debug_op, ":"));
}
};
} // namespace std
// TODO(cais): Support grpc:// debug URLs in open source once Python grpc
// genrule becomes available. See b/23796275.
#ifndef PLATFORM_WINDOWS
#include "grpcpp/channel.h"
#include "tensorflow/core/debug/debug_service.grpc.pb.h"
namespace tensorflow {
class DebugGrpcChannel {
public:
// Constructor of DebugGrpcChannel.
//
// Args:
// server_stream_addr: Address (host name and port) of the debug stream
// server implementing the EventListener service (see
// debug_service.proto). E.g., "127.0.0.1:12345".
explicit DebugGrpcChannel(const std::string& server_stream_addr);
virtual ~DebugGrpcChannel() {}
// Attempt to establish connection with server.
//
// Args:
// timeout_micros: Timeout (in microseconds) for the attempt to establish
// the connection.
//
// Returns:
// OK Status iff connection is successfully established before timeout,
// otherwise return an error Status.
absl::Status Connect(const int64_t timeout_micros);
// Write an Event proto to the debug gRPC stream.
//
// Thread-safety: Safe with respect to other calls to the same method and
// calls to ReadEventReply() and Close().
//
// Args:
// event: The event proto to be written to the stream.
//
// Returns:
// True iff the write is successful.
bool WriteEvent(const Event& event);
// Read an EventReply proto from the debug gRPC stream.
//
// This method blocks and waits for an EventReply from the server.
// Thread-safety: Safe with respect to other calls to the same method and
// calls to WriteEvent() and Close().
//
// Args:
// event_reply: the to-be-modified EventReply proto passed as reference.
//
// Returns:
// True iff the read is successful.
bool ReadEventReply(EventReply* event_reply);
// Receive and process EventReply protos from the gRPC debug server.
//
// The processing includes setting debug watch key states using the
// DebugOpStateChange fields of the EventReply.
//
// Args:
// max_replies: Maximum number of replies to receive. Will receive all
// remaining replies iff max_replies == 0.
void ReceiveAndProcessEventReplies(size_t max_replies);
// Receive EventReplies from server (if any) and close the stream and the
// channel.
absl::Status ReceiveServerRepliesAndClose();
private:
std::string server_stream_addr_;
std::string url_;
::grpc::ClientContext ctx_;
std::shared_ptr<::grpc::Channel> channel_;
std::unique_ptr<grpc::EventListener::Stub> stub_;
std::unique_ptr<::grpc::ClientReaderWriterInterface<Event, EventReply>>
reader_writer_;
mutex mu_;
};
class DebugGrpcIO {
public:
static const size_t kGrpcMessageSizeLimitBytes;
static const size_t kGrpcMaxVarintLengthSize;
// Sends a tensor through a debug gRPC stream.
static absl::Status SendTensorThroughGrpcStream(
const DebugNodeKey& debug_node_key, const Tensor& tensor,
const uint64_t wall_time_us, const std::string& grpc_stream_url,
const bool gated);
// Sends an Event proto through a debug gRPC stream.
// Thread-safety: Safe with respect to other calls to the same method and
// calls to CloseGrpcStream().
//
// Args:
// event_proto: The Event proto to be sent.
// grpc_stream_url: The grpc:// URL of the stream to use, e.g.,
// "grpc://localhost:11011", "localhost:22022".
// receive_reply: Whether an EventReply proto will be read after event_proto
// is sent and before the function returns.
//
// Returns:
// The Status of the operation.
static absl::Status SendEventProtoThroughGrpcStream(
const Event& event_proto, const std::string& grpc_stream_url,
const bool receive_reply = false);
// Receive an EventReply proto through a debug gRPC stream.
static absl::Status ReceiveEventReplyProtoThroughGrpcStream(
EventReply* event_reply, const std::string& grpc_stream_url);
// Check whether a debug watch key is read-activated at a given gRPC URL.
static bool IsReadGateOpen(const std::string& grpc_debug_url,
const std::string& watch_key);
// Check whether a debug watch key is write-activated (i.e., read- and
// write-activated) at a given gRPC URL.
static bool IsWriteGateOpen(const std::string& grpc_debug_url,
const std::string& watch_key);
// Closes a gRPC stream to the given address, if it exists.
// Thread-safety: Safe with respect to other calls to the same method and
// calls to SendTensorThroughGrpcStream().
static absl::Status CloseGrpcStream(const std::string& grpc_stream_url);
// Set the gRPC state of a debug node key.
// TODO(cais): Include device information in watch_key.
static void SetDebugNodeKeyGrpcState(
const std::string& grpc_debug_url, const std::string& watch_key,
const EventReply::DebugOpStateChange::State new_state);
private:
using DebugNodeName2State =
std::unordered_map<std::string, EventReply::DebugOpStateChange::State>;
// Returns a global map from grpc debug URLs to the corresponding
// DebugGrpcChannels.
static std::unordered_map<std::string, std::unique_ptr<DebugGrpcChannel>>*
GetStreamChannels();
// Get a DebugGrpcChannel object at a given URL, creating one if necessary.
//
// Args:
// grpc_stream_url: grpc:// URL of the stream, e.g., "grpc://localhost:6064"
// debug_grpc_channel: A pointer to the DebugGrpcChannel object, passed as a
// a pointer to the pointer. The DebugGrpcChannel object is owned
// statically elsewhere, not by the caller of this function.
//
// Returns:
// Status of this operation.
static absl::Status GetOrCreateDebugGrpcChannel(
const std::string& grpc_stream_url,
DebugGrpcChannel** debug_grpc_channel);
// Returns a map from debug URL to a map from debug op name to enabled state.
static std::unordered_map<std::string, DebugNodeName2State>*
GetEnabledDebugOpStates();
// Returns a map from debug op names to enabled state, for a given debug URL.
static DebugNodeName2State* GetEnabledDebugOpStatesAtUrl(
const std::string& grpc_debug_url);
// Clear enabled debug op state from all debug URLs (if any).
static void ClearEnabledWatchKeys();
static mutex streams_mu_;
static int64_t channel_connection_timeout_micros_;
friend class GrpcDebugTest;
friend class DebugNumericSummaryOpTest;
};
} // namespace tensorflow
#endif // #ifndef(PLATFORM_WINDOWS)
#endif // TENSORFLOW_CORE_DEBUG_DEBUG_IO_UTILS_H_
tensorflow/core/debug/debug_io_utils_test.cc
+525
View File
@@ -0,0 +1,525 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/debug/debug_io_utils.h"
#include <cstdlib>
#include <memory>
#include <unordered_set>
#include "absl/synchronization/notification.h"
#include "tensorflow/core/debug/debug_callback_registry.h"
#include "tensorflow/core/debug/debug_node_key.h"
#include "tensorflow/core/debug/debugger_event_metadata.pb.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/framework/tensor.pb.h"
#include "tensorflow/core/framework/tensor_testutil.h"
#include "tensorflow/core/lib/core/notification.h"
#include "tensorflow/core/lib/core/status_test_util.h"
#include "tensorflow/core/lib/core/threadpool.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/util/event.pb.h"
namespace tensorflow {
namespace {
class DebugIOUtilsTest : public ::testing::Test {
public:
void Initialize() {
env_ = Env::Default();
tensor_a_ = std::make_unique<Tensor>(DT_FLOAT, TensorShape({2, 2}));
tensor_a_->flat<float>()(0) = 5.0;
tensor_a_->flat<float>()(1) = 3.0;
tensor_a_->flat<float>()(2) = -1.0;
tensor_a_->flat<float>()(3) = 0.0;
tensor_b_.reset(new Tensor(DT_STRING, TensorShape{2}));
tensor_b_->flat<tstring>()(0) = "corge";
tensor_b_->flat<tstring>()(1) = "garply";
}
Env* env_;
std::unique_ptr<Tensor> tensor_a_;
std::unique_ptr<Tensor> tensor_b_;
};
TEST_F(DebugIOUtilsTest, ConstructDebugNodeKey) {
DebugNodeKey debug_node_key("/job:worker/replica:1/task:0/device:GPU:2",
"hidden_1/MatMul", 0, "DebugIdentity");
EXPECT_EQ("/job:worker/replica:1/task:0/device:GPU:2",
debug_node_key.device_name);
EXPECT_EQ("hidden_1/MatMul", debug_node_key.node_name);
EXPECT_EQ(0, debug_node_key.output_slot);
EXPECT_EQ("DebugIdentity", debug_node_key.debug_op);
EXPECT_EQ("hidden_1/MatMul:0:DebugIdentity", debug_node_key.debug_node_name);
EXPECT_EQ("_tfdbg_device_,job_worker,replica_1,task_0,device_GPU_2",
debug_node_key.device_path);
}
TEST_F(DebugIOUtilsTest, EqualityOfDebugNodeKeys) {
const DebugNodeKey debug_node_key_1("/job:worker/replica:1/task:0/gpu:2",
"hidden_1/MatMul", 0, "DebugIdentity");
const DebugNodeKey debug_node_key_2("/job:worker/replica:1/task:0/gpu:2",
"hidden_1/MatMul", 0, "DebugIdentity");
const DebugNodeKey debug_node_key_3("/job:worker/replica:1/task:0/gpu:2",
"hidden_1/BiasAdd", 0, "DebugIdentity");
const DebugNodeKey debug_node_key_4("/job:worker/replica:1/task:0/gpu:2",
"hidden_1/MatMul", 0,
"DebugNumericSummary");
EXPECT_EQ(debug_node_key_1, debug_node_key_2);
EXPECT_NE(debug_node_key_1, debug_node_key_3);
EXPECT_NE(debug_node_key_1, debug_node_key_4);
EXPECT_NE(debug_node_key_3, debug_node_key_4);
}
TEST_F(DebugIOUtilsTest, DebugNodeKeysIsHashable) {
const DebugNodeKey debug_node_key_1("/job:worker/replica:1/task:0/gpu:2",
"hidden_1/MatMul", 0, "DebugIdentity");
const DebugNodeKey debug_node_key_2("/job:worker/replica:1/task:0/gpu:2",
"hidden_1/MatMul", 0, "DebugIdentity");
const DebugNodeKey debug_node_key_3("/job:worker/replica:1/task:0/gpu:2",
"hidden_1/BiasAdd", 0, "DebugIdentity");
std::unordered_set<DebugNodeKey> keys;
keys.insert(debug_node_key_1);
ASSERT_EQ(1, keys.size());
keys.insert(debug_node_key_3);
ASSERT_EQ(2, keys.size());
keys.erase(debug_node_key_2);
ASSERT_EQ(1, keys.size());
}
TEST_F(DebugIOUtilsTest, DumpFloatTensorToFileSunnyDay) {
Initialize();
const std::string test_dir =
absl::StrCat(testing::TmpDir(), "/DumpFloatTensorToFileSunnyDay");
if (!env_->FileExists(test_dir).ok()) {
ASSERT_TRUE(env_->RecursivelyCreateDir(test_dir).ok());
}
// Append levels of nonexisting directories, to test that the function can
// create directories.
const uint64_t wall_time = env_->NowMicros();
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"foo/bar/qux/tensor_a", 0, "DebugIdentity");
std::string dump_file_path;
TF_ASSERT_OK(DebugFileIO::DumpTensorToDir(
kDebugNodeKey, *tensor_a_, wall_time, test_dir, &dump_file_path));
// Read the file into a Event proto.
Event event;
TF_ASSERT_OK(ReadEventFromFile(dump_file_path, &event));
ASSERT_GE(wall_time, event.wall_time());
ASSERT_EQ(1, event.summary().value().size());
ASSERT_EQ(kDebugNodeKey.debug_node_name,
event.summary().value(0).node_name());
Tensor a_prime(DT_FLOAT);
ASSERT_TRUE(a_prime.FromProto(event.summary().value(0).tensor()));
// Verify tensor shape and value.
ASSERT_EQ(tensor_a_->shape(), a_prime.shape());
for (int i = 0; i < a_prime.flat<float>().size(); ++i) {
ASSERT_EQ(tensor_a_->flat<float>()(i), a_prime.flat<float>()(i));
}
// Tear down temporary file and directories.
int64_t undeleted_files = 0;
int64_t undeleted_dirs = 0;
ASSERT_TRUE(
env_->DeleteRecursively(test_dir, &undeleted_files, &undeleted_dirs)
.ok());
ASSERT_EQ(0, undeleted_files);
ASSERT_EQ(0, undeleted_dirs);
}
TEST_F(DebugIOUtilsTest, DumpStringTensorToFileSunnyDay) {
Initialize();
const std::string test_dir =
absl::StrCat(testing::TmpDir(), "/DumpStringTensorToFileSunnyDay");
if (!env_->FileExists(test_dir).ok()) {
ASSERT_TRUE(env_->RecursivelyCreateDir(test_dir).ok());
}
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"quux/grault/tensor_b", 1, "DebugIdentity");
const uint64_t wall_time = env_->NowMicros();
std::string dump_file_name;
absl::Status s = DebugFileIO::DumpTensorToDir(
kDebugNodeKey, *tensor_b_, wall_time, test_dir, &dump_file_name);
ASSERT_TRUE(s.ok());
// Read the file into a Event proto.
Event event;
TF_ASSERT_OK(ReadEventFromFile(dump_file_name, &event));
ASSERT_GE(wall_time, event.wall_time());
ASSERT_EQ(1, event.summary().value().size());
ASSERT_EQ(kDebugNodeKey.node_name, event.summary().value(0).tag());
ASSERT_EQ(kDebugNodeKey.debug_node_name,
event.summary().value(0).node_name());
// Determine and validate some information from the metadata.
third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
auto status = tensorflow::protobuf::util::JsonStringToMessage(
event.summary().value(0).metadata().plugin_data().content(), &metadata);
ASSERT_TRUE(status.ok());
ASSERT_EQ(kDebugNodeKey.device_name, metadata.device());
ASSERT_EQ(kDebugNodeKey.output_slot, metadata.output_slot());
Tensor b_prime(DT_STRING);
ASSERT_TRUE(b_prime.FromProto(event.summary().value(0).tensor()));
// Verify tensor shape and value.
ASSERT_EQ(tensor_b_->shape(), b_prime.shape());
for (int i = 0; i < b_prime.flat<tstring>().size(); ++i) {
ASSERT_EQ(tensor_b_->flat<tstring>()(i), b_prime.flat<tstring>()(i));
}
// Tear down temporary file and directories.
int64_t undeleted_files = 0;
int64_t undeleted_dirs = 0;
ASSERT_TRUE(
env_->DeleteRecursively(test_dir, &undeleted_files, &undeleted_dirs)
.ok());
ASSERT_EQ(0, undeleted_files);
ASSERT_EQ(0, undeleted_dirs);
}
TEST_F(DebugIOUtilsTest, DumpTensorToFileCannotCreateDirectory) {
Initialize();
// First, create the file at the path.
const std::string test_dir =
absl::StrCat(testing::TmpDir(), "/DumpTensorToFileCannotCreateDirectory");
if (!env_->FileExists(test_dir).ok()) {
ASSERT_TRUE(env_->RecursivelyCreateDir(test_dir).ok());
}
const std::string kDeviceName = "/job:localhost/replica:0/task:0/cpu:0";
const DebugNodeKey kDebugNodeKey(kDeviceName, "baz/tensor_a", 0,
"DebugIdentity");
const std::string txt_file_dir =
io::JoinPath(test_dir, DebugNodeKey::DeviceNameToDevicePath(kDeviceName));
const std::string txt_file_name = io::JoinPath(txt_file_dir, "baz");
if (!env_->FileExists(txt_file_dir).ok()) {
ASSERT_TRUE(env_->RecursivelyCreateDir(txt_file_dir).ok());
}
ASSERT_EQ(error::Code::NOT_FOUND, env_->FileExists(txt_file_name).code());
std::unique_ptr<WritableFile> file;
ASSERT_TRUE(env_->NewWritableFile(txt_file_name, &file).ok());
TF_EXPECT_OK(file->Append("text in baz"));
TF_EXPECT_OK(file->Flush());
TF_ASSERT_OK(file->Close());
// Verify that the path exists and that it is a file, not a directory.
ASSERT_TRUE(env_->FileExists(txt_file_name).ok());
ASSERT_FALSE(env_->IsDirectory(txt_file_name).ok());
// Second, try to dump the tensor to a path that requires "baz" to be a
// directory, which should lead to an error.
const uint64_t wall_time = env_->NowMicros();
std::string dump_file_name;
absl::Status s = DebugFileIO::DumpTensorToDir(
kDebugNodeKey, *tensor_a_, wall_time, test_dir, &dump_file_name);
ASSERT_FALSE(s.ok());
// Tear down temporary file and directories.
int64_t undeleted_files = 0;
int64_t undeleted_dirs = 0;
ASSERT_TRUE(
env_->DeleteRecursively(test_dir, &undeleted_files, &undeleted_dirs)
.ok());
ASSERT_EQ(0, undeleted_files);
ASSERT_EQ(0, undeleted_dirs);
}
TEST_F(DebugIOUtilsTest, PublishTensorToMultipleFileURLs) {
Initialize();
const int kNumDumpRoots = 3;
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"foo/bar/qux/tensor_a", 0, "DebugIdentity");
const uint64_t wall_time = env_->NowMicros();
std::vector<std::string> dump_roots;
std::vector<std::string> dump_file_paths;
std::vector<std::string> urls;
for (int i = 0; i < kNumDumpRoots; ++i) {
std::string dump_root =
absl::StrCat(testing::TmpDir(), "/PublicTensorToMultipleFileUrls_", i);
dump_roots.push_back(dump_root);
dump_file_paths.push_back(
DebugFileIO::GetDumpFilePath(dump_root, kDebugNodeKey, wall_time));
urls.push_back(absl::StrCat("file://", dump_root));
}
for (int i = 1; i < kNumDumpRoots; ++i) {
ASSERT_NE(dump_roots[0], dump_roots[i]);
}
absl::Status s =
DebugIO::PublishDebugTensor(kDebugNodeKey, *tensor_a_, wall_time, urls);
ASSERT_TRUE(s.ok());
// Try reading the file into a Event proto.
for (int i = 0; i < kNumDumpRoots; ++i) {
// Read the file into a Event proto.
Event event;
TF_ASSERT_OK(ReadEventFromFile(dump_file_paths[i], &event));
ASSERT_GE(wall_time, event.wall_time());
ASSERT_EQ(1, event.summary().value().size());
ASSERT_EQ(kDebugNodeKey.node_name, event.summary().value(0).tag());
ASSERT_EQ(kDebugNodeKey.debug_node_name,
event.summary().value(0).node_name());
// Determine and validate some information from the metadata.
third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
auto status = tensorflow::protobuf::util::JsonStringToMessage(
event.summary().value(0).metadata().plugin_data().content(), &metadata);
ASSERT_TRUE(status.ok());
ASSERT_EQ(kDebugNodeKey.device_name, metadata.device());
ASSERT_EQ(kDebugNodeKey.output_slot, metadata.output_slot());
Tensor a_prime(DT_FLOAT);
ASSERT_TRUE(a_prime.FromProto(event.summary().value(0).tensor()));
// Verify tensor shape and value.
ASSERT_EQ(tensor_a_->shape(), a_prime.shape());
for (int i = 0; i < a_prime.flat<float>().size(); ++i) {
ASSERT_EQ(tensor_a_->flat<float>()(i), a_prime.flat<float>()(i));
}
}
// Tear down temporary file and directories.
for (int i = 0; i < kNumDumpRoots; ++i) {
int64_t undeleted_files = 0;
int64_t undeleted_dirs = 0;
ASSERT_TRUE(env_->DeleteRecursively(dump_roots[i], &undeleted_files,
&undeleted_dirs)
.ok());
ASSERT_EQ(0, undeleted_files);
ASSERT_EQ(0, undeleted_dirs);
}
}
TEST_F(DebugIOUtilsTest, PublishTensorToMemoryCallback) {
Initialize();
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"foo/bar/qux/tensor_a", 0, "DebugIdentity");
const uint64_t wall_time = env_->NowMicros();
bool called = false;
std::vector<std::string> urls = {"memcbk://test_callback"};
;
auto* callback_registry = DebugCallbackRegistry::singleton();
callback_registry->RegisterCallback(
"test_callback", [this, &kDebugNodeKey, &called](const DebugNodeKey& key,
const Tensor& tensor) {
called = true;
ASSERT_EQ(kDebugNodeKey.device_name, key.device_name);
ASSERT_EQ(kDebugNodeKey.node_name, key.node_name);
ASSERT_EQ(tensor_a_->shape(), tensor.shape());
for (int i = 0; i < tensor.flat<float>().size(); ++i) {
ASSERT_EQ(tensor_a_->flat<float>()(i), tensor.flat<float>()(i));
}
});
absl::Status s =
DebugIO::PublishDebugTensor(kDebugNodeKey, *tensor_a_, wall_time, urls);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(called);
callback_registry->UnregisterCallback("test_callback");
}
TEST_F(DebugIOUtilsTest, PublishTensorConcurrentlyToPartiallyOverlappingPaths) {
Initialize();
const int kConcurrentPubs = 3;
const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0",
"tensor_a", 0, "DebugIdentity");
thread::ThreadPool* tp =
new thread::ThreadPool(Env::Default(), "test", kConcurrentPubs);
const uint64_t wall_time = env_->NowMicros();
const std::string dump_root_base =
absl::StrCat(testing::TmpDir(),
"/PublishTensorConcurrentlyToPartiallyOverlappingPaths");
if (!env_->FileExists(dump_root_base).ok()) {
ASSERT_TRUE(env_->RecursivelyCreateDir(dump_root_base).ok());
}
mutex mu;
std::vector<std::string> dump_roots TF_GUARDED_BY(mu);
std::vector<std::string> dump_file_paths TF_GUARDED_BY(mu);
int dump_count TF_GUARDED_BY(mu) = 0;
int done_count TF_GUARDED_BY(mu) = 0;
absl::Notification all_done;
auto fn = [this, &dump_count, &done_count, &mu, &dump_root_base, &dump_roots,
&dump_file_paths, &wall_time, &kDebugNodeKey, &kConcurrentPubs,
&all_done]() {
// "gumpy" is the shared directory part of the path.
std::string dump_root;
std::string debug_url;
{
mutex_lock l(mu);
dump_root =
absl::StrCat(dump_root_base, "grumpy/", "dump_", dump_count++);
dump_roots.push_back(dump_root);
dump_file_paths.push_back(
DebugFileIO::GetDumpFilePath(dump_root, kDebugNodeKey, wall_time));
debug_url = absl::StrCat("file://", dump_root);
}
std::vector<std::string> urls;
urls.push_back(debug_url);
absl::Status s =
DebugIO::PublishDebugTensor(kDebugNodeKey, *tensor_a_, wall_time, urls);
ASSERT_TRUE(s.ok());
{
mutex_lock l(mu);
done_count++;
if (done_count == kConcurrentPubs) {
all_done.Notify();
}
}
};
for (int i = 0; i < kConcurrentPubs; ++i) {
tp->Schedule(fn);
}
// Wait for all dumping calls to finish.
all_done.WaitForNotification();
delete tp;
{
mutex_lock l(mu);
for (int i = 1; i < kConcurrentPubs; ++i) {
ASSERT_NE(dump_roots[0], dump_roots[i]);
}
// Try reading the file into a Event proto.
for (int i = 0; i < kConcurrentPubs; ++i) {
// Read the file into a Event proto.
Event event;
TF_ASSERT_OK(ReadEventFromFile(dump_file_paths[i], &event));
ASSERT_GE(wall_time, event.wall_time());
ASSERT_EQ(1, event.summary().value().size());
ASSERT_EQ(kDebugNodeKey.node_name, event.summary().value(0).tag());
ASSERT_EQ(kDebugNodeKey.debug_node_name,
event.summary().value(0).node_name());
// Determine and validate some information from the metadata.
third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
auto status = tensorflow::protobuf::util::JsonStringToMessage(
event.summary().value(0).metadata().plugin_data().content(),
&metadata);
ASSERT_TRUE(status.ok());
ASSERT_EQ(kDebugNodeKey.device_name, metadata.device());
ASSERT_EQ(kDebugNodeKey.output_slot, metadata.output_slot());
Tensor a_prime(DT_FLOAT);
ASSERT_TRUE(a_prime.FromProto(event.summary().value(0).tensor()));
// Verify tensor shape and value.
ASSERT_EQ(tensor_a_->shape(), a_prime.shape());
for (int i = 0; i < a_prime.flat<float>().size(); ++i) {
ASSERT_EQ(tensor_a_->flat<float>()(i), a_prime.flat<float>()(i));
}
}
// Tear down temporary file and directories.
int64_t undeleted_files = 0;
int64_t undeleted_dirs = 0;
auto delete_files = env_->DeleteRecursively(
dump_root_base, &undeleted_files, &undeleted_dirs);
ASSERT_TRUE(delete_files.ok()) << delete_files;
ASSERT_EQ(0, undeleted_files);
ASSERT_EQ(0, undeleted_dirs);
}
}
class DiskUsageLimitTest : public ::testing::Test {
public:
void Initialize() {
setenv("TFDBG_DISK_BYTES_LIMIT", "", 1);
DebugFileIO::resetDiskByteUsage();
DebugFileIO::global_disk_bytes_limit_ = 0;
}
};
TEST_F(DiskUsageLimitTest, RequestWithZeroByteIsOkay) {
Initialize();
ASSERT_TRUE(DebugFileIO::requestDiskByteUsage(0L));
}
TEST_F(DiskUsageLimitTest, ExceedingLimitAfterOneCall) {
Initialize();
ASSERT_FALSE(DebugFileIO::requestDiskByteUsage(100L * 1024L * 1024L * 1024L));
}
TEST_F(DiskUsageLimitTest, ExceedingLimitAfterTwoCalls) {
Initialize();
ASSERT_TRUE(DebugFileIO::requestDiskByteUsage(50L * 1024L * 1024L * 1024L));
ASSERT_FALSE(DebugFileIO::requestDiskByteUsage(50L * 1024L * 1024L * 1024L));
ASSERT_TRUE(DebugFileIO::requestDiskByteUsage(1024L));
}
TEST_F(DiskUsageLimitTest, ResetDiskByteUsageWorks) {
Initialize();
ASSERT_TRUE(DebugFileIO::requestDiskByteUsage(50L * 1024L * 1024L * 1024L));
ASSERT_FALSE(DebugFileIO::requestDiskByteUsage(50L * 1024L * 1024L * 1024L));
DebugFileIO::resetDiskByteUsage();
ASSERT_TRUE(DebugFileIO::requestDiskByteUsage(50L * 1024L * 1024L * 1024L));
}
TEST_F(DiskUsageLimitTest, CustomEnvVarIsObeyed) {
Initialize();
setenv("TFDBG_DISK_BYTES_LIMIT", "1024", 1);
ASSERT_FALSE(DebugFileIO::requestDiskByteUsage(1024L));
ASSERT_TRUE(DebugFileIO::requestDiskByteUsage(1000L));
ASSERT_TRUE(DebugFileIO::requestDiskByteUsage(23L));
ASSERT_FALSE(DebugFileIO::requestDiskByteUsage(1L));
DebugFileIO::resetDiskByteUsage();
ASSERT_TRUE(DebugFileIO::requestDiskByteUsage(1023L));
}
} // namespace
} // namespace tensorflow
tensorflow/core/debug/debug_node_key.cc
+65
View File
@@ -0,0 +1,65 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/debug/debug_node_key.h"
#include <cstdint>
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
namespace tensorflow {
const char* const DebugNodeKey::kMetadataFilePrefix = "_tfdbg_";
const char* const DebugNodeKey::kDeviceTag = "device_";
DebugNodeKey::DebugNodeKey(const std::string& device_name,
const std::string& node_name,
const int32_t output_slot,
const std::string& debug_op,
const std::string& io_of_node, const bool is_input,
const int32_t io_index)
: device_name(device_name),
node_name(node_name),
output_slot(output_slot),
debug_op(debug_op),
debug_node_name(
strings::StrCat(node_name, ":", output_slot, ":", debug_op)),
device_path(DeviceNameToDevicePath(device_name)),
io_of_node(io_of_node),
is_input(is_input),
io_index(io_index) {}
bool DebugNodeKey::operator==(const DebugNodeKey& other) const {
return (device_name == other.device_name && node_name == other.node_name &&
output_slot == other.output_slot && debug_op == other.debug_op &&
io_of_node == other.io_of_node && is_input == other.is_input &&
io_index == other.io_index);
}
bool DebugNodeKey::operator!=(const DebugNodeKey& other) const {
return !((*this) == other);
}
const std::string DebugNodeKey::DeviceNameToDevicePath(
const std::string& device_name) {
return absl::StrCat(kMetadataFilePrefix, kDeviceTag,
str_util::StringReplace(
str_util::StringReplace(device_name, ":", "_", true),
"/", ",", true));
}
} // namespace tensorflow
tensorflow/core/debug/debug_node_key.h
+57
View File
@@ -0,0 +1,57 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_DEBUG_DEBUG_NODE_KEY_H_
#define TENSORFLOW_CORE_DEBUG_DEBUG_NODE_KEY_H_
#include <string>
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// Encapsulates debug information for a node that was observed.
struct DebugNodeKey {
static const char* const kMetadataFilePrefix;
static const char* const kDeviceTag;
DebugNodeKey(const std::string& device_name, const std::string& node_name,
int32_t output_slot, const std::string& debug_op,
const std::string& io_of_node = "", bool is_input = false,
int32_t io_index = -1);
// Converts a device name string to a device path string.
// E.g., /job:localhost/replica:0/task:0/cpu:0 will be converted to
// ,job_localhost,replica_0,task_0,cpu_0.
static const std::string DeviceNameToDevicePath(
const std::string& device_name);
bool operator==(const DebugNodeKey& other) const;
bool operator!=(const DebugNodeKey& other) const;
const std::string device_name;
const std::string node_name;
const int32_t output_slot;
const std::string debug_op;
const std::string debug_node_name;
const std::string device_path;
const std::string io_of_node;
const bool is_input;
const int32_t io_index;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_DEBUG_DEBUG_NODE_KEY_H_
tensorflow/core/debug/debug_service.proto
+100
View File
@@ -0,0 +1,100 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
syntax = "proto3";
package tensorflow;
import "tensorflow/core/framework/tensor.proto";
import "tensorflow/core/profiler/tfprof_log.proto";
import "tensorflow/core/protobuf/debug.proto";
import "tensorflow/core/util/event.proto";
// Reply message from EventListener to the client, i.e., to the source of the
// Event protocol buffers, e.g., debug ops inserted by a debugged runtime to a
// TensorFlow graph being executed.
message EventReply {
message DebugOpStateChange {
enum State {
STATE_UNSPECIFIED = 0;
DISABLED = 1;
READ_ONLY = 2;
READ_WRITE = 3;
}
State state = 1;
string node_name = 2;
int32 output_slot = 3;
string debug_op = 4;
}
repeated DebugOpStateChange debug_op_state_changes = 1;
// New tensor value to override the current tensor value with.
TensorProto tensor = 2;
// TODO(cais): Make use of this field to implement overriding of tensor value
// during debugging.
}
// Data on the traceback of a debugged call, e.g., a Session.run() call, or the
// execution of an eager operation.
message CallTraceback {
enum CallType {
UNSPECIFIED = 0;
GRAPH_EXECUTION = 1;
EAGER_EXECUTION = 2;
}
CallType call_type = 1;
// A key for the call. For example, for graph execution, this is a key
// consisting of the names of the fed and fetched tensors.
string call_key = 2;
// Traceback stack for the origin of the call event.
// For graph execution, this is the stack of the Session.run() call.
// For eager execution, this is the stack of the Python line that invokes
// the execution of the eager op.
tfprof.CodeDef origin_stack = 3;
// Keeps track of the mapping from integer IDs in `origin_stack` to actual
// string values (e.g., file paths, function names).
map<int64, string> origin_id_to_string = 4;
// Traceback for the graph (if any) involved in the call.
tfprof.OpLogProto graph_traceback = 5;
// Version of the graph in `graph_traceback` (if any).
int64 graph_version = 6;
}
// EventListener: Receives Event protos, e.g., from debugged TensorFlow
// runtime(s).
service EventListener {
// Client(s) can use this RPC method to send the EventListener Event protos.
// The Event protos can hold information such as:
// 1) intermediate tensors from a debugged graph being executed, which can
// be sent from DebugIdentity ops configured with grpc URLs.
// 2) GraphDefs of partition graphs, which can be sent from special debug
// ops that get executed immediately after the beginning of the graph
// execution.
rpc SendEvents(stream Event) returns (stream EventReply);
// Send the tracebacks of a TensorFlow execution call.
rpc SendTracebacks(CallTraceback) returns (EventReply);
// Send a collection of source code files being debugged.
rpc SendSourceFiles(DebuggedSourceFiles) returns (EventReply);
}
tensorflow/core/debug/debugger_event_metadata.proto
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2026 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ==============================================================================
syntax = "proto3";
package third_party.tensorflow.core.debug;
// Encapsulates per-event data related to debugging.
message DebuggerEventMetadata {
string device = 1;
int32 output_slot = 2;
int32 num_chunks = 3;
int32 chunk_index = 4;
}
tensorflow/core/debug/debugger_state_impl.cc
+71
View File
@@ -0,0 +1,71 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/debug/debugger_state_impl.h"
#include "tensorflow/core/debug/debug_graph_utils.h"
#include "tensorflow/core/debug/debug_io_utils.h"
namespace tensorflow {
DebuggerState::DebuggerState(const DebugOptions& debug_options) {
for (const DebugTensorWatch& watch :
debug_options.debug_tensor_watch_opts()) {
for (const std::string& url : watch.debug_urls()) {
debug_urls_.insert(url);
}
}
if (debug_options.reset_disk_byte_usage()) {
DebugFileIO::resetDiskByteUsage();
}
}
DebuggerState::~DebuggerState() {
for (const std::string& debug_url : debug_urls_) {
DebugIO::CloseDebugURL(debug_url).IgnoreError();
}
}
absl::Status DebuggerState::PublishDebugMetadata(
const int64_t global_step, const int64_t session_run_index,
const int64_t executor_step_index,
const std::vector<std::string>& input_names,
const std::vector<std::string>& output_names,
const std::vector<std::string>& target_names) {
return DebugIO::PublishDebugMetadata(global_step, session_run_index,
executor_step_index, input_names,
output_names, target_names, debug_urls_);
}
absl::Status DebugGraphDecorator::DecorateGraph(Graph* graph, Device* device) {
DebugNodeInserter::DeparallelizeWhileLoops(graph, device);
return DebugNodeInserter::InsertNodes(
debug_options_.debug_tensor_watch_opts(), graph, device);
}
absl::Status DebugGraphDecorator::PublishGraph(const Graph& graph,
const std::string& device_name) {
std::unordered_set<std::string> debug_urls;
for (const DebugTensorWatch& watch :
debug_options_.debug_tensor_watch_opts()) {
for (const std::string& url : watch.debug_urls()) {
debug_urls.insert(url);
}
}
return DebugIO::PublishGraph(graph, device_name, debug_urls);
}
} // namespace tensorflow
tensorflow/core/debug/debugger_state_impl.h
+62
View File
@@ -0,0 +1,62 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_DEBUG_DEBUGGER_STATE_IMPL_H_
#define TENSORFLOW_CORE_DEBUG_DEBUGGER_STATE_IMPL_H_
#include "tensorflow/core/common_runtime/debugger_state_interface.h"
#include <unordered_set>
#include <vector>
namespace tensorflow {
class DebuggerState : public DebuggerStateInterface {
public:
DebuggerState(const DebugOptions& debug_options);
~DebuggerState() override;
// Publish metadata about the debugged Session::Run() call.
//
// See the doc string of DebuggerStateInterface::PublishDebugMetadata() for
// details.
absl::Status PublishDebugMetadata(
const int64_t global_step, const int64_t session_run_count,
const int64_t executor_step_count,
const std::vector<std::string>& input_names,
const std::vector<std::string>& output_names,
const std::vector<std::string>& target_names) override;
private:
std::unordered_set<std::string> debug_urls_;
};
class DebugGraphDecorator : public DebugGraphDecoratorInterface {
public:
DebugGraphDecorator(const DebugOptions& debug_options)
: debug_options_(debug_options) {}
~DebugGraphDecorator() override {}
absl::Status DecorateGraph(Graph* graph, Device* device) override;
absl::Status PublishGraph(const Graph& graph,
const std::string& device_name) override;
private:
DebugOptions debug_options_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_DEBUG_DEBUGGER_STATE_IMPL_H_
tensorflow/core/debug/grpc_session_debug_test.cc
+307
View File
@@ -0,0 +1,307 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/distributed_runtime/rpc/grpc_session.h"
#include <memory>
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/debug/debug_io_utils.h"
#include "tensorflow/core/distributed_runtime/rpc/grpc_testlib.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/framework/tensor_testutil.h"
#include "tensorflow/core/graph/default_device.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/graph/testlib.h"
#include "tensorflow/core/lib/core/status_test_util.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/protobuf/debug.pb.h"
#include "tensorflow/core/protobuf/rewriter_config.pb.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/util/port.h"
namespace tensorflow {
namespace {
SessionOptions Devices(int num_cpus, int num_gpus) {
SessionOptions result;
(*result.config.mutable_device_count())["CPU"] = num_cpus;
(*result.config.mutable_device_count())["GPU"] = num_gpus;
return result;
}
void CreateGraphDef(GraphDef* graph_def, std::string node_names[3]) {
Graph graph(OpRegistry::Global());
Tensor a_tensor(DT_FLOAT, TensorShape({1, 2}));
test::FillValues<float>(&a_tensor, {1.0, 2.0});
Node* a = test::graph::Constant(&graph, a_tensor);
node_names[0] = a->name();
Tensor b_tensor(DT_FLOAT, TensorShape({2, 1}));
test::FillValues<float>(&b_tensor, {2.0, 1.0});
Node* b = test::graph::Constant(&graph, b_tensor);
node_names[1] = b->name();
// c = a * b
Node* c = test::graph::Matmul(&graph, a, b, false, false);
node_names[2] = c->name();
test::graph::ToGraphDef(&graph, graph_def);
}
// Asserts that "val" is a single float tensor. The only float is
// "expected_val".
void IsSingleFloatValue(const Tensor& val, float expected_val) {
ASSERT_EQ(val.dtype(), DT_FLOAT);
ASSERT_EQ(val.NumElements(), 1);
ASSERT_EQ(val.flat<float>()(0), expected_val);
}
SessionOptions Options(const std::string& target, int placement_period) {
SessionOptions options;
// NOTE(mrry): GrpcSession requires a grpc:// scheme prefix in the target
// string.
options.target = absl::StrCat("grpc://", target);
options.config.set_placement_period(placement_period);
options.config.mutable_graph_options()
->mutable_optimizer_options()
->set_opt_level(OptimizerOptions::L0);
options.config.mutable_graph_options()
->mutable_rewrite_options()
->set_constant_folding(RewriterConfig::OFF);
return options;
}
std::unique_ptr<Session> NewRemote(const SessionOptions& options) {
return std::unique_ptr<Session>(CHECK_NOTNULL(NewSession(options)));
}
class GrpcSessionDebugTest : public ::testing::Test {
protected:
void SetUp() override { CreateDumpDir(); }
void TearDown() override { DeleteDumpDir(); }
void DeleteDumpDir() {
if (Env::Default()->IsDirectory(dump_dir_).ok()) {
int64_t undeleted_files = 0;
int64_t undeleted_dirs = 0;
ASSERT_TRUE(
Env::Default()
->DeleteRecursively(dump_dir_, &undeleted_files, &undeleted_dirs)
.ok());
ASSERT_EQ(0, undeleted_files);
ASSERT_EQ(0, undeleted_dirs);
}
}
const std::string GetDebugURL() { return debug_url_; }
void LoadTensorDumps(const std::string& subdir,
std::vector<Tensor>* tensors) {
const std::string dirpath = io::JoinPath(dump_dir_, subdir);
if (!(Env::Default()->IsDirectory(dirpath).ok())) {
return;
}
std::vector<std::string> filenames;
TF_ASSERT_OK(Env::Default()->GetChildren(dirpath, &filenames));
for (const std::string& filename : filenames) {
Event event;
TF_ASSERT_OK(ReadEventFromFile(io::JoinPath(dirpath, filename), &event));
if (event.summary().value().size() == 1) {
Tensor tensor;
ASSERT_TRUE(tensor.FromProto(event.summary().value(0).tensor()));
tensors->push_back(tensor);
}
}
}
private:
void CreateDumpDir() {
char dir_template[] = "/tmp/tfdbg_grpc_sessions_XXXXXX";
dump_dir_ = mkdtemp(dir_template);
debug_url_ = absl::StrCat("file://", dump_dir_);
}
std::string dump_dir_;
std::string debug_url_;
};
TEST_F(GrpcSessionDebugTest, FileDebugURL) {
GraphDef graph;
std::string node_names[3];
CreateGraphDef(&graph, node_names);
std::unique_ptr<test::TestCluster> cluster;
TF_CHECK_OK(test::TestCluster::MakeTestCluster(
test::TestClusterConfig()
.Options(Devices(1, 0))
.Jobs({test::TestJob{/*name=*/"localhost", /*num_tasks=*/2}}),
&cluster));
auto session = NewRemote(Options(cluster->targets()[0], 1));
TF_CHECK_OK(session->Create(graph));
// Iteration 0: No watch.
// Iterations 1 and 2: Watch one Tensor.
// Iterations 3 and 4: Watch two Tensors.
// Iteration 5: No watch.
for (size_t i = 0; i < 6; ++i) {
RunOptions options;
if (i >= 1 && i < 5) {
DebugOptions* debug_options = options.mutable_debug_options();
DebugTensorWatch* watch = debug_options->add_debug_tensor_watch_opts();
watch->set_node_name(node_names[0]);
watch->set_output_slot(0);
watch->add_debug_ops("DebugIdentity");
watch->add_debug_urls(GetDebugURL());
if (i >= 3) {
watch = debug_options->add_debug_tensor_watch_opts();
watch->set_node_name(node_names[1]);
watch->set_output_slot(0);
watch->add_debug_ops("DebugIdentity");
watch->add_debug_urls(GetDebugURL());
}
}
RunMetadata metadata;
std::vector<Tensor> outputs;
TF_CHECK_OK(
session->Run(options, {}, {node_names[2]}, {}, &outputs, &metadata));
ASSERT_EQ(1, outputs.size());
IsSingleFloatValue(outputs[0], 4.0);
std::vector<Tensor> dumped_tensors;
LoadTensorDumps(io::JoinPath(DebugNodeKey::DeviceNameToDevicePath(
cluster->devices()[0].name()),
"n"),
&dumped_tensors);
if (i == 0 || i == 5) {
ASSERT_EQ(0, dumped_tensors.size());
} else {
if (i == 1 || i == 2) {
ASSERT_EQ(1, dumped_tensors.size());
ASSERT_EQ(TensorShape({1, 2}), dumped_tensors[0].shape());
ASSERT_EQ(1.0, dumped_tensors[0].flat<float>()(0));
ASSERT_EQ(2.0, dumped_tensors[0].flat<float>()(1));
} else {
ASSERT_EQ(2, dumped_tensors.size());
}
DeleteDumpDir();
}
}
TF_CHECK_OK(session->Close());
}
void SetDevice(GraphDef* graph, const std::string& name,
const std::string& dev) {
for (size_t i = 0; i < graph->node_size(); ++i) {
if (graph->node(i).name() == name) {
graph->mutable_node(i)->set_device(dev);
return;
}
}
LOG(FATAL) << "Name '" << name << "' not found.";
}
TEST_F(GrpcSessionDebugTest, MultiDevices_String) {
std::unique_ptr<test::TestCluster> cluster;
TF_CHECK_OK(test::TestCluster::MakeTestCluster(
test::TestClusterConfig()
.Options(Devices(1, 1))
.Jobs({test::TestJob{/*name=*/"localhost", /*num_tasks=*/2}}),
&cluster));
auto session = NewRemote(Options(cluster->targets()[0], 1000));
// b = a
Graph graph(OpRegistry::Global());
Tensor a_tensor(DT_STRING, TensorShape({2, 2}));
for (size_t i = 0; i < 4; ++i) {
a_tensor.flat<tstring>()(i) = "hello, world";
}
Node* a = test::graph::Constant(&graph, a_tensor);
Node* b = test::graph::Identity(&graph, a);
GraphDef def;
test::graph::ToGraphDef(&graph, &def);
// In this test, we force each node (a, b) on every possible device.
// We test all possible cases.
for (const auto& a_dev : cluster->devices()) {
for (const auto& b_dev : cluster->devices()) {
LOG(INFO) << "a: " << a_dev.name() << " b: " << b_dev.name();
SetDevice(&def, a->name(), a_dev.name());
SetDevice(&def, b->name(), b_dev.name());
absl::Status s = session->Create(def);
if (s.ok()) {
std::vector<Tensor> outputs;
RunOptions options;
DebugOptions* debug_options = options.mutable_debug_options();
DebugTensorWatch* watch = debug_options->add_debug_tensor_watch_opts();
watch->set_node_name(a->name());
watch->set_output_slot(0);
watch->add_debug_ops("DebugIdentity");
watch->add_debug_urls(GetDebugURL());
RunMetadata metadata;
TF_CHECK_OK(
session->Run(options, {}, {b->name()}, {}, &outputs, &metadata));
ASSERT_EQ(1, outputs.size());
ASSERT_EQ(outputs[0].dtype(), DT_STRING);
ASSERT_EQ(outputs[0].NumElements(), 4);
for (size_t i = 0; i < outputs[0].NumElements(); ++i) {
EXPECT_EQ(outputs[0].flat<tstring>()(i), "hello, world");
}
TF_CHECK_OK(session->Close());
std::vector<Tensor> dumped_tensors;
LoadTensorDumps(
io::JoinPath(DebugNodeKey::DeviceNameToDevicePath(a_dev.name()),
"n"),
&dumped_tensors);
ASSERT_EQ(1, dumped_tensors.size());
ASSERT_EQ(TensorShape({2, 2}), dumped_tensors[0].shape());
for (size_t i = 0; i < 4; ++i) {
ASSERT_EQ("hello, world", dumped_tensors[0].flat<tstring>()(i));
}
DeleteDumpDir();
} else {
// The CUDA device does not have an Identity op for strings
LOG(ERROR) << "Error: " << s;
ASSERT_TRUE((a_dev.device_type() == DEVICE_GPU) ||
(b_dev.device_type() == DEVICE_GPU));
ASSERT_FALSE(s.ok());
}
}
}
}
} // namespace
} // namespace tensorflow