wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/python/debug/lib/BUILD
+684
View File
@@ -0,0 +1,684 @@
load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary")
load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
load("//tensorflow:tensorflow.bzl", "py_test")
load("//tensorflow:tensorflow.default.bzl", "cuda_py_strict_test")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:internal"],
licenses = ["notice"],
)
py_library(
name = "op_callbacks_common",
srcs = ["op_callbacks_common.py"],
strict_deps = True,
)
py_library(
name = "check_numerics_callback",
srcs = ["check_numerics_callback.py"],
strict_deps = True,
deps = [
":op_callbacks_common",
":source_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/eager:monitoring",
"//tensorflow/python/framework:op_callbacks",
"//tensorflow/python/framework:ops",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:debug_ops_gen",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/util:compat",
"//tensorflow/python/util:object_identity",
"//tensorflow/python/util:tf_export",
"//third_party/py/numpy",
],
)
py_library(
name = "dumping_callback",
srcs = ["dumping_callback.py"],
strict_deps = True,
deps = [
":debug_events_writer",
":op_callbacks_common",
":source_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/eager:function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:op_callbacks",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:debug_ops_gen",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/util:compat",
"//tensorflow/python/util:object_identity",
"//tensorflow/python/util:tf_export",
"//tensorflow/python/util:tf_stack",
],
)
py_library(
name = "dumping_callback_test_lib",
srcs = ["dumping_callback_test_lib.py"],
strict_deps = True,
deps = [
":check_numerics_callback",
":debug_events_reader",
":dumping_callback",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/framework:versions",
],
)
py_library(
name = "common",
srcs = ["common.py"],
strict_deps = True,
)
py_library(
name = "debug_events_reader",
srcs = ["debug_events_reader.py"],
strict_deps = True,
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/lib/io:tf_record",
"//tensorflow/python/util:compat",
],
)
py_library(
name = "debug_events_monitors",
srcs = ["debug_events_monitors.py"],
strict_deps = True,
deps = [
"//tensorflow/core:protos_all_py",
"//third_party/py/numpy",
],
)
py_library(
name = "debug_events_writer",
srcs = ["debug_events_writer.py"],
strict_deps = True,
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:_pywrap_debug_events_writer",
],
)
py_library(
name = "debug_graphs",
srcs = ["debug_graphs.py"],
strict_deps = True,
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/framework:op_def_registry",
"//tensorflow/python/platform:tf_logging",
],
)
py_library(
name = "debug_data",
srcs = ["debug_data.py"],
strict_deps = True,
visibility = [
"//tensorflow:internal",
"//third_party/py/tf_slim:__subpackages__",
"//waymo/ml/deploy/numeric_debugging:__subpackages__",
],
deps = [
":debug_graphs",
"//tensorflow/core:protos_all_py",
"//tensorflow/core/framework:graph_proto_py_proto",
"//tensorflow/core/util:event_proto_py_proto",
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/platform:gfile",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/util:compat",
"//third_party/py/numpy",
],
)
py_library(
name = "debug_gradients",
srcs = ["debug_gradients.py"],
strict_deps = True,
deps = [
":debug_data",
":debug_graphs",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:tensor",
"//tensorflow/python/ops:array_ops_gen",
"//tensorflow/python/ops:variables",
],
)
py_library(
name = "debug_utils",
srcs = ["debug_utils.py"],
strict_deps = True,
)
py_binary(
name = "grpc_tensorflow_server",
srcs = ["grpc_tensorflow_server.py"],
strict_deps = True,
deps = [
":grpc_tensorflow_server_lib",
"//tensorflow/python/training:server_lib",
],
)
py_library(
name = "grpc_tensorflow_server_lib",
srcs = [
"grpc_tensorflow_server.py",
],
strict_deps = True,
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/training:server_lib",
"@absl_py//absl:app",
],
)
py_library(
name = "source_utils",
srcs = ["source_utils.py"],
strict_deps = True,
deps = [
":profiling",
"//third_party/py/numpy",
"@absl_py//absl:app",
],
)
py_library(
name = "source_remote",
srcs = ["source_remote.py"],
strict_deps = True,
deps = [
":common",
":debug_service_pb2_grpc",
":source_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/core/debug:debug_service_proto_py",
"//tensorflow/python/platform:gfile",
"//tensorflow/python/profiler:tfprof_logger",
],
)
py_library(
name = "profiling",
srcs = ["profiling.py"],
strict_deps = True,
)
py_test(
name = "common_test",
size = "small",
srcs = ["common_test.py"],
strict_deps = True,
deps = [
":common",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:test",
],
)
py_test(
name = "debug_events_monitors_test",
size = "medium",
srcs = ["debug_events_monitors_test.py"],
strict_deps = True,
tags = [
"no_windows", # b/142475891
],
deps = [
":debug_events_monitors",
":debug_events_reader",
":dumping_callback",
":dumping_callback_test_lib",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:test",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
py_test(
name = "debug_events_writer_test",
size = "medium",
srcs = ["debug_events_writer_test.py"],
strict_deps = True,
tags = [
"no_windows", # b/142475891
],
deps = [
":debug_events_reader",
":debug_events_writer",
":dumping_callback_test_lib",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/framework:versions",
"//tensorflow/python/platform:test",
"@absl_py//absl/testing:parameterized",
],
)
py_test(
name = "debug_graphs_test",
size = "small",
srcs = ["debug_graphs_test.py"],
strict_deps = True,
deps = [
":debug_graphs",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
],
)
py_test(
name = "debug_data_test",
size = "small",
srcs = ["debug_data_test.py"],
strict_deps = True,
tags = ["no_windows"], # TODO(b/184424727): Enable this test on Windows.
deps = [
":debug_data",
"//tensorflow/core:protos_all_py",
"//tensorflow/core/framework:graph_proto_py_proto",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:gfile",
"//tensorflow/python/platform:test",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "check_numerics_callback_test",
size = "medium",
srcs = ["check_numerics_callback_test.py"],
tags = [
"no_mac", # TODO(b/175322370): Detected Infinity or NaN in output 0 of graph op "RealDiv"
"no_windows",
],
deps = [
":check_numerics_callback",
"//tensorflow/python/data/ops:dataset_ops",
"//tensorflow/python/eager:backprop",
"//tensorflow/python/eager:context",
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_grad",
"//tensorflow/python/ops:custom_gradient",
"//tensorflow/python/ops:gradient_checker_v2",
"//tensorflow/python/ops:math_grad",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:nn_ops_gen",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:test",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "dumping_callback_test",
size = "medium",
srcs = ["dumping_callback_test.py"],
shard_count = 4,
tags = [
"no_windows", # TODO(b/142475891): Enable this test on Windows.
],
xla_enable_strict_auto_jit = False, # Node names are different with autojit
deps = [
":debug_events_reader",
":dumping_callback",
":dumping_callback_test_lib",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/eager:context",
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:test",
"//tensorflow/python/platform:tf_logging",
"//third_party/py/numpy",
"@absl_py//absl/testing:parameterized",
],
)
cuda_py_strict_test(
name = "debug_v2_ops_test",
size = "medium",
srcs = ["debug_v2_ops_test.py"],
tags = ["no_windows_gpu"],
deps = [
":debug_events_reader",
":debug_events_writer",
":dumping_callback_test_lib",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/eager:def_function",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:tensor_util",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:debug_ops_gen",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/platform:test",
"//third_party/py/numpy",
],
)
cuda_py_strict_test(
name = "debug_gradients_test",
size = "small",
srcs = ["debug_gradients_test.py"],
xla_enable_strict_auto_jit = False, # Node names are different with autojit
deps = [
":debug_data",
":debug_gradients",
":debug_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:tensor",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/ops:gradients_impl",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:test",
"//tensorflow/python/training:gradient_descent",
],
)
py_test(
name = "debug_utils_test",
size = "small",
srcs = ["debug_utils_test.py"],
strict_deps = True,
deps = [
":debug_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:resource_variable_ops",
"//tensorflow/python/ops:variable_v1",
"//tensorflow/python/platform:test",
"//third_party/py/numpy",
],
)
py_test(
name = "source_utils_test",
size = "small",
srcs = ["source_utils_test.py"],
strict_deps = True,
tags = [
"no_windows",
],
deps = [
":debug_data",
":debug_utils",
":source_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:resource_variable_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/ops:while_loop",
"//tensorflow/python/platform:test",
"//tensorflow/python/util:tf_inspect",
"//third_party/py/numpy",
],
)
py_test(
name = "source_remote_test",
size = "small",
srcs = ["source_remote_test.py"],
strict_deps = True,
tags = [
"no_windows",
"oss_serial",
],
deps = [
":grpc_debug_test_server",
":source_remote",
":source_utils",
"//tensorflow/core/debug:debug_service_proto_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:resource_variable_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:test",
"//tensorflow/python/util:tf_inspect",
],
)
py_test(
name = "profiling_test",
size = "small",
srcs = ["profiling_test.py"],
strict_deps = True,
deps = [
":profiling",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:test",
],
)
py_library(
name = "session_debug_testlib",
srcs = ["session_debug_testlib.py"],
strict_deps = True,
deps = [
":debug_data",
":debug_graphs",
":debug_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:cond",
"//tensorflow/python/ops:control_flow_ops",
"//tensorflow/python/ops:data_flow_ops",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:parsing_ops",
"//tensorflow/python/ops:rnn",
"//tensorflow/python/ops:rnn_cell_impl",
"//tensorflow/python/ops:state_ops",
"//tensorflow/python/ops:tensor_array_grad",
"//tensorflow/python/ops:variable_v1",
"//tensorflow/python/ops:variables",
"//tensorflow/python/ops:while_loop",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:test",
"//tensorflow/python/training:gradient_descent",
"//third_party/py/numpy",
],
)
py_library(
name = "debug_service_pb2_grpc",
srcs = ["debug_service_pb2_grpc.py"],
strict_deps = True,
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/core/debug:debug_service_proto_py",
],
)
py_library(
name = "grpc_debug_server",
srcs = ["grpc_debug_server.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
":debug_graphs",
":debug_service_pb2_grpc",
"//tensorflow/core:protos_all_py",
"//tensorflow/core/debug:debug_service_proto_py",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/util:compat",
],
)
py_library(
name = "grpc_debug_test_server",
srcs = ["grpc_debug_test_server.py"],
strict_deps = True,
deps = [
":debug_data",
":debug_utils",
":grpc_debug_server",
"//tensorflow/core:protos_all_py",
"//tensorflow/core/debug:debug_service_proto_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:errors",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/ops:variables",
"//tensorflow/python/util:compat",
"@pypi//portpicker",
],
)
cuda_py_strict_test(
name = "debug_grappler_test",
size = "small",
srcs = ["debug_grappler_test.py"],
xla_enable_strict_auto_jit = False, # Tests TF:Classic implementation.
deps = [
":debug_data",
":debug_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:variable_v1",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:test",
],
)
cuda_py_strict_test(
name = "session_debug_file_test",
size = "small",
srcs = ["session_debug_file_test.py"],
tags = ["notsan"],
xla_enable_strict_auto_jit = False, # Node names are different with autojit
deps = [
":debug_data",
":debug_utils",
":session_debug_testlib",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:variable_v1",
"//tensorflow/python/platform:test",
],
)
cuda_py_strict_test(
name = "debug_graph_reconstruction_test",
size = "small",
srcs = ["debug_graph_reconstruction_test.py"],
xla_enable_strict_auto_jit = False, # Node names are different with autojit
deps = [
":debug_data",
":debug_graphs",
":debug_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/ops:cond",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/ops:while_loop",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/training:gradient_descent",
],
)
cuda_py_strict_test(
name = "session_debug_multi_gpu_test",
size = "small",
srcs = ["session_debug_multi_gpu_test.py"],
tags = [
"no_windows", # TODO(b/184424727): Re-enable this.
"no_windows_gpu",
],
xla_enable_strict_auto_jit = False, # Node names are different with autojit
deps = [
":debug_data",
":debug_utils",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:device_lib",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:for_generated_wrappers",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/lib/io:file_io",
"//tensorflow/python/ops:math_ops",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:test",
],
)
tensorflow/python/debug/lib/__init__.py
View File
tensorflow/python/debug/lib/check_numerics_callback.py
+469
View File
@@ -0,0 +1,469 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Eager-graph unified check numerics callback."""
import collections
import threading
import numpy as np
from tensorflow.core.protobuf import debug_event_pb2
from tensorflow.python.debug.lib import op_callbacks_common
from tensorflow.python.debug.lib import source_utils
from tensorflow.python.eager import monitoring
from tensorflow.python.framework import op_callbacks
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_debug_ops
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import compat
from tensorflow.python.util import object_identity
from tensorflow.python.util.tf_export import tf_export
# Many ops have benign NaN outputs, and running them with check_numerics
# on will create unwanted errors
# TODO(b/142497024): Replace this allowlist with function decorators in the ops
IGNORE_OP_OUTPUTS = (
# For FusedBatchNorm, if the input tensor is empty then batch_mean and
# batch_variance will be NaN. reserve_space holds intermediate values
# derived from batch_mean and batch_variance used for gradient calculation
(b"FusedBatchNorm", 1), # batch_mean
(b"FusedBatchNorm", 2), # batch_variance
(b"FusedBatchNorm", 3), # reserve_space_1
(b"FusedBatchNorm", 4), # reserve_space_2
# Same as above
(b"FusedBatchNormV2", 1), # batch_mean
(b"FusedBatchNormV2", 2), # batch_variance
(b"FusedBatchNormV2", 3), # reserve_space_1
(b"FusedBatchNormV2", 4), # reserve_space_2
# Same as above, but reserve_space_3 holds additional intermediate values
(b"FusedBatchNormV3", 1), # batch_mean
(b"FusedBatchNormV3", 2), # batch_variance
(b"FusedBatchNormV3", 3), # reserve_space_1
(b"FusedBatchNormV3", 4), # reserve_space_2
(b"FusedBatchNormV3", 5), # reserve_space_3
)
# Some frequently used ops are generally safe and we can skip them to reduce
# overhead. NOTE: This list is compiled by observing operations called by
# models in practice and is not a comprehensive list of safe operations.
SAFE_OPS = (
b"Concat",
b"ConcatV2",
b"ExpandDims",
b"Fill",
b"Gather",
b"Maximum",
b"Minimum",
b"Reshape",
b"Slice",
b"Squeeze",
b"Stack",
b"StridedSlice",
b"StridedSliceGrad",
b"TensorListConcatV2",
b"TensorListGather",
b"TensorListGetItem",
b"TensorListPopBack",
b"TensorListStack",
b"Transpose",
b"Unpack",
)
_state = threading.local()
_check_numerics_callback_create_counter = monitoring.Counter(
"/tensorflow/api/python/debugging/check_numerics_callback_create_counter",
"Counter for number of times the check_numerics op callback is created.")
def limit_string_length(string, max_len=50):
"""Limit the length of input string.
Args:
string: Input string.
max_len: (int or None) If int, the length limit. If None, no limit.
Returns:
Possibly length-limited string.
"""
if max_len is None or len(string) <= max_len:
return string
else:
return "..." + string[len(string) - max_len:]
# A dictionary that supports looking up the original input tensor names.
_CHECK_NUMERICS_INPUT_LOOKUP = collections.defaultdict(dict)
def _maybe_lookup_original_input_tensor(graph, tensor):
if (graph and
graph in _CHECK_NUMERICS_INPUT_LOOKUP and
tensor.name in _CHECK_NUMERICS_INPUT_LOOKUP[graph]):
return _CHECK_NUMERICS_INPUT_LOOKUP[graph][tensor.name]
else:
return tensor
def get_check_numerics_error_message(slot,
num_outputs,
op_type,
tensor,
inputs,
graph=None,
traceback=None,
stack_height_limit=30,
path_length_limit=50):
"""Create a meaningful and user-friendly error message about offending tensor.
The error message reveals the following info about the op that outputs
NaN/Infinity: dtype, shape (to the extent known at graph-construction time),
input tensors, stack trace for op creation (if is graph mode).
Args:
slot: (int) slot index of the tensor output.
num_outputs: (int) total number of outputs of the op.
op_type: (str) Type of the that generates `tensor`.
tensor: (Tensor) the offending tensor, i.e., the tensor that contains
Infinities or NaNs.
inputs: (array of Tensor) inputs to the op that generates `tensor`.
graph: (tf.Graph) the graph object that `tensor` belongs to. Available only
under graph mode.
traceback: (list of trace frames) the stack trace of the op's creation.
Available only under graph model.
stack_height_limit: (int or None) If int, limit to the height of the stack
trace printed in the error message. If None, no limit to the height.
path_length_limit: (int or None) Length limit for file paths included in the
formatted stack trace.
Returns:
(str) A formatted error message.
"""
eager_vs_graph_qualifier = "graph" if graph else "eagerly-executing"
message = "\n"
message += (
"\n!!! Detected Infinity or NaN in output %d of "
"%s op \"%s\" (# of outputs: %d) !!!\n" %
(slot, eager_vs_graph_qualifier, op_type, num_outputs))
message += " dtype: %s\n" % tensor.dtype
message += " shape: %s\n" % (tensor.shape,)
if not graph:
# This is an eager tensor. We can get its numpy value and count
# NaNs and Infs.
is_inf = np.isinf(tensor)
num_neg_inf = np.sum(np.logical_and(np.less(tensor, 0.), is_inf))
num_pos_inf = np.sum(np.logical_and(np.greater(tensor, 0.), is_inf))
num_nan = np.sum(np.isnan(tensor))
if num_neg_inf > 0:
message += " # of -Inf elements: %s\n" % num_neg_inf
if num_pos_inf > 0:
message += " # of +Inf elements: %s\n" % num_pos_inf
if num_nan:
message += " # of +NaN elements: %s\n" % num_nan
if len(inputs) > 1:
message += "\n Input tensors (%d):\n" % len(inputs)
for slot, input_tensor in enumerate(inputs):
message += " %d: %s\n" % (
slot, _maybe_lookup_original_input_tensor(graph, input_tensor))
elif len(inputs) == 1:
message += "\n Input tensor: %s\n" % (
_maybe_lookup_original_input_tensor(graph, inputs[0]))
if graph and hasattr(graph, "name") and graph.name:
message += " Graph name: \"%s\"\n" % graph.name
# Format the stack trace for the op's creation. We omit files that
# belong to tensorflow itself.
if graph and traceback:
message += (
"\n Stack trace of op's creation (\"->\": inferred user code):\n")
if stack_height_limit is not None and len(traceback) > stack_height_limit:
num_omitted_frames = len(traceback) - stack_height_limit
message += " + ... (Omitted %d frames)\n" % num_omitted_frames
for filepath, lineno, function_name, source_line in traceback[
-stack_height_limit:]:
user_code_indicator = " "
if not source_utils.guess_is_tensorflow_py_library(filepath):
user_code_indicator = " -> "
message += " + %s (L%d) %s\n" % (
limit_string_length(filepath, path_length_limit), lineno,
function_name)
if source_line is not None:
message += "%s| %s\n" % (user_code_indicator, source_line)
message += "\n"
return message
def _debug_summary(x):
return gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(
debug_event_pb2.TensorDebugMode.REDUCE_INF_NAN_THREE_SLOTS))
class CheckNumericsCallback(object):
"""Wrapper for the numerics-checking callback for thread locality."""
def __init__(self, stack_height_limit, path_length_limit):
self._stack_height_limit = stack_height_limit
self._path_length_limit = path_length_limit
# A dict mapping Placeholder tensors to their instrumenting debug tensors.
# Used only under V1 graph mode, where we can't rely on auto control
# dependency to execute the debug tensors and hence need to attach the debug
# tensors as control dependencies of the ops that consume the Placeholder.
self._placeholder_to_debug_tensor = (
object_identity.ObjectIdentityDictionary())
def callback(self,
op_type,
inputs,
attrs,
outputs,
op_name=None,
graph=None):
"""Eager-function unified callback for checking numerics."""
del attrs, op_name # Unused
op_type_bytes = compat.as_bytes(op_type)
is_v1_graph_mode = not ops.executing_eagerly_outside_functions()
if (op_type_bytes in op_callbacks_common.OP_CALLBACK_SKIP_OPS or
op_type_bytes in SAFE_OPS):
return None
if graph:
# Under graph mode. Insert check_numerics op.
instrumented_outputs = []
if is_v1_graph_mode:
for input_tensor in inputs:
if input_tensor in self._placeholder_to_debug_tensor and outputs:
outputs[0].op._add_control_input( # pylint: disable=protected-access
self._placeholder_to_debug_tensor[input_tensor].op)
for slot, output in enumerate(outputs):
if (output.dtype.is_floating and
(op_type_bytes, slot) not in IGNORE_OP_OUTPUTS):
checked_output = array_ops.check_numerics_v2(
# TF v2 has automatic control dependencies added to stateful async
# ops, which allows us to run check_numerics asynchronously.
# In the above case we use debug_summary to reduce all output
# tensors asynchronously from the op being checked and then
# process the tensor summary with check_numerics.
output if is_v1_graph_mode else _debug_summary(output),
get_check_numerics_error_message(
slot,
len(outputs),
op_type,
output,
inputs,
graph=graph,
traceback=output.op.traceback,
stack_height_limit=self._stack_height_limit,
path_length_limit=self._path_length_limit))
_CHECK_NUMERICS_INPUT_LOOKUP[graph][checked_output.name] = output
instrumented_outputs.append(self._get_output_tensor(
op_type_bytes, output, checked_output, is_v1_graph_mode))
else:
instrumented_outputs.append(output)
return instrumented_outputs
else:
if op_type_bytes == b"CheckNumericsV2":
# TODO(b/140334369): Remove this special casing logic once op_callback.
# automatically prevents infinite recursion in eager mode.
return None
# Under eager mode. Eagerly execute check_numerics op.
for slot, output in enumerate(outputs):
if (output.dtype.is_floating and
(op_type_bytes, slot) not in IGNORE_OP_OUTPUTS):
array_ops.check_numerics_v2(
output,
get_check_numerics_error_message(
slot, len(outputs), op_type, output, inputs,
stack_height_limit=self._stack_height_limit,
path_length_limit=self._path_length_limit))
def _get_output_tensor(self,
op_type,
tensor,
checked_tensor,
is_v1_graph_mode):
"""Determine what tensor to output from callback.
Args:
op_type: Type of the op that outputs the original symbolic tensor, as
`bytes`.
tensor: The original output symbolic tensor.
checked_tensor: The debugger-instrumented, numerics-checking tensor.
is_v1_graph_mode: Whether the debugged proggram is running under V1 graph
mode.
Returns:
A symbolic tensor to be returned by the dumping op_callback.
"""
if is_v1_graph_mode:
# Placeholders need special treatment under V1 graph mode. The
# callback can't simply override the Placeholder tensor to the debug
# tensor, as that would cause the Placeholder op to lack a value.
# The debug tensor is remembered and will be attached as control
# inputs to ops that consumer the Placeholders later.
if op_type == b"Placeholder":
self._placeholder_to_debug_tensor[tensor] = checked_tensor
return tensor
else:
return checked_tensor
else:
# Under non-v1 graph mode, rely on auto control dependency to run the
# checked tensor.
return tensor
@tf_export("debugging.enable_check_numerics")
def enable_check_numerics(stack_height_limit=30,
path_length_limit=50):
r"""Enable tensor numerics checking in an eager/graph unified fashion.
The numerics checking mechanism will cause any TensorFlow eager execution or
graph execution to error out as soon as an op's output tensor contains
infinity or NaN.
This method is idempotent. Calling it multiple times has the same effect
as calling it once.
This method takes effect only on the thread in which it is called.
When a op's float-type output tensor contains any Infinity or NaN, an
`tf.errors.InvalidArgumentError` will be thrown, with an error message that
reveals the following information:
- The type of the op that generated the tensor with bad numerics.
- Data type (dtype) of the tensor.
- Shape of the tensor (to the extent known at the time of eager execution
or graph construction).
- Name of the containing graph (if available).
- (Graph mode only): The stack trace of the intra-graph op's creation,
with a stack-height limit and a path-length limit for visual clarity.
The stack frames that belong to the user's code (as opposed to
tensorflow's internal code) are highlighted with a text arrow ("->").
- (Eager mode only): How many of the offending tensor's elements are
`Infinity` and `NaN`, respectively.
Once enabled, the check-numerics mechanism can be disabled by using
`tf.debugging.disable_check_numerics()`.
Example usage:
1. Catching infinity during the execution of a `tf.function` graph:
```py
import tensorflow as tf
tf.debugging.enable_check_numerics()
@tf.function
def square_log_x_plus_1(x):
v = tf.math.log(x + 1)
return tf.math.square(v)
x = -1.0
# When the following line runs, a function graph will be compiled
# from the Python function `square_log_x_plus_1()`. Due to the
# `enable_check_numerics()` call above, the graph will contain
# numerics checking ops that will run during the function graph's
# execution. The function call generates an -infinity when the Log
# (logarithm) op operates on the output tensor of the Add op.
# The program errors out at this line, printing an error message.
y = square_log_x_plus_1(x)
z = -y
```
2. Catching NaN during eager execution:
```py
import numpy as np
import tensorflow as tf
tf.debugging.enable_check_numerics()
x = np.array([[0.0, -1.0], [4.0, 3.0]])
# The following line executes the Sqrt op eagerly. Due to the negative
# element in the input array, a NaN is generated. Due to the
# `enable_check_numerics()` call above, the program errors immediately
# at this line, printing an error message.
y = tf.math.sqrt(x)
z = tf.matmul(y, y)
```
NOTE: If your code is running on TPUs, be sure to call
`tf.config.set_soft_device_placement(True)` before calling
`tf.debugging.enable_check_numerics()` as this API uses automatic outside
compilation on TPUs. For example:
```py
tf.config.set_soft_device_placement(True)
tf.debugging.enable_check_numerics()
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')
strategy = tf.distribute.TPUStrategy(resolver)
with strategy.scope():
# ...
```
Args:
stack_height_limit: Limit to the height of the printed stack trace.
Applicable only to ops in `tf.function`s (graphs).
path_length_limit: Limit to the file path included in the printed stack
trace. Applicable only to ops in `tf.function`s (graphs).
"""
if not hasattr(_state, "check_numerics_callback"):
_state.check_numerics_callback = CheckNumericsCallback(
stack_height_limit, path_length_limit)
op_callbacks.add_op_callback(_state.check_numerics_callback.callback)
logging.info(
"Enabled check-numerics callback in thread %s",
threading.current_thread().name)
_check_numerics_callback_create_counter.get_cell().increase_by(1)
@tf_export("debugging.disable_check_numerics")
def disable_check_numerics():
"""Disable the eager/graph unified numerics checking mechanism.
This method can be used after a call to `tf.debugging.enable_check_numerics()`
to disable the numerics-checking mechanism that catches infinity and NaN
values output by ops executed eagerly or in tf.function-compiled graphs.
This method is idempotent. Calling it multiple times has the same effect
as calling it once.
This method takes effect only on the thread in which it is called.
"""
if not hasattr(_state, "check_numerics_callback"):
return
try:
op_callbacks.remove_op_callback(_state.check_numerics_callback.callback)
delattr(_state, "check_numerics_callback")
logging.info(
"Disabled check-numerics callback in thread %s",
threading.current_thread().name)
except KeyError:
# Tolerate disabling the check numerics callback without
# enable_check_numerics() being called first.
pass
tensorflow/python/debug/lib/check_numerics_callback_test.py
+434
View File
@@ -0,0 +1,434 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
import re
import numpy as np
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.debug.lib import check_numerics_callback
from tensorflow.python.eager import backprop
from tensorflow.python.eager import context
from tensorflow.python.eager import def_function
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_grad # pylint: disable=unused-import
from tensorflow.python.ops import custom_gradient
from tensorflow.python.ops import gen_nn_ops
from tensorflow.python.ops import gradient_checker_v2
from tensorflow.python.ops import math_grad # pylint: disable=unused-import
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import googletest
from tensorflow.python.platform import test
class LimitStringLengthTest(test_util.TensorFlowTestCase):
def testLimitStringLengthWithExplicitLimit(self):
self.assertEqual(
check_numerics_callback.limit_string_length("", max_len=2), "")
self.assertEqual(
check_numerics_callback.limit_string_length("e", max_len=2), "e")
self.assertEqual(
check_numerics_callback.limit_string_length("de", max_len=2), "de")
self.assertEqual(
check_numerics_callback.limit_string_length("abcde", max_len=2),
"...de")
def testLimitStringLengthWithNoLimit(self):
self.assertEqual(check_numerics_callback.limit_string_length(
"A" * 100 + "B", max_len=None), "A" * 100 + "B")
self.assertEqual(
check_numerics_callback.limit_string_length("", max_len=None), "")
def testLimitStringLengthWithDefaultLimit(self):
self.assertEqual(
check_numerics_callback.limit_string_length("A" * 50 + "B"),
"..." + "A" * 49 + "B")
class CheckNumericsCallbackTest(test_util.TensorFlowTestCase):
def tearDown(self):
check_numerics_callback.disable_check_numerics()
super(CheckNumericsCallbackTest, self).tearDown()
def testCallingDisableCheckNumericsWithoutEnablingFirstIsTolerated(self):
check_numerics_callback.disable_check_numerics()
def testNoCatchEagerOpExecution(self):
"""Test running multiple steps of eager execution without Inf/NaN."""
check_numerics_callback.enable_check_numerics()
x = constant_op.constant([2.0, 3.0])
y = constant_op.constant([1.0, 0.0])
self.assertAllClose((x + y) * (x - y), [3.0, 9.0])
@test_util.run_in_graph_and_eager_modes
def testDatasetMapHealthyResults(self):
check_numerics_callback.enable_check_numerics()
tensor = constant_op.constant(
[0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0])
def map_fn(x):
return math_ops.log(math_ops.square(x) + 1)
dataset = dataset_ops.Dataset.from_tensor_slices(tensor).batch(2).map(
map_fn)
@def_function.function
def get_batches():
iterator = iter(dataset)
return [next(iterator), next(iterator)]
batches = self.evaluate(get_batches())
self.assertLen(batches, 2)
self.assertAllClose(batches[0], np.log([1.25, 2]))
self.assertAllClose(batches[1], np.log([3.25, 5]))
@test_util.run_in_graph_and_eager_modes
def testGraphModeUsesCorrectPathLengthAndStackHeightLimits(self):
check_numerics_callback.enable_check_numerics(
stack_height_limit=123, path_length_limit=1200)
@def_function.function
def add_fn(x, y):
return x + y
fake_get_check_numerics_error_message = test.mock.MagicMock(
return_value="dummy_message")
with test.mock.patch.object(check_numerics_callback,
"get_check_numerics_error_message",
fake_get_check_numerics_error_message):
x = constant_op.constant(2.0)
y = constant_op.constant(3.0)
self.assertAllClose(self.evaluate(add_fn(x, y)), 5.0)
(_, call_kwargs) = fake_get_check_numerics_error_message.call_args
self.assertEqual(call_kwargs["stack_height_limit"], 123)
self.assertEqual(call_kwargs["path_length_limit"], 1200)
class CheckNumericsCallbackUnhealthyTest(test_util.TensorFlowTestCase):
"""Test for cases in which enable_check_numerics() catches infs or nans."""
def tearDown(self):
check_numerics_callback.disable_check_numerics()
super(CheckNumericsCallbackUnhealthyTest, self).tearDown()
def _assertRaisesInvalidArgumentErrorAndGetMessage(self, func):
caught = None
try:
func()
except errors.InvalidArgumentError as error:
caught = error
self.assertTrue(caught, "Failed to catch expected InvalidArgumentError")
return caught.message
def testCatchEagerOpFloat32Inf(self):
"""Test catching Infinity in eager op execution: float32."""
check_numerics_callback.enable_check_numerics()
x = constant_op.constant([2.0, 3.0])
y = constant_op.constant([1.0, 0.0])
message = self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: x / y)
# Check the content of the error message.
self.assertTrue(re.search(r"eagerly-executing op.*\"RealDiv\"", message))
self.assertTrue(re.search(r"dtype.*float32", message))
self.assertIn("shape: (2,)\n", message)
self.assertIn("# of +Inf elements: 1\n", message)
self.assertIn("0: %s" % x, message)
self.assertIn("1: %s" % y, message)
def testEnableCheckNumericsIsIdempotent(self):
"""Two calls to enable_check_numerics() have same effect as one call."""
check_numerics_callback.enable_check_numerics()
check_numerics_callback.enable_check_numerics()
x = constant_op.constant([2.0, 3.0])
y = constant_op.constant([1.0, 0.0])
message = self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: x / y)
# Check the content of the error message.
self.assertTrue(re.search(r"eagerly-executing op.*\"RealDiv\"", message))
self.assertTrue(re.search(r"dtype.*float32", message))
self.assertIn("shape: (2,)\n", message)
self.assertIn("# of +Inf elements: 1\n", message)
self.assertIn("0: %s" % x, message)
self.assertIn("1: %s" % y, message)
def testCatchEagerOpFloat16NaN(self):
"""Test catching Infinity in eager op execution: float16."""
check_numerics_callback.enable_check_numerics()
def log1p(x):
y = 1.0 + x
return math_ops.log(y)
x = constant_op.constant([[-1.0]], dtype=dtypes.float16)
message = self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: log1p(x))
# Check the content of the error message.
self.assertTrue(re.search(r"eagerly-executing op.*\"Log\"", message))
self.assertTrue(re.search(r"dtype.*float16", message))
self.assertIn("shape: (1, 1)\n", message)
self.assertIn("# of -Inf elements: 1\n", message)
self.assertTrue(re.search(r"Input tensor.*0\.", message))
@test_util.enable_eager_op_as_function
def testCatchEagerOpFloat16NaNWithEagerOpAsFunctionEnabled(self):
self.testCatchEagerOpFloat16NaN()
@test_util.run_in_graph_and_eager_modes
def testCatchFunctionOpInfFloat64(self):
"""Test catching infinites generated in a FuncGraph."""
check_numerics_callback.enable_check_numerics()
@def_function.function
def divide_sum_with_diff(x, y):
w1 = x + y
w2 = x - y
u = w1 / w2
return u * 2.0
x = constant_op.constant(2.0, dtype=dtypes.float64)
y = constant_op.constant(2.0, dtype=dtypes.float64)
message = self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: self.evaluate(divide_sum_with_diff(x, y)))
# Check the content of the error message.
self.assertTrue(re.search(r"graph op.*\"RealDiv\"", message))
self.assertTrue(re.search(r"dtype.*float64", message))
self.assertIn("shape: ()\n", message)
self.assertIn("Input tensors (2):", message)
# Check that the correct input ops are printed.
self.assertTrue(re.search(r"0:.*Tensor.*add:0", message))
self.assertTrue(re.search(r"1:.*Tensor.*sub:0", message))
# Check that the correct line for op creation is printed.
self.assertTrue(re.search(r"Stack trace of op's creation", message))
self.assertIn("divide_sum_with_diff", message)
@test_util.run_in_graph_and_eager_modes
@test_util.disable_xla(
"TODO(b/141100809): XLA has no way to assert inside of a kernel.")
def testControlFlowGraphWithNaNBFloat16(self):
"""Test catching bfloat16 NaNs in a control-flow-v2 FuncGraph."""
check_numerics_callback.enable_check_numerics()
@def_function.function
def my_conditional(x):
if math_ops.less(math_ops.reduce_sum(x), 0.0):
return math_ops.log(x)
else:
return math_ops.log(-x)
x = constant_op.constant([1.0, 2.0, 3.0], dtype=dtypes.bfloat16)
message = self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: self.evaluate(my_conditional(x)))
# Check the content of the error message.
self.assertTrue(re.search(r"graph op.*\"Log\"", message))
self.assertTrue(re.search(r"dtype.*bfloat16", message))
self.assertIn("shape: (3,)\n", message)
# Check that the correct input op is printed.
self.assertTrue(re.search(r"Input tensor.*Tensor.*Neg", message))
# Check that the correct line for op creation is printed.
self.assertTrue(re.search(r"Stack trace of op's creation", message))
self.assertIn("my_conditional", message)
@test_util.run_in_graph_and_eager_modes
@test_util.disable_xla(
"There is a small inconsistency in the step at which overflow happens: "
"128 (without XLA) and 127 (with XLA).")
@test_util.disable_tfrt("b/177261532: TFRT cannot detect overflow yet.")
def testOverflowInTfFunction(self):
"""Test catching Infinity caused by overflow in a tf.function with while."""
check_numerics_callback.enable_check_numerics()
@def_function.function
def accumulation_function(counter, lim, accum):
while math_ops.less(counter, lim):
accum.assign(accum * 2.0)
counter.assign_add(1)
return 1
counter = variables.Variable(0, dtype=dtypes.int32)
# Repeated `* 2.0` overflows a float32 tensor in 128 steps. So the
# 1000-step limit is sufficient.
lim = constant_op.constant(1000, dtype=dtypes.int32)
accum = variables.Variable(1.0)
if not context.executing_eagerly():
self.evaluate([counter.initializer, accum.initializer])
caught = None
try:
self.evaluate(accumulation_function(counter, lim, accum))
except (errors.InvalidArgumentError, errors.InternalError) as error:
caught = error
self.assertTrue(caught, "Failed to catch expected error")
message = caught.message
# With XLA, overflow might happen at step 127 instead of 128.
self.assertIn(self.evaluate(counter), (127, 128))
# Check the content of the error message.
# The overflow to +Infinity happens during the `* 2.0` operation.
self.assertTrue(re.search(r"graph op.*\"Mul\"", message))
self.assertTrue(re.search(r"dtype.*float32", message))
self.assertIn("shape: ()\n", message)
# Check that the correct input op is printed.
self.assertIn("Input tensors (2):", message)
# Check that the correct input ops are printed.
self.assertTrue(re.search(r"0:.*Tensor.*ReadVariableOp:0", message))
self.assertTrue(re.search(r"1:.*Tensor.*mul/y:0", message))
# Check that the correct line for op creation is printed.
self.assertTrue(re.search(r"Stack trace of op's creation", message))
self.assertIn("accumulation_function", message)
@test_util.run_in_graph_and_eager_modes
def testNanInConstIsCaptured(self):
check_numerics_callback.enable_check_numerics()
v = variables.Variable(3.0, dtype=dtypes.float32)
@def_function.function
def add_a_bad_constant(x):
c = constant_op.constant(np.nan)
return x + c
if not context.executing_eagerly():
self.evaluate(v.initializer)
message = self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: self.evaluate(add_a_bad_constant(v)))
self.assertTrue(re.search(r"graph op.*\"Const\"", message))
self.assertTrue(re.search(r"dtype:.*float32", message))
self.assertTrue(re.search(r"shape:.*\(\)", message))
self.assertTrue(re.search(r"Graph name:.*add_a_bad_constant", message))
@test_util.run_in_graph_and_eager_modes
def testCatchInfinityInDatasetMapFunction(self):
"""Test that callback catches NaN in a tf.dataset map function."""
check_numerics_callback.enable_check_numerics()
def generate_nan(x):
"""Intentionally generates NaNs by taking log of negative number."""
casted_x = math_ops.cast(x, dtypes.float32)
return math_ops.log([[-1.0, 1.0], [3.0, 5.0]]) + casted_x
dataset = dataset_ops.Dataset.range(10).map(generate_nan)
iterator = dataset_ops.make_one_shot_iterator(dataset)
message = self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: self.evaluate(iterator.get_next()))
# Check the content of the error message.
self.assertTrue(re.search(r"graph op.*\"Log\"", message))
self.assertTrue(re.search(r"dtype.*float32", message))
self.assertIn("shape: (2, 2)\n", message)
self.assertTrue(re.search(r"Input tensor.*Tensor.*Log/x:0", message))
self.assertIn("generate_nan", message)
@test_util.run_in_graph_and_eager_modes
def testCustomGradientWithNaNWithTfFunction(self):
"""Test that callback catches NaN in a gradient function during backprop."""
check_numerics_callback.enable_check_numerics()
@custom_gradient.custom_gradient
def func_with_bad_grad(x):
output = math_ops.sin(x)
@def_function.function
def grad(dy):
# `dy` will come in as 1.0. Taking log of -1.0 leads to NaN.
return math_ops.log(-dy)
return output, grad
x = constant_op.constant(-2.0, dtype=dtypes.float16)
def f(x):
return func_with_bad_grad(x)
message = self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: gradient_checker_v2.compute_gradient(f, [x]))
# Check the content of the error message.
self.assertTrue(re.search(r"graph op.*\"Log\"", message))
self.assertTrue(re.search(r"dtype.*float16", message))
if context.executing_eagerly():
self.assertIn("shape: ()\n", message)
self.assertTrue(re.search(r"Input tensor.*Tensor.*Neg:0", message))
self.assertIn("grad", message)
@test_util.run_in_graph_and_eager_modes
def testNestedFunctionGradientCall(self):
"""Catching inf in the inner nested tf.function during backprop."""
check_numerics_callback.enable_check_numerics()
x = constant_op.constant(1.0 - 1e-8, dtype=dtypes.float32)
@def_function.function
def asinp1(x):
# asin()'s gradient overflows at the value close to 1.0.
return math_ops.asin(x) + 1.0
@def_function.function
def loss(x):
return math_ops.square(asinp1(x))
with backprop.GradientTape() as tape:
tape.watch(x)
y = loss(x)
message = self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: self.evaluate(tape.gradient(y, x)))
self.assertTrue(re.search(r"gradient", message))
def testEagerModeUsesCorrectPathLengthAndStackHeightLimits(self):
check_numerics_callback.enable_check_numerics(
stack_height_limit=123, path_length_limit=1200)
fake_get_check_numerics_error_message = test.mock.MagicMock(
return_value="dummy_message")
with test.mock.patch.object(check_numerics_callback,
"get_check_numerics_error_message",
fake_get_check_numerics_error_message):
x = constant_op.constant(2.0)
y = constant_op.constant(0.0)
self._assertRaisesInvalidArgumentErrorAndGetMessage(
lambda: x / y) # Expected to generate an inf.
(_, call_kwargs) = fake_get_check_numerics_error_message.call_args
self.assertEqual(call_kwargs["stack_height_limit"], 123)
self.assertEqual(call_kwargs["path_length_limit"], 1200)
@test_util.run_in_graph_and_eager_modes
def testExpectedNaNOpOutputs(self):
"""Test calling operations with benign NaN output."""
check_numerics_callback.enable_check_numerics()
# Empty input tensor
x = constant_op.constant(1, dtype=dtypes.float32, shape=[0, 1, 1, 1])
scale = constant_op.constant([1], dtype=dtypes.float32)
offset = constant_op.constant([1], dtype=dtypes.float32)
# Calling fused_batch_norm with an empty input should output a NaN in the
# latter four outputs without triggering the check_numerics callback
batch_norm_res = gen_nn_ops._fused_batch_norm(
x=x, scale=scale, offset=offset, mean=[], variance=[])
_, batch_mean, batch_variance, _, _ = self.evaluate(batch_norm_res)
self.assertTrue(np.isnan(batch_mean.squeeze()))
self.assertTrue(np.isnan(batch_variance.squeeze()))
if __name__ == "__main__":
ops.enable_eager_execution()
googletest.main()
tensorflow/python/debug/lib/common.py
+83
View File
@@ -0,0 +1,83 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Common values and methods for TensorFlow Debugger."""
import collections
import json
GRPC_URL_PREFIX = "grpc://"
# A key for a Session.run() call.
RunKey = collections.namedtuple("RunKey", ["feed_names", "fetch_names"])
def get_graph_element_name(elem):
"""Obtain the name or string representation of a graph element.
If the graph element has the attribute "name", return name. Otherwise, return
a __str__ representation of the graph element. Certain graph elements, such as
`SparseTensor`s, do not have the attribute "name".
Args:
elem: The graph element in question.
Returns:
If the attribute 'name' is available, return the name. Otherwise, return
str(fetch).
"""
return elem.name if hasattr(elem, "name") else str(elem)
def get_flattened_names(feeds_or_fetches):
"""Get a flattened list of the names in run() call feeds or fetches.
Args:
feeds_or_fetches: Feeds or fetches of the `Session.run()` call. It maybe
a Tensor, an Operation or a Variable. It may also be nested lists, tuples
or dicts. See doc of `Session.run()` for more details.
Returns:
(list of str) A flattened list of fetch names from `feeds_or_fetches`.
"""
lines = []
if isinstance(feeds_or_fetches, (list, tuple)):
for item in feeds_or_fetches:
lines.extend(get_flattened_names(item))
elif isinstance(feeds_or_fetches, dict):
for key in feeds_or_fetches:
lines.extend(get_flattened_names(feeds_or_fetches[key]))
else:
# This ought to be a Tensor, an Operation or a Variable, for which the name
# attribute should be available. (Bottom-out condition of the recursion.)
lines.append(get_graph_element_name(feeds_or_fetches))
return lines
def get_run_key(feed_dict, fetches):
"""Summarize the names of feeds and fetches as a RunKey JSON string.
Args:
feed_dict: The feed_dict given to the `Session.run()` call.
fetches: The fetches from the `Session.run()` call.
Returns:
A JSON Array consisting of two items. They first items is a flattened
Array of the names of the feeds. The second item is a flattened Array of
the names of the fetches.
"""
return json.dumps(RunKey(get_flattened_names(feed_dict),
get_flattened_names(fetches)))
tensorflow/python/debug/lib/common_test.py
+58
View File
@@ -0,0 +1,58 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Unit tests for common values and methods of TensorFlow Debugger."""
import json
from tensorflow.python.debug.lib import common
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import test_util
from tensorflow.python.platform import googletest
class CommonTest(test_util.TensorFlowTestCase):
@test_util.run_v1_only("Relies on tensor name, which is unavailable in TF2")
def testOnFeedOneFetch(self):
a = constant_op.constant(10.0, name="a")
b = constant_op.constant(20.0, name="b")
run_key = common.get_run_key({"a": a}, [b])
loaded = json.loads(run_key)
self.assertItemsEqual(["a:0"], loaded[0])
self.assertItemsEqual(["b:0"], loaded[1])
@test_util.run_v1_only("Relies on tensor name, which is unavailable in TF2")
def testGetRunKeyFlat(self):
a = constant_op.constant(10.0, name="a")
b = constant_op.constant(20.0, name="b")
run_key = common.get_run_key({"a": a}, [a, b])
loaded = json.loads(run_key)
self.assertItemsEqual(["a:0"], loaded[0])
self.assertItemsEqual(["a:0", "b:0"], loaded[1])
@test_util.run_v1_only("Relies on tensor name, which is unavailable in TF2")
def testGetRunKeyNestedFetches(self):
a = constant_op.constant(10.0, name="a")
b = constant_op.constant(20.0, name="b")
c = constant_op.constant(30.0, name="c")
d = constant_op.constant(30.0, name="d")
run_key = common.get_run_key(
{}, {"set1": [a, b], "set2": {"c": c, "d": d}})
loaded = json.loads(run_key)
self.assertItemsEqual([], loaded[0])
self.assertItemsEqual(["a:0", "b:0", "c:0", "d:0"], loaded[1])
if __name__ == "__main__":
googletest.main()
tensorflow/python/debug/lib/debug_data.py
+1626
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/debug/lib/debug_data_test.py
+391
View File
@@ -0,0 +1,391 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tfdbg module debug_data."""
import os
import platform
import tempfile
import numpy as np
from tensorflow.core.framework import graph_pb2
from tensorflow.core.framework import tensor_pb2
from tensorflow.python.debug.lib import debug_data
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.platform import gfile
from tensorflow.python.platform import googletest
from tensorflow.python.platform import test
class DeviceNamePathConversionTest(test_util.TensorFlowTestCase):
def testDeviceNameToDevicePath(self):
self.assertEqual(
debug_data.METADATA_FILE_PREFIX
+ debug_data.DEVICE_TAG
+ ",job_ps,replica_1,task_2,cpu_0",
debug_data.device_name_to_device_path("/job:ps/replica:1/task:2/cpu:0"),
)
def testDevicePathToDeviceName(self):
self.assertEqual(
"/job:ps/replica:1/task:2/cpu:0",
debug_data.device_path_to_device_name(
debug_data.METADATA_FILE_PREFIX
+ debug_data.DEVICE_TAG
+ ",job_ps,replica_1,task_2,cpu_0"
),
)
class HasNanOrInfTest(test_util.TensorFlowTestCase):
def setUp(self):
self._dummy_datum = dummy_datum = debug_data.DebugTensorDatum(
"/foo", "bar_0_DebugIdentity_42"
)
def testNaN(self):
a = np.array([np.nan, np.nan, 7.0])
self.assertTrue(debug_data.has_inf_or_nan(self._dummy_datum, a))
def testInf(self):
a = np.array([np.inf, np.inf, 7.0])
self.assertTrue(debug_data.has_inf_or_nan(self._dummy_datum, a))
def testNanAndInf(self):
a = np.array([np.inf, np.nan, 7.0])
self.assertTrue(debug_data.has_inf_or_nan(self._dummy_datum, a))
def testNoNanOrInf(self):
a = np.array([0.0, 0.0, 7.0])
self.assertFalse(debug_data.has_inf_or_nan(self._dummy_datum, a))
def testEmpty(self):
a = np.array([])
self.assertFalse(debug_data.has_inf_or_nan(self._dummy_datum, a))
def testInconvertibleTensorProto(self):
self.assertFalse(
debug_data.has_inf_or_nan(
self._dummy_datum,
debug_data.InconvertibleTensorProto(
tensor_pb2.TensorProto(), initialized=False
),
)
)
self.assertFalse(
debug_data.has_inf_or_nan(
self._dummy_datum,
debug_data.InconvertibleTensorProto(
tensor_pb2.TensorProto(), initialized=True
),
)
)
def testDTypeComplexWorks(self):
a = np.array([1j, 3j, 3j, 7j], dtype=np.complex128)
self.assertFalse(debug_data.has_inf_or_nan(self._dummy_datum, a))
b = np.array([1j, 3j, 3j, 7j, np.nan], dtype=np.complex128)
self.assertTrue(debug_data.has_inf_or_nan(self._dummy_datum, b))
def testDTypeIntegerWorks(self):
a = np.array([1, 3, 3, 7], dtype=np.int16)
self.assertFalse(debug_data.has_inf_or_nan(self._dummy_datum, a))
def testDTypeStringGivesFalse(self):
"""isnan and isinf are not applicable to strings."""
a = np.array(["s", "p", "a", "m"])
self.assertFalse(debug_data.has_inf_or_nan(self._dummy_datum, a))
def testDTypeObjectGivesFalse(self):
dt = np.dtype([("spam", np.str_, 16), ("eggs", np.float64, (2,))])
a = np.array([("spam", (8.0, 7.0)), ("eggs", (6.0, 5.0))], dtype=dt)
self.assertFalse(debug_data.has_inf_or_nan(self._dummy_datum, a))
class DebugTensorDatumTest(test_util.TensorFlowTestCase):
def testDebugDatum(self):
dump_root = "/tmp/tfdbg_1"
debug_dump_rel_path = (
debug_data.METADATA_FILE_PREFIX
+ debug_data.DEVICE_TAG
+ ",job_localhost,replica_0,task_0,cpu_0"
+ "/ns1/ns2/node_a_1_2_DebugIdentity_1472563253536385"
)
datum = debug_data.DebugTensorDatum(dump_root, debug_dump_rel_path)
self.assertEqual("DebugIdentity", datum.debug_op)
self.assertEqual("ns1/ns2/node_a_1", datum.node_name)
self.assertEqual(2, datum.output_slot)
self.assertEqual("ns1/ns2/node_a_1:2", datum.tensor_name)
self.assertEqual(1472563253536385, datum.timestamp)
self.assertEqual("ns1/ns2/node_a_1:2:DebugIdentity", datum.watch_key)
self.assertEqual(
os.path.join(dump_root, debug_dump_rel_path), datum.file_path
)
self.assertEqual(
"{DebugTensorDatum (/job:localhost/replica:0/task:0/cpu:0) "
"%s:%d @ %s @ %d}"
% (datum.node_name, datum.output_slot, datum.debug_op, datum.timestamp),
str(datum),
)
self.assertEqual(
"{DebugTensorDatum (/job:localhost/replica:0/task:0/cpu:0) "
"%s:%d @ %s @ %d}"
% (datum.node_name, datum.output_slot, datum.debug_op, datum.timestamp),
repr(datum),
)
def testDumpSizeBytesIsNoneForNonexistentFilePath(self):
dump_root = "/tmp/tfdbg_1"
debug_dump_rel_path = "ns1/ns2/node_foo_1_2_DebugIdentity_1472563253536385"
datum = debug_data.DebugTensorDatum(dump_root, debug_dump_rel_path)
self.assertIsNone(datum.dump_size_bytes)
class DebugDumpDirTest(test_util.TensorFlowTestCase):
def setUp(self):
self._dump_root = tempfile.mkdtemp()
def tearDown(self):
# Tear down temporary dump directory.
file_io.delete_recursively(self._dump_root)
def _makeDataDirWithMultipleDevicesAndDuplicateNodeNames(self):
cpu_0_dir = os.path.join(
self._dump_root,
debug_data.METADATA_FILE_PREFIX
+ debug_data.DEVICE_TAG
+ ",job_localhost,replica_0,task_0,cpu_0",
)
gpu_0_dir = os.path.join(
self._dump_root,
debug_data.METADATA_FILE_PREFIX
+ debug_data.DEVICE_TAG
+ ",job_localhost,replica_0,task_0,device_GPU_0",
)
gpu_1_dir = os.path.join(
self._dump_root,
debug_data.METADATA_FILE_PREFIX
+ debug_data.DEVICE_TAG
+ ",job_localhost,replica_0,task_0,device_GPU_1",
)
os.makedirs(cpu_0_dir)
os.makedirs(gpu_0_dir)
os.makedirs(gpu_1_dir)
open(
os.path.join(cpu_0_dir, "node_foo_1_2_DebugIdentity_1472563253536386"),
"wb",
)
open(
os.path.join(gpu_0_dir, "node_foo_1_2_DebugIdentity_1472563253536385"),
"wb",
)
open(
os.path.join(gpu_1_dir, "node_foo_1_2_DebugIdentity_1472563253536387"),
"wb",
)
def testDebugDumpDir_nonexistentDumpRoot(self):
with self.assertRaisesRegex(IOError, "does not exist"):
debug_data.DebugDumpDir(tempfile.mkdtemp() + "_foo")
def testDebugDumpDir_invalidFileNamingPattern(self):
# File name with too few underscores should lead to an exception.
device_dir = os.path.join(
self._dump_root,
debug_data.METADATA_FILE_PREFIX
+ debug_data.DEVICE_TAG
+ ",job_localhost,replica_0,task_0,cpu_0",
)
os.makedirs(device_dir)
open(os.path.join(device_dir, "node1_DebugIdentity_1234"), "wb")
with self.assertRaisesRegex(
ValueError, "does not conform to the naming pattern"
):
debug_data.DebugDumpDir(self._dump_root)
def testDebugDumpDir_validDuplicateNodeNamesWithMultipleDevices(self):
self._makeDataDirWithMultipleDevicesAndDuplicateNodeNames()
graph_cpu_0 = graph_pb2.GraphDef()
node = graph_cpu_0.node.add()
node.name = "node_foo_1"
node.op = "FooOp"
node.device = "/job:localhost/replica:0/task:0/cpu:0"
graph_gpu_0 = graph_pb2.GraphDef()
node = graph_gpu_0.node.add()
node.name = "node_foo_1"
node.op = "FooOp"
node.device = "/job:localhost/replica:0/task:0/device:GPU:0"
graph_gpu_1 = graph_pb2.GraphDef()
node = graph_gpu_1.node.add()
node.name = "node_foo_1"
node.op = "FooOp"
node.device = "/job:localhost/replica:0/task:0/device:GPU:1"
dump_dir = debug_data.DebugDumpDir(
self._dump_root,
partition_graphs=[graph_cpu_0, graph_gpu_0, graph_gpu_1],
)
self.assertCountEqual(
[
"/job:localhost/replica:0/task:0/cpu:0",
"/job:localhost/replica:0/task:0/device:GPU:0",
"/job:localhost/replica:0/task:0/device:GPU:1",
],
dump_dir.devices(),
)
self.assertEqual(1472563253536385, dump_dir.t0)
self.assertEqual(3, dump_dir.size)
with self.assertRaisesRegex(ValueError, r"Invalid device name: "):
dump_dir.nodes("/job:localhost/replica:0/task:0/device:GPU:2")
self.assertCountEqual(
["node_foo_1", "node_foo_1", "node_foo_1"], dump_dir.nodes()
)
self.assertCountEqual(
["node_foo_1"],
dump_dir.nodes(device_name="/job:localhost/replica:0/task:0/cpu:0"),
)
def testDuplicateNodeNamesInGraphDefOfSingleDeviceRaisesException(self):
self._makeDataDirWithMultipleDevicesAndDuplicateNodeNames()
graph_cpu_0 = graph_pb2.GraphDef()
node = graph_cpu_0.node.add()
node.name = "node_foo_1"
node.op = "FooOp"
node.device = "/job:localhost/replica:0/task:0/cpu:0"
graph_gpu_0 = graph_pb2.GraphDef()
node = graph_gpu_0.node.add()
node.name = "node_foo_1"
node.op = "FooOp"
node.device = "/job:localhost/replica:0/task:0/device:GPU:0"
graph_gpu_1 = graph_pb2.GraphDef()
node = graph_gpu_1.node.add()
node.name = "node_foo_1"
node.op = "FooOp"
node.device = "/job:localhost/replica:0/task:0/device:GPU:1"
node = graph_gpu_1.node.add() # Here is the duplicate.
node.name = "node_foo_1"
node.op = "FooOp"
node.device = "/job:localhost/replica:0/task:0/device:GPU:1"
with self.assertRaisesRegex(ValueError, r"Duplicate node name on device "):
debug_data.DebugDumpDir(
self._dump_root,
partition_graphs=[graph_cpu_0, graph_gpu_0, graph_gpu_1],
)
def testDebugDumpDir_emptyDumpDir(self):
dump_dir = debug_data.DebugDumpDir(self._dump_root)
self.assertIsNone(dump_dir.t0)
self.assertEqual([], dump_dir.dumped_tensor_data)
def testDebugDumpDir_usesGfileGlob(self):
if platform.system() == "Windows":
self.skipTest("gfile.Glob is not used on Windows.")
self._makeDataDirWithMultipleDevicesAndDuplicateNodeNames()
def fake_gfile_glob(glob_pattern):
del glob_pattern
return []
with test.mock.patch.object(
gfile, "Glob", side_effect=fake_gfile_glob, autospec=True
) as fake:
debug_data.DebugDumpDir(self._dump_root)
expected_calls = [
test.mock.call(
os.path.join(
self._dump_root,
(
debug_data.METADATA_FILE_PREFIX
+ debug_data.CORE_METADATA_TAG
+ "*"
),
)
),
test.mock.call(
os.path.join(
self._dump_root,
(
debug_data.METADATA_FILE_PREFIX
+ debug_data.FETCHES_INFO_FILE_TAG
+ "*"
),
)
),
test.mock.call(
os.path.join(
self._dump_root,
(
debug_data.METADATA_FILE_PREFIX
+ debug_data.FEED_KEYS_INFO_FILE_TAG
+ "*"
),
)
),
test.mock.call(
os.path.join(
self._dump_root,
(
debug_data.METADATA_FILE_PREFIX
+ debug_data.DEVICE_TAG
+ "*"
),
)
),
]
fake.assert_has_calls(expected_calls, any_order=True)
def testValidationSucceedsOnDoubleSlashNodeName(self):
device_dir = os.path.join(
self._dump_root,
debug_data.device_name_to_device_path(
"/job:localhost/replica:0/task:0/cpu:0"
),
)
node_scope_dir = os.path.join(device_dir, "scope_A")
os.makedirs(node_scope_dir)
file_io.write_string_to_file(
os.path.join(node_scope_dir, "op_B_0_DebugIdentity_12345"), "dummy"
)
graph_def = graph_pb2.GraphDef()
node = graph_def.node.add()
# Previously double slash would have caused validation to fail. b/429335661
node.name = "scope_A//op_B"
node.op = "NoOp"
node.device = "/job:localhost/replica:0/task:0/cpu:0"
dump_dir = debug_data.DebugDumpDir(
self._dump_root, partition_graphs=[graph_def]
)
self.assertEqual(1, dump_dir.size)
self.assertIn("scope_A/op_B", dump_dir.nodes()[0])
if __name__ == "__main__":
googletest.main()
tensorflow/python/debug/lib/debug_events_monitors.py
+307
View File
@@ -0,0 +1,307 @@
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Monitors for Debug Events in the tfdbg2 format.
Monitors get access to graph-building- and execution-related data
objects as the DebugDataReader (see `debug_events_reader.py`) reads the
data in a continuous fashion, via a set of callbacks. This mechanism enables
hooking custom logic into the DebugEvent reading stream without the need for
any polling or iterating over the entire data held by DebugDataReader.
This module includes the following built-in hooks:
- InfNanMonitor: Monitors infinity and nan values in top-level execution and
intra-graph execution events.
When a monitor (subtype of `BaseMonitor`) is constructed with a DebugDataReader
as the first argument of the constructor call, the monitor is automatically
registered with the DebugDataReader. For example:
```py
debug_data_reader = debug_events_reader.DebugDataReader(dump_dir)
inf_nan_monitor = debug_events_monitors.InfNanMonitor(debug_data_reader)
debug_data_reader.update()
# `inf_nan_monitor`'s on_* methods will get called as the execution-related
# and other types of data are read by `debug_data_reader`.
```
"""
import numpy as np
from tensorflow.core.protobuf import debug_event_pb2
class BaseMonitor(object):
"""Base class for debug event data monitors."""
def __init__(self, debug_events_reader):
self._debug_data_reader = debug_events_reader
debug_events_reader._add_monitor(self) # pylint:disable=protected-access
def on_execution(self, execution_index, execution):
"""Monitor method for top-level execution events.
Return values (if any) are ignored by the associated DebugDataReader.
Args:
execution_index: The index of the top-level execution event, as an int.
execution: An Execution data object, for a top-level op or function
execution event.
"""
def on_graph_execution_trace(self,
graph_execution_trace_index,
graph_execution_trace):
"""Monitor method for intra-graph execution events.
Return values (if any) are ignored by the associated DebugDataReader.
Args:
graph_execution_trace_index: The index of the intra-graph execution
event, as an int.
graph_execution_trace: A GraphExecutionTrace data object, for an
intra-graph tensor event.
"""
# TODO(cais): Add more monitor methods such as on_graph_op_creation().
class InfNanAlert(object):
"""Alert for Infinity and NaN values."""
def __init__(self,
wall_time,
op_type,
output_slot,
size=None,
num_neg_inf=None,
num_pos_inf=None,
num_nan=None,
execution_index=None,
graph_execution_trace_index=None):
self._wall_time = wall_time
self._op_type = op_type
self._output_slot = output_slot
self._size = size
self._num_neg_inf = num_neg_inf
self._num_pos_inf = num_pos_inf
self._num_nan = num_nan
self._execution_index = execution_index
self._graph_execution_trace_index = graph_execution_trace_index
@property
def wall_time(self):
return self._wall_time
@property
def op_type(self):
return self._op_type
@property
def output_slot(self):
return self._output_slot
@property
def size(self):
return self._size
@property
def num_neg_inf(self):
return self._num_neg_inf
@property
def num_pos_inf(self):
return self._num_pos_inf
@property
def num_nan(self):
return self._num_nan
@property
def execution_index(self):
return self._execution_index
@property
def graph_execution_trace_index(self):
return self._graph_execution_trace_index
class InfNanMonitor(BaseMonitor):
"""Monitor for Infinity and NaN in tensor values."""
def __init__(self, debug_events_reader, limit=0):
super(InfNanMonitor, self).__init__(debug_events_reader)
self._limit = limit # Track only the first _ alert events, for efficiency.
self._alerts = []
def _check_full_tensor_value(self,
tensor_value,
wall_time,
op_type,
output_slot,
execution_index=None,
graph_execution_trace_index=None):
"""Check a full tensor value.
Appends to the list of alerts if any inf or nan is found in the full tensor
value.
Args:
tensor_value: The full tensor value as a `np.ndarray`.
wall_time: Wall timestamp for the execution event that generated the
tensor value.
op_type: Op type executed.
output_slot: The output slot of the op.
execution_index: Index to the top-level execution event.
graph_execution_trace_index: Index to the intra-graph execution trace
(if applicable.)
"""
size = np.size(tensor_value)
if not size or not np.issubdtype(tensor_value.dtype, np.floating):
return
is_inf = np.isinf(tensor_value)
num_neg_inf = np.count_nonzero(
np.logical_and(is_inf, np.less(tensor_value, 0.0)))
num_pos_inf = np.count_nonzero(
np.logical_and(is_inf, np.greater(tensor_value, 0.0)))
num_nan = np.count_nonzero(np.isnan(tensor_value))
if num_neg_inf or num_pos_inf or num_nan:
self._alerts.append(InfNanAlert(
wall_time,
op_type,
output_slot,
size=size,
num_neg_inf=num_neg_inf,
num_pos_inf=num_pos_inf,
num_nan=num_nan,
execution_index=execution_index,
graph_execution_trace_index=graph_execution_trace_index))
def _check_debug_tensor_value(self,
tensor_debug_mode,
debug_tensor_value,
wall_time,
op_type,
output_slot,
execution_index=None,
graph_execution_trace_index=None):
"""Check for bad numerical values based on debug summary of tensor value.
If tensor_debug_mode is one in which debug_tensor_value does not carry
information about the presence or count of inf / nan values (e.g., SHAPE),
this method is a no-op.
When infs and/or nans are found, `InfNanAlert` objects are created and
appended to `self._alerts`.
Args:
tensor_debug_mode: TensorDebugMode proto enum.
debug_tensor_value: Debug tensor value as a list of numbers.
wall_time: Wall timestamp for the tensor event.
op_type: Type of the op that generated the tensor (e.g., "Conv2D").
output_slot: Output slot index of the tensor for the op.
execution_index: Top-level execution index.
graph_execution_trace_index: Intra-graph execution index.
"""
# FULL_TENSOR mode is handled by a separate code path.
assert tensor_debug_mode != debug_event_pb2.TensorDebugMode.FULL_TENSOR
if not debug_tensor_value:
return
if tensor_debug_mode == debug_event_pb2.TensorDebugMode.CURT_HEALTH:
_, any_nan_inf = debug_tensor_value
if any_nan_inf:
self._alerts.append(InfNanAlert(
wall_time,
op_type,
output_slot,
execution_index=execution_index,
graph_execution_trace_index=graph_execution_trace_index))
elif tensor_debug_mode == debug_event_pb2.TensorDebugMode.CONCISE_HEALTH:
_, size, num_neg_inf, num_pos_inf, num_nan = debug_tensor_value
if num_neg_inf or num_pos_inf or num_nan:
self._alerts.append(InfNanAlert(
wall_time,
op_type,
output_slot,
size=size,
num_neg_inf=num_neg_inf,
num_pos_inf=num_pos_inf,
num_nan=num_nan,
execution_index=execution_index,
graph_execution_trace_index=graph_execution_trace_index))
elif tensor_debug_mode == debug_event_pb2.TensorDebugMode.FULL_HEALTH:
(_, _, _, _, size, num_neg_inf, num_pos_inf, num_nan,
_, _, _) = debug_tensor_value
if num_neg_inf or num_pos_inf or num_nan:
self._alerts.append(InfNanAlert(
wall_time,
op_type,
output_slot,
size=size,
num_neg_inf=num_neg_inf,
num_pos_inf=num_pos_inf,
num_nan=num_nan,
execution_index=execution_index,
graph_execution_trace_index=graph_execution_trace_index))
def on_execution(self,
execution_index,
execution):
if self._limit > 0 and len(self._alerts) >= self._limit:
return
if (execution.tensor_debug_mode ==
debug_event_pb2.TensorDebugMode.FULL_TENSOR):
tensor_values = self._debug_data_reader.execution_to_tensor_values(
execution)
for output_slot, tensor_value in enumerate(tensor_values):
self._check_full_tensor_value(
tensor_value, execution.wall_time, execution.op_type, output_slot,
execution_index=execution_index)
elif execution.debug_tensor_values:
for output_slot, debug_tensor_value in enumerate(
execution.debug_tensor_values):
self._check_debug_tensor_value(
execution.tensor_debug_mode,
debug_tensor_value,
execution.wall_time,
execution.op_type,
output_slot,
execution_index=execution_index)
def on_graph_execution_trace(self,
graph_execution_trace_index,
graph_execution_trace):
"""Monitor method for GraphExecutionTrace data object."""
if self._limit > 0 and len(self._alerts) >= self._limit:
return
if (graph_execution_trace.tensor_debug_mode ==
debug_event_pb2.TensorDebugMode.FULL_TENSOR):
tensor_value = (
self._debug_data_reader.graph_execution_trace_to_tensor_value(
graph_execution_trace))
self._check_full_tensor_value(
tensor_value, graph_execution_trace.wall_time,
graph_execution_trace.op_type, graph_execution_trace.output_slot,
graph_execution_trace_index=graph_execution_trace_index)
elif graph_execution_trace.debug_tensor_value:
self._check_debug_tensor_value(
graph_execution_trace.tensor_debug_mode,
graph_execution_trace.debug_tensor_value,
graph_execution_trace.wall_time,
graph_execution_trace.op_type,
graph_execution_trace.output_slot,
graph_execution_trace_index=graph_execution_trace_index)
def alerts(self):
return self._alerts
tensorflow/python/debug/lib/debug_events_monitors_test.py
+487
View File
@@ -0,0 +1,487 @@
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for the debug events writer Python class."""
from absl.testing import parameterized
import numpy as np
from tensorflow.core.protobuf import debug_event_pb2
from tensorflow.python.debug.lib import debug_events_monitors
from tensorflow.python.debug.lib import debug_events_reader
from tensorflow.python.debug.lib import dumping_callback
from tensorflow.python.debug.lib import dumping_callback_test_lib
from tensorflow.python.eager import def_function
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.platform import googletest
from tensorflow.python.platform import test
class TestMonitor(debug_events_monitors.BaseMonitor):
def __init__(self, debug_data_reader):
super(TestMonitor, self).__init__(debug_data_reader)
# Mapping execution index to Execution data objects.
self.executions = dict()
# Mapping graph execution trace index to GraphExecutionTrace data objects.
self.graph_execution_traces = dict()
def on_execution(self, execution_index, execution):
if execution_index in self.executions:
raise ValueError("Duplicate execution index: %d" % execution_index)
self.executions[execution_index] = execution
def on_graph_execution_trace(self, graph_execution_trace_index,
graph_execution_trace):
if graph_execution_trace_index in self.graph_execution_traces:
raise ValueError("Duplicate graph-execution-trace index: %d" %
graph_execution_trace_index)
self.graph_execution_traces[
graph_execution_trace_index] = graph_execution_trace
class DebugEventsMonitorTest(dumping_callback_test_lib.DumpingCallbackTestBase,
parameterized.TestCase):
@parameterized.named_parameters(
("NoTensor", "NO_TENSOR"),
("ConciseHealth", "CONCISE_HEALTH"),
("FullHealth", "FULL_HEALTH"),
("FullTensor", "FULL_TENSOR"),
)
def testOnExecutionIsCalled(self, tensor_debug_mode):
x = constant_op.constant([[1, 2], [3, 4]], dtype=dtypes.float32)
y = constant_op.constant([[-1], [1]], dtype=dtypes.float32)
writer = dumping_callback.enable_dump_debug_info(
self.dump_root, tensor_debug_mode=tensor_debug_mode)
math_ops.matmul(x, y)
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
test_monitor = TestMonitor(reader)
reader.update()
self.assertLen(test_monitor.executions, 1)
self.assertEmpty(test_monitor.graph_execution_traces)
execution = test_monitor.executions[0]
self.assertTrue(execution.wall_time)
self.assertEqual(execution.op_type, "MatMul")
self.assertLen(execution.output_tensor_device_ids, 1)
self.assertLen(execution.input_tensor_ids, 2)
self.assertLen(execution.output_tensor_ids, 1)
self.assertEqual(execution.num_outputs, 1)
self.assertEqual(execution.graph_id, "")
if tensor_debug_mode == "NO_TENSOR":
self.assertIsNone(execution.debug_tensor_values)
elif tensor_debug_mode == "CONCISE_HEALTH":
self.assertLen(execution.debug_tensor_values, 1)
# [tensor_id, element_count, neg_inf_count, pos_inf_count, nan_count].
self.assertLen(execution.debug_tensor_values[0], 5)
elif tensor_debug_mode == "FULL_HEALTH":
self.assertLen(execution.debug_tensor_values, 1)
# [tensor_id, device_id, dtype, rank, element_count,
# neg_inf_count, pos_inf_count, nan_count,
# neg_finite_count, zero_count, pos_finite_count].
self.assertLen(execution.debug_tensor_values[0], 11)
elif tensor_debug_mode == "FULL_TENSOR":
# Full tensor values are not stored in the debug_tensor_values field.
self.assertIsNone(execution.debug_tensor_values)
self.assertAllClose(
reader.execution_to_tensor_values(execution), [[[1.], [1.]]])
@parameterized.named_parameters(
("ConciseHealth", "CONCISE_HEALTH"),
("FullHealth", "FULL_HEALTH"),
("FullTensor", "FULL_TENSOR"),
)
def testOnGraphExecutionTraceIsCalled(self, tensor_debug_mode):
xs = constant_op.constant([2., 6., 8., 1., 2.], dtype=dtypes.float32)
writer = dumping_callback.enable_dump_debug_info(
self.dump_root, tensor_debug_mode=tensor_debug_mode)
@def_function.function
def unique_sum(xs):
"""Sum over the unique values, for testing."""
unique_xs, indices = array_ops.unique(xs)
return math_ops.reduce_sum(unique_xs), indices
unique_sum(xs)
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
test_monitor = TestMonitor(reader)
reader.update()
self.assertLen(test_monitor.executions, 1)
execution = test_monitor.executions[0]
self.assertTrue(execution.wall_time)
self.assertStartsWith(execution.op_type, "__inference_unique_sum")
self.assertLen(execution.output_tensor_device_ids, 2)
self.assertLen(execution.input_tensor_ids, 1)
self.assertLen(execution.output_tensor_ids, 2)
self.assertEqual(execution.num_outputs, 2)
self.assertTrue(execution.graph_id)
traces = test_monitor.graph_execution_traces
if tensor_debug_mode == "CONCISE_HEALTH":
self.assertLen(traces, 2) # [Unique:0 , Sum:0].
self.assertEqual(traces[0].op_type, "Unique")
self.assertEqual(traces[0].output_slot, 0)
# Unique:1 is not traced under CONCISE_HEALTH mode, as it's int-dtype.
self.assertEqual(traces[1].op_type, "Sum")
self.assertEqual(traces[1].output_slot, 0)
# [tensor_id, element_count, neg_inf_count, pos_inf_count, nan_count].
self.assertLen(traces[0].debug_tensor_value, 5)
self.assertLen(traces[1].debug_tensor_value, 5)
elif tensor_debug_mode == "FULL_HEALTH":
self.assertLen(traces, 2) # [Unique:0 , Sum:0].
self.assertEqual(traces[0].op_type, "Unique")
self.assertEqual(traces[0].output_slot, 0)
# Unique:1 is not traced under FULL_HEALTH mode, as it's int-dtype.
self.assertEqual(traces[1].op_type, "Sum")
self.assertEqual(traces[1].output_slot, 0)
# [tensor_id, device_id, dtype, rank, element_count,
# neg_inf_count, pos_inf_count, nan_count,
# neg_finite_count, zero_count, pos_finite_count].
self.assertLen(traces[0].debug_tensor_value, 11)
self.assertLen(traces[1].debug_tensor_value, 11)
elif tensor_debug_mode == "FULL_TENSOR":
# [Unique:0, Unique:1, Const:0, Sum:0].
self.assertEqual(traces[0].op_type, "Unique")
self.assertEqual(traces[0].output_slot, 0)
self.assertIsNone(traces[0].debug_tensor_value)
self.assertAllEqual(
reader.graph_execution_trace_to_tensor_value(traces[0]),
[2., 6., 8., 1.])
self.assertEqual(traces[1].op_type, "Unique")
self.assertEqual(traces[1].output_slot, 1)
self.assertIsNone(traces[1].debug_tensor_value)
self.assertAllEqual(
reader.graph_execution_trace_to_tensor_value(traces[1]),
[0, 1, 2, 3, 0])
self.assertEqual(traces[2].op_type, "Const")
self.assertEqual(traces[2].output_slot, 0)
self.assertIsNone(traces[2].debug_tensor_value)
self.assertAllClose(
reader.graph_execution_trace_to_tensor_value(traces[2]), [0])
self.assertEqual(traces[3].op_type, "Sum")
self.assertEqual(traces[3].output_slot, 0)
self.assertIsNone(traces[3].debug_tensor_value)
self.assertAllClose(
reader.graph_execution_trace_to_tensor_value(traces[3]), 17.)
class AlertDataObjectsTest(test_util.TensorFlowTestCase):
"""Unit tests for alert-class objects."""
def testInfNanMonitor(self):
alert = debug_events_monitors.InfNanAlert(
1234,
"FooOp",
1,
size=1000,
num_neg_inf=5,
num_pos_inf=10,
num_nan=20,
execution_index=777,
graph_execution_trace_index=888)
self.assertEqual(alert.wall_time, 1234)
self.assertEqual(alert.op_type, "FooOp")
self.assertEqual(alert.output_slot, 1)
self.assertEqual(alert.size, 1000)
self.assertEqual(alert.num_neg_inf, 5)
self.assertEqual(alert.num_pos_inf, 10)
self.assertEqual(alert.num_nan, 20)
self.assertEqual(alert.execution_index, 777)
self.assertEqual(alert.graph_execution_trace_index, 888)
class InfNanMonitorTest(test_util.TensorFlowTestCase, parameterized.TestCase):
def testInfNanMonitorStartsWithEmptyAlerts(self):
mock_reader = test.mock.MagicMock()
monitor = debug_events_monitors.InfNanMonitor(mock_reader)
self.assertEmpty(monitor.alerts())
def testInfNanMonitorOnExecutionUnderCurtHealthMode(self):
mock_reader = test.mock.MagicMock()
monitor = debug_events_monitors.InfNanMonitor(mock_reader)
execution_digest = debug_events_reader.ExecutionDigest(
1234, 1, "FooOp", output_tensor_device_ids=[0, 1])
execution = debug_events_reader.Execution(
execution_digest,
"worker01", ["a1", "b2", "e3"],
debug_event_pb2.TensorDebugMode.CURT_HEALTH,
graph_id=None,
input_tensor_ids=[12, 34],
output_tensor_ids=[56, 78],
debug_tensor_values=[[-1, 0], [-1, 1]]) # [tensor_id, any_inf_nan].
monitor.on_execution(50, execution)
self.assertLen(monitor.alerts(), 1)
alert = monitor.alerts()[0]
self.assertEqual(alert.wall_time, 1234)
self.assertEqual(alert.op_type, "FooOp")
self.assertEqual(alert.output_slot, 1)
# The four fields below are unavailable under CURT_HEALTH mode by design.
self.assertIsNone(alert.size)
self.assertIsNone(alert.num_neg_inf)
self.assertIsNone(alert.num_pos_inf)
self.assertIsNone(alert.num_nan)
self.assertEqual(alert.execution_index, 50)
self.assertIsNone(alert.graph_execution_trace_index)
@parameterized.named_parameters(
("ConciseHealth",
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
# [tensor_id, size, num_neg_inf, num_pos_inf, num_nan].
[[-1, 10, 1, 2, 3],
[-1, 100, 0, 0, 0]]),
("FullHealth",
debug_event_pb2.TensorDebugMode.FULL_HEALTH,
# [tensor_id, device_id, dtype, rank, element_count,
# neg_inf_count, pos_inf_count, nan_count,
# neg_finite_count, zero_count, pos_finite_count].
[[-1, -1, 1, 1, 10, 1, 2, 3, 0, 0, 0],
[-1, -1, 1, 1, 100, 0, 0, 0, 10, 30, 60]]),
)
def testInfNanMonitorOnExecutionUnderHealthMode(self,
tensor_debug_mode,
debug_tensor_values):
mock_reader = test.mock.MagicMock()
monitor = debug_events_monitors.InfNanMonitor(mock_reader)
execution_digest = debug_events_reader.ExecutionDigest(
1234, 1, "BarOp", output_tensor_device_ids=[0, 1])
execution = debug_events_reader.Execution(
execution_digest,
"worker01",
["a1", "b2", "e3"],
tensor_debug_mode,
graph_id=None,
input_tensor_ids=[12, 34],
output_tensor_ids=[56, 78],
debug_tensor_values=debug_tensor_values)
monitor.on_execution(60, execution)
self.assertLen(monitor.alerts(), 1)
alert = monitor.alerts()[0]
self.assertEqual(alert.wall_time, 1234)
self.assertEqual(alert.op_type, "BarOp")
self.assertEqual(alert.output_slot, 0)
self.assertEqual(alert.size, 10)
self.assertEqual(alert.num_neg_inf, 1)
self.assertEqual(alert.num_pos_inf, 2)
self.assertEqual(alert.num_nan, 3)
self.assertEqual(alert.execution_index, 60)
self.assertIsNone(alert.graph_execution_trace_index)
@parameterized.named_parameters(
("Shape",
debug_event_pb2.TensorDebugMode.SHAPE,
# [tensor_id, dtype, rank, element_cont, ...shape_truncate_6]
[[-1, 1, 2, 6, 3, 2, 0, 0, 0, 0],
[-1, 10, 1, 7, 7, 0, 0, 0, 0, 0]]),
)
def testInfNanMonitorOnExecutionUnderModeWithNoInfNanInfo(
self,
tensor_debug_mode,
debug_tensor_values):
mock_reader = test.mock.MagicMock()
monitor = debug_events_monitors.InfNanMonitor(mock_reader)
execution_digest = debug_events_reader.ExecutionDigest(
1234, 1, "BarOp", output_tensor_device_ids=[0, 1])
execution = debug_events_reader.Execution(
execution_digest,
"worker01",
["a1", "b2", "e3"],
tensor_debug_mode,
graph_id=None,
input_tensor_ids=[12, 34],
output_tensor_ids=[56, 78],
debug_tensor_values=debug_tensor_values)
monitor.on_execution(60, execution)
self.assertEmpty(monitor.alerts())
@parameterized.named_parameters(
("FloatsScalarWithInfAndNan", np.inf, np.float32, 1, 0, 1, 0),
("Floats2DWithInfAndNan", [[0, np.nan, np.nan, -np.inf]
], np.float32, 4, 1, 0, 2),
("Floats1DWithoutInfOrNan", [0, -1e6, 1e6, 9e5], np.float32, 4, 0, 0, 0),
("Integers", [[0, 1000, -200, -300]], np.int32, 4, 0, 0, 0),
("Booleans", [False, True, False, False], np.int32, 4, 0, 0, 0),
)
def testInfNanMonitorOnExecutionUnderFullTensorModeWorks(
self, tensor_value, dtype, expected_size, expected_num_neg_inf,
expected_num_pos_inf, expected_num_nan):
mock_reader = test.mock.MagicMock()
mock_reader.execution_to_tensor_values.return_value = [
np.array([[0.0, -1.0, 1.0]]),
np.array(tensor_value, dtype=dtype)
]
monitor = debug_events_monitors.InfNanMonitor(mock_reader)
execution_digest = debug_events_reader.ExecutionDigest(
1234,
1,
"__inference_bar_function_1234",
output_tensor_device_ids=[0, 1])
execution = debug_events_reader.Execution(
execution_digest,
"worker01", ["a1", "b2", "e3"],
debug_event_pb2.TensorDebugMode.FULL_TENSOR,
graph_id=None,
input_tensor_ids=[12, 34],
output_tensor_ids=[56, 78])
monitor.on_execution(70, execution)
if expected_num_neg_inf or expected_num_pos_inf or expected_num_nan:
self.assertLen(monitor.alerts(), 1)
alert = monitor.alerts()[0]
self.assertEqual(alert.wall_time, 1234)
self.assertEqual(alert.op_type, "__inference_bar_function_1234")
self.assertEqual(alert.output_slot, 1)
self.assertEqual(alert.size, expected_size)
self.assertEqual(alert.num_neg_inf, expected_num_neg_inf)
self.assertEqual(alert.num_pos_inf, expected_num_pos_inf)
self.assertEqual(alert.num_nan, expected_num_nan)
self.assertEqual(alert.execution_index, 70)
self.assertIsNone(alert.graph_execution_trace_index, 70)
else:
self.assertEmpty(monitor.alerts())
def testInfNaNMonitorOnGraphExecutionTraceCurtHealthMode(self):
mock_reader = test.mock.MagicMock()
monitor = debug_events_monitors.InfNanMonitor(mock_reader)
trace_digest = debug_events_reader.GraphExecutionTraceDigest(
1234, 1, "FooOp", "FooOp_1", 2, "g1")
trace = debug_events_reader.GraphExecutionTrace(
trace_digest, ["g0", "g1"],
debug_event_pb2.TensorDebugMode.CURT_HEALTH,
debug_tensor_value=[9, 1]) # [tensor_id, any_inf_nan].
monitor.on_graph_execution_trace(55, trace)
self.assertLen(monitor.alerts(), 1)
alert = monitor.alerts()[0]
self.assertEqual(alert.wall_time, 1234)
self.assertEqual(alert.op_type, "FooOp")
self.assertEqual(alert.output_slot, 2)
# The four fields below are unavailable under CURT_HEALTH mode by design.
self.assertIsNone(alert.size)
self.assertIsNone(alert.num_neg_inf)
self.assertIsNone(alert.num_pos_inf)
self.assertIsNone(alert.num_nan)
self.assertIsNone(alert.execution_index)
self.assertEqual(alert.graph_execution_trace_index, 55)
def testInfNaNMonitorOnGraphExecutionTraceConciseHealthMode(self):
mock_reader = test.mock.MagicMock()
monitor = debug_events_monitors.InfNanMonitor(mock_reader)
trace_digest = debug_events_reader.GraphExecutionTraceDigest(
1234, 1, "FooOp", "FooOp_1", 2, "g1")
trace = debug_events_reader.GraphExecutionTrace(
trace_digest,
["g0", "g1"],
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
# [tensor_id, size, num_neg_inf, num_pos_inf, num_nan].
debug_tensor_value=[9, 100, 3, 2, 1])
monitor.on_graph_execution_trace(55, trace)
self.assertLen(monitor.alerts(), 1)
alert = monitor.alerts()[0]
self.assertEqual(alert.wall_time, 1234)
self.assertEqual(alert.op_type, "FooOp")
self.assertEqual(alert.output_slot, 2)
self.assertEqual(alert.size, 100)
self.assertEqual(alert.num_neg_inf, 3)
self.assertEqual(alert.num_pos_inf, 2)
self.assertEqual(alert.num_nan, 1)
self.assertEqual(alert.graph_execution_trace_index, 55)
@parameterized.named_parameters(
("FloatsScalarWithInfAndNan", np.inf, np.float32, 1, 0, 1, 0),
("Floats2DWithInfAndNan", [[0, np.nan, np.nan, -np.inf]
], np.float32, 4, 1, 0, 2),
("Floats1DWithoutInfOrNan", [0, -1e6, 1e6, 9e5], np.float32, 4, 0, 0, 0),
("Integers", [[0, 1000, -200, -300]], np.int32, 4, 0, 0, 0),
("Booleans", [False, True, False, False], np.int32, 4, 0, 0, 0),
)
def testInfNanMonitorOnGraphExecutionTraceUnderFullTensorModeWorks(
self, tensor_value, dtype, expected_size, expected_num_neg_inf,
expected_num_pos_inf, expected_num_nan):
mock_reader = test.mock.MagicMock()
mock_reader.graph_execution_trace_to_tensor_value.return_value = np.array(
tensor_value, dtype=dtype)
monitor = debug_events_monitors.InfNanMonitor(mock_reader)
trace_digest = debug_events_reader.GraphExecutionTraceDigest(
1234, 1, "BazOp", "name_scope_3/BazOp_1", 2, "g1")
trace = debug_events_reader.GraphExecutionTrace(
trace_digest, ["g0", "g1"], debug_event_pb2.TensorDebugMode.FULL_TENSOR)
monitor.on_graph_execution_trace(80, trace)
if expected_num_neg_inf or expected_num_pos_inf or expected_num_nan:
self.assertLen(monitor.alerts(), 1)
alert = monitor.alerts()[0]
self.assertEqual(alert.wall_time, 1234)
self.assertEqual(alert.op_type, "BazOp")
self.assertEqual(alert.output_slot, 2)
self.assertEqual(alert.size, expected_size)
self.assertEqual(alert.num_neg_inf, expected_num_neg_inf)
self.assertEqual(alert.num_pos_inf, expected_num_pos_inf)
self.assertEqual(alert.num_nan, expected_num_nan)
self.assertIsNone(alert.execution_index)
self.assertEqual(alert.graph_execution_trace_index, 80)
else:
self.assertEmpty(monitor.alerts())
def testLimitingInfNanMonitorAlertCountWorks(self):
mock_reader = test.mock.MagicMock()
monitor = debug_events_monitors.InfNanMonitor(mock_reader, limit=3)
for i in range(10):
execution_digest = debug_events_reader.ExecutionDigest(
i * 1000, 1, "FooOp", output_tensor_device_ids=[0, 1])
execution = debug_events_reader.Execution(
execution_digest,
"worker01", ["a1", "b2", "e3"],
debug_event_pb2.TensorDebugMode.CURT_HEALTH,
graph_id=None,
input_tensor_ids=[12, 34],
output_tensor_ids=[56, 78],
debug_tensor_values=[[-1, 0], [-1, 1]]) # [tensor_id, any_inf_nan].
monitor.on_execution(i, execution)
alerts = monitor.alerts()
self.assertLen(alerts, 3)
for i, alert in enumerate(alerts):
self.assertEqual(alert.wall_time, i * 1000)
self.assertEqual(alert.op_type, "FooOp")
self.assertEqual(alert.output_slot, 1)
# The four fields below are unavailable under CURT_HEALTH mode by design.
self.assertIsNone(alert.size)
self.assertIsNone(alert.num_neg_inf)
self.assertIsNone(alert.num_pos_inf)
self.assertIsNone(alert.num_nan)
self.assertEqual(alert.execution_index, i)
self.assertIsNone(alert.graph_execution_trace_index)
if __name__ == "__main__":
ops.enable_eager_execution()
googletest.main()
tensorflow/python/debug/lib/debug_events_reader.py
+1362
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/debug/lib/debug_events_writer.py
+158
View File
@@ -0,0 +1,158 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Writer class for `DebugEvent` protos in tfdbg v2."""
import time
from tensorflow.core.protobuf import debug_event_pb2
from tensorflow.python.client import _pywrap_debug_events_writer
# Default size of each circular buffer (unit: number of DebugEvent protos).
DEFAULT_CIRCULAR_BUFFER_SIZE = 1000
class DebugEventsWriter(object):
"""A writer for TF debugging events. Used by tfdbg v2."""
def __init__(self,
dump_root,
tfdbg_run_id,
circular_buffer_size=DEFAULT_CIRCULAR_BUFFER_SIZE):
"""Construct a DebugEventsWriter object.
NOTE: Given the same `dump_root`, all objects from this constructor
will point to the same underlying set of writers. In other words, they
will write to the same set of debug events files in the `dump_root`
folder.
Args:
dump_root: The root directory for dumping debug data. If `dump_root` does
not exist as a directory, it will be created.
tfdbg_run_id: Debugger Run ID.
circular_buffer_size: Size of the circular buffer for each of the two
execution-related debug events files: with the following suffixes: -
.execution - .graph_execution_traces If <= 0, the circular-buffer
behavior will be abolished in the constructed object.
"""
if not dump_root:
raise ValueError("Empty or None dump root")
self._dump_root = dump_root
self._tfdbg_run_id = tfdbg_run_id
_pywrap_debug_events_writer.Init(self._dump_root, self._tfdbg_run_id,
circular_buffer_size)
def WriteSourceFile(self, source_file):
"""Write a SourceFile proto with the writer.
Args:
source_file: A SourceFile proto, describing the content of a source file
involved in the execution of the debugged TensorFlow program.
"""
# TODO(cais): Explore performance optimization that avoids memcpy.
debug_event = debug_event_pb2.DebugEvent(source_file=source_file)
self._EnsureTimestampAdded(debug_event)
_pywrap_debug_events_writer.WriteSourceFile(self._dump_root, debug_event)
def WriteStackFrameWithId(self, stack_frame_with_id):
"""Write a StackFrameWithId proto with the writer.
Args:
stack_frame_with_id: A StackFrameWithId proto, describing the content a
stack frame involved in the execution of the debugged TensorFlow
program.
"""
debug_event = debug_event_pb2.DebugEvent(
stack_frame_with_id=stack_frame_with_id)
self._EnsureTimestampAdded(debug_event)
_pywrap_debug_events_writer.WriteStackFrameWithId(self._dump_root,
debug_event)
def WriteGraphOpCreation(self, graph_op_creation):
"""Write a GraphOpCreation proto with the writer.
Args:
graph_op_creation: A GraphOpCreation proto, describing the details of the
creation of an op inside a TensorFlow Graph.
"""
debug_event = debug_event_pb2.DebugEvent(
graph_op_creation=graph_op_creation)
self._EnsureTimestampAdded(debug_event)
_pywrap_debug_events_writer.WriteGraphOpCreation(self._dump_root,
debug_event)
def WriteDebuggedGraph(self, debugged_graph):
"""Write a DebuggedGraph proto with the writer.
Args:
debugged_graph: A DebuggedGraph proto, describing the details of a
TensorFlow Graph that has completed its construction.
"""
debug_event = debug_event_pb2.DebugEvent(debugged_graph=debugged_graph)
self._EnsureTimestampAdded(debug_event)
_pywrap_debug_events_writer.WriteDebuggedGraph(self._dump_root, debug_event)
def WriteExecution(self, execution):
"""Write a Execution proto with the writer.
Args:
execution: An Execution proto, describing a TensorFlow op or graph
execution event.
"""
debug_event = debug_event_pb2.DebugEvent(execution=execution)
self._EnsureTimestampAdded(debug_event)
_pywrap_debug_events_writer.WriteExecution(self._dump_root, debug_event)
def WriteGraphExecutionTrace(self, graph_execution_trace):
"""Write a GraphExecutionTrace proto with the writer.
Args:
graph_execution_trace: A GraphExecutionTrace proto, concerning the value
of an intermediate tensor or a list of intermediate tensors that are
computed during the graph's execution.
"""
debug_event = debug_event_pb2.DebugEvent(
graph_execution_trace=graph_execution_trace)
self._EnsureTimestampAdded(debug_event)
_pywrap_debug_events_writer.WriteGraphExecutionTrace(
self._dump_root, debug_event)
def RegisterDeviceAndGetId(self, device_name):
return _pywrap_debug_events_writer.RegisterDeviceAndGetId(
self._dump_root, device_name)
def FlushNonExecutionFiles(self):
"""Flush the non-execution debug event files."""
_pywrap_debug_events_writer.FlushNonExecutionFiles(self._dump_root)
def FlushExecutionFiles(self):
"""Flush the execution debug event files.
Causes the current content of the cyclic buffers to be written to
the .execution and .graph_execution_traces debug events files.
Also clears those cyclic buffers.
"""
_pywrap_debug_events_writer.FlushExecutionFiles(self._dump_root)
def Close(self):
"""Close the writer."""
_pywrap_debug_events_writer.Close(self._dump_root)
@property
def dump_root(self):
return self._dump_root
def _EnsureTimestampAdded(self, debug_event):
if debug_event.wall_time == 0:
debug_event.wall_time = time.time()
tensorflow/python/debug/lib/debug_events_writer_test.py
+921
View File
@@ -0,0 +1,921 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for the debug events writer Python class."""
import glob
import json as json_lib
import os
import re
import threading
import time
from absl.testing import parameterized
from tensorflow.core.protobuf import debug_event_pb2
from tensorflow.python.debug.lib import debug_events_reader
from tensorflow.python.debug.lib import debug_events_writer
from tensorflow.python.debug.lib import dumping_callback_test_lib
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.framework import versions
from tensorflow.python.platform import googletest
class DebugEventsWriterTest(dumping_callback_test_lib.DumpingCallbackTestBase,
parameterized.TestCase):
def testMultiThreadedConstructorCallWorks(self):
def init_writer():
debug_events_writer.DebugEventsWriter(self.dump_root, self.tfdbg_run_id)
num_threads = 4
threads = []
for _ in range(num_threads):
thread = threading.Thread(target=init_writer)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
# Verify that there is only one debug event file of each type.
metadata_paths = glob.glob(os.path.join(self.dump_root, "*.metadata"))
self.assertLen(metadata_paths, 1)
source_files_paths = glob.glob(
os.path.join(self.dump_root, "*.source_files"))
self.assertLen(source_files_paths, 1)
stack_frames_paths = glob.glob(
os.path.join(self.dump_root, "*.stack_frames"))
self.assertLen(stack_frames_paths, 1)
graphs_paths = glob.glob(os.path.join(self.dump_root, "*.graphs"))
self.assertLen(graphs_paths, 1)
self._readAndCheckMetadataFile()
def testWriteSourceFilesAndStackFrames(self):
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id)
num_protos = 10
for i in range(num_protos):
source_file = debug_event_pb2.SourceFile()
source_file.file_path = "/home/tf2user/main.py"
source_file.host_name = "machine.cluster"
source_file.lines.append("print(%d)" % i)
writer.WriteSourceFile(source_file)
stack_frame = debug_event_pb2.StackFrameWithId()
stack_frame.id = "stack_%d" % i
stack_frame.file_line_col.file_index = i * 10
writer.WriteStackFrameWithId(stack_frame)
writer.FlushNonExecutionFiles()
with debug_events_reader.DebugEventsReader(self.dump_root) as reader:
actuals = list(item.debug_event.source_file
for item in reader.source_files_iterator())
self.assertLen(actuals, num_protos)
for i in range(num_protos):
self.assertEqual(actuals[i].file_path, "/home/tf2user/main.py")
self.assertEqual(actuals[i].host_name, "machine.cluster")
self.assertEqual(actuals[i].lines, ["print(%d)" % i])
actuals = list(item.debug_event.stack_frame_with_id
for item in reader.stack_frames_iterator())
self.assertLen(actuals, num_protos)
for i in range(num_protos):
self.assertEqual(actuals[i].id, "stack_%d" % i)
self.assertEqual(actuals[i].file_line_col.file_index, i * 10)
def testWriteGraphOpCreationAndDebuggedGraphs(self):
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id)
num_op_creations = 10
for i in range(num_op_creations):
graph_op_creation = debug_event_pb2.GraphOpCreation()
graph_op_creation.op_type = "Conv2D"
graph_op_creation.op_name = "Conv2D_%d" % i
writer.WriteGraphOpCreation(graph_op_creation)
debugged_graph = debug_event_pb2.DebuggedGraph()
debugged_graph.graph_id = "deadbeaf"
debugged_graph.graph_name = "MyGraph1"
writer.WriteDebuggedGraph(debugged_graph)
writer.FlushNonExecutionFiles()
reader = debug_events_reader.DebugEventsReader(self.dump_root)
actuals = list(item.debug_event for item in reader.graphs_iterator())
self.assertLen(actuals, num_op_creations + 1)
for i in range(num_op_creations):
self.assertEqual(actuals[i].graph_op_creation.op_type, "Conv2D")
self.assertEqual(actuals[i].graph_op_creation.op_name, "Conv2D_%d" % i)
self.assertEqual(actuals[num_op_creations].debugged_graph.graph_id,
"deadbeaf")
def testConcurrentWritesToNonExecutionFilesWorks(self):
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id)
source_file_state = {"counter": 0, "lock": threading.Lock()}
def writer_source_file():
source_file = debug_event_pb2.SourceFile()
with source_file_state["lock"]:
source_file.file_path = "/home/tf2user/file_%d.py" % source_file_state[
"counter"]
source_file_state["counter"] += 1
writer.WriteSourceFile(source_file)
# More-frequent-than-necessary concurrent flushing is not recommended,
# but tolerated.
writer.FlushNonExecutionFiles()
stack_frame_state = {"counter": 0, "lock": threading.Lock()}
def write_stack_frame():
stack_frame = debug_event_pb2.StackFrameWithId()
with stack_frame_state["lock"]:
stack_frame.id = "stack_frame_%d" % stack_frame_state["counter"]
stack_frame_state["counter"] += 1
writer.WriteStackFrameWithId(stack_frame)
# More-frequent-than-necessary concurrent flushing is not recommended,
# but tolerated.
writer.FlushNonExecutionFiles()
graph_op_state = {"counter": 0, "lock": threading.Lock()}
def write_graph_op_creation():
graph_op_creation = debug_event_pb2.GraphOpCreation()
with graph_op_state["lock"]:
graph_op_creation.op_name = "Op%d" % graph_op_state["counter"]
graph_op_state["counter"] += 1
writer.WriteGraphOpCreation(graph_op_creation)
# More-frequent-than-necessary concurrent flushing is not recommended,
# but tolerated.
writer.FlushNonExecutionFiles()
num_threads = 9
threads = []
for i in range(num_threads):
if i % 3 == 0:
target = writer_source_file
elif i % 3 == 1:
target = write_stack_frame
else:
target = write_graph_op_creation
thread = threading.Thread(target=target)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
# Verify the content of the .source_files file.
with debug_events_reader.DebugEventsReader(self.dump_root) as reader:
source_files_iter = reader.source_files_iterator()
actuals = list(item.debug_event.source_file for item in source_files_iter)
file_paths = sorted([actual.file_path for actual in actuals])
self.assertEqual(file_paths, [
"/home/tf2user/file_0.py", "/home/tf2user/file_1.py",
"/home/tf2user/file_2.py"
])
# Verify the content of the .stack_frames file.
actuals = list(item.debug_event.stack_frame_with_id
for item in reader.stack_frames_iterator())
stack_frame_ids = sorted([actual.id for actual in actuals])
self.assertEqual(stack_frame_ids,
["stack_frame_0", "stack_frame_1", "stack_frame_2"])
# Verify the content of the .graphs file.
actuals = list(item.debug_event.graph_op_creation
for item in reader.graphs_iterator())
graph_op_names = sorted([actual.op_name for actual in actuals])
self.assertEqual(graph_op_names, ["Op0", "Op1", "Op2"])
def testWriteAndReadMetadata(self):
t0 = time.time()
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id)
writer.Close()
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
self.assertIsInstance(reader.starting_wall_time(), float)
self.assertGreaterEqual(reader.starting_wall_time(), t0)
self.assertEqual(reader.tensorflow_version(), versions.__version__)
self.assertTrue(reader.tfdbg_run_id())
def testWriteExecutionEventsWithCircularBuffer(self):
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id)
num_execution_events = debug_events_writer.DEFAULT_CIRCULAR_BUFFER_SIZE * 2
for i in range(num_execution_events):
execution = debug_event_pb2.Execution()
execution.op_type = "OpType%d" % i
writer.WriteExecution(execution)
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
# Before FlushExecutionFiles() is called. No data should have been written
# to the file.
reader.update()
self.assertFalse(reader.executions())
writer.FlushExecutionFiles()
reader.update()
executions = reader.executions()
for i, execution in enumerate(executions):
self.assertEqual(
execution.op_type,
"OpType%d" % (i + debug_events_writer.DEFAULT_CIRCULAR_BUFFER_SIZE))
def testWriteExecutionEventsWithoutCircularBufferBehavior(self):
# A circular buffer size of 0 abolishes the circular buffer behavior.
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id, 0)
num_execution_events = debug_events_writer.DEFAULT_CIRCULAR_BUFFER_SIZE * 2
for i in range(num_execution_events):
execution = debug_event_pb2.Execution()
execution.op_type = "OpType%d" % i
writer.WriteExecution(execution)
writer.FlushExecutionFiles()
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
reader.update()
executions = reader.executions()
self.assertLen(executions, num_execution_events)
for i, execution in enumerate(executions):
self.assertEqual(execution.op_type, "OpType%d" % i)
def testWriteGraphExecutionTraceEventsWithCircularBuffer(self):
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id)
num_execution_events = debug_events_writer.DEFAULT_CIRCULAR_BUFFER_SIZE * 2
for i in range(num_execution_events):
trace = debug_event_pb2.GraphExecutionTrace()
trace.op_name = "Op%d" % i
writer.WriteGraphExecutionTrace(trace)
with debug_events_reader.DebugEventsReader(self.dump_root) as reader:
actuals = list(reader.graph_execution_traces_iterators()[0])
# Before FlushExecutionFiles() is called. No data should have been written
# to the file.
self.assertEmpty(actuals)
writer.FlushExecutionFiles()
actuals = list(item.debug_event.graph_execution_trace
for item in reader.graph_execution_traces_iterators()[0])
self.assertLen(actuals, debug_events_writer.DEFAULT_CIRCULAR_BUFFER_SIZE)
for i in range(debug_events_writer.DEFAULT_CIRCULAR_BUFFER_SIZE):
self.assertEqual(
actuals[i].op_name,
"Op%d" % (i + debug_events_writer.DEFAULT_CIRCULAR_BUFFER_SIZE))
def testWriteGraphExecutionTraceEventsWithoutCircularBufferBehavior(self):
# A circular buffer size of 0 abolishes the circular buffer behavior.
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id, 0)
num_execution_events = debug_events_writer.DEFAULT_CIRCULAR_BUFFER_SIZE * 2
for i in range(num_execution_events):
trace = debug_event_pb2.GraphExecutionTrace()
trace.op_name = "Op%d" % i
writer.WriteGraphExecutionTrace(trace)
writer.FlushExecutionFiles()
with debug_events_reader.DebugEventsReader(self.dump_root) as reader:
actuals = list(item.debug_event.graph_execution_trace
for item in reader.graph_execution_traces_iterators()[0])
self.assertLen(actuals, num_execution_events)
for i in range(num_execution_events):
self.assertEqual(actuals[i].op_name, "Op%d" % i)
def testConcurrentWritesToExecutionFiles(self):
circular_buffer_size = 5
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id,
circular_buffer_size)
debugged_graph = debug_event_pb2.DebuggedGraph(graph_id="graph1",
graph_name="graph1")
writer.WriteDebuggedGraph(debugged_graph)
execution_state = {"counter": 0, "lock": threading.Lock()}
def write_execution():
execution = debug_event_pb2.Execution()
with execution_state["lock"]:
execution.op_type = "OpType%d" % execution_state["counter"]
execution_state["counter"] += 1
writer.WriteExecution(execution)
graph_execution_trace_state = {"counter": 0, "lock": threading.Lock()}
def write_graph_execution_trace():
with graph_execution_trace_state["lock"]:
op_name = "Op%d" % graph_execution_trace_state["counter"]
graph_op_creation = debug_event_pb2.GraphOpCreation(
op_type="FooOp", op_name=op_name, graph_id="graph1")
trace = debug_event_pb2.GraphExecutionTrace(
op_name=op_name, tfdbg_context_id="graph1")
graph_execution_trace_state["counter"] += 1
writer.WriteGraphOpCreation(graph_op_creation)
writer.WriteGraphExecutionTrace(trace)
threads = []
for i in range(circular_buffer_size * 4):
if i % 2 == 0:
target = write_execution
else:
target = write_graph_execution_trace
thread = threading.Thread(target=target)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
reader.update()
# Verify the content of the .execution file.
executions = reader.executions()
executed_op_types = [execution.op_type for execution in executions]
self.assertLen(executed_op_types, circular_buffer_size)
self.assertLen(executed_op_types, len(set(executed_op_types)))
# Verify the content of the .graph_execution_traces file.
op_names = [trace.op_name for trace in reader.graph_execution_traces()]
self.assertLen(op_names, circular_buffer_size)
self.assertLen(op_names, len(set(op_names)))
def testConcurrentSourceFileRandomReads(self):
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id)
for i in range(100):
source_file = debug_event_pb2.SourceFile(
host_name="localhost", file_path="/tmp/file_%d.py" % i)
source_file.lines.append("# File %d" % i)
writer.WriteSourceFile(source_file)
writer.FlushNonExecutionFiles()
reader = debug_events_reader.DebugDataReader(self.dump_root)
reader.update()
lines = [None] * 100
def read_job_1():
# Read in the reverse order to enhance randomness of the read access.
for i in range(49, -1, -1):
lines[i] = reader.source_lines("localhost", "/tmp/file_%d.py" % i)
def read_job_2():
for i in range(99, 49, -1):
lines[i] = reader.source_lines("localhost", "/tmp/file_%d.py" % i)
thread_1 = threading.Thread(target=read_job_1)
thread_2 = threading.Thread(target=read_job_2)
thread_1.start()
thread_2.start()
thread_1.join()
thread_2.join()
for i in range(100):
self.assertEqual(lines[i], ["# File %d" % i])
def testConcurrentExecutionUpdateAndRandomRead(self):
circular_buffer_size = -1
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id,
circular_buffer_size)
writer_state = {"counter": 0, "done": False}
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
def write_and_update_job():
while True:
if writer_state["done"]:
break
execution = debug_event_pb2.Execution()
execution.op_type = "OpType%d" % writer_state["counter"]
writer_state["counter"] += 1
writer.WriteExecution(execution)
writer.FlushExecutionFiles()
reader.update()
# On the sub-thread, keep writing and reading new Execution protos.
write_and_update_thread = threading.Thread(target=write_and_update_job)
write_and_update_thread.start()
# On the main thread, do concurrent random read.
while True:
exec_digests = reader.executions(digest=True)
if exec_digests:
exec_0 = reader.read_execution(exec_digests[0])
self.assertEqual(exec_0.op_type, "OpType0")
writer_state["done"] = True
break
else:
time.sleep(0.1)
continue
write_and_update_thread.join()
def testConcurrentExecutionRandomReads(self):
circular_buffer_size = -1
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id,
circular_buffer_size)
for i in range(100):
execution = debug_event_pb2.Execution()
execution.op_type = "OpType%d" % i
writer.WriteExecution(execution)
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
reader = debug_events_reader.DebugDataReader(self.dump_root)
reader.update()
executions = [None] * 100
def read_job_1():
execution_digests = reader.executions(digest=True)
# Read in the reverse order to enhance randomness of the read access.
for i in range(49, -1, -1):
execution = reader.read_execution(execution_digests[i])
executions[i] = execution
def read_job_2():
execution_digests = reader.executions(digest=True)
for i in range(99, 49, -1):
execution = reader.read_execution(execution_digests[i])
executions[i] = execution
thread_1 = threading.Thread(target=read_job_1)
thread_2 = threading.Thread(target=read_job_2)
thread_1.start()
thread_2.start()
thread_1.join()
thread_2.join()
for i in range(100):
self.assertEqual(executions[i].op_type, "OpType%d" % i)
def testConcurrentGraphExecutionTraceUpdateAndRandomRead(self):
circular_buffer_size = -1
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id,
circular_buffer_size)
debugged_graph = debug_event_pb2.DebuggedGraph(graph_id="graph1",
graph_name="graph1")
writer.WriteDebuggedGraph(debugged_graph)
writer_state = {"counter": 0, "done": False}
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
def write_and_update_job():
while True:
if writer_state["done"]:
break
op_name = "Op%d" % writer_state["counter"]
graph_op_creation = debug_event_pb2.GraphOpCreation(
op_type="FooOp", op_name=op_name, graph_id="graph1")
writer.WriteGraphOpCreation(graph_op_creation)
trace = debug_event_pb2.GraphExecutionTrace(
op_name=op_name, tfdbg_context_id="graph1")
writer.WriteGraphExecutionTrace(trace)
writer_state["counter"] += 1
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
reader.update()
# On the sub-thread, keep writing and reading new GraphExecutionTraces.
write_and_update_thread = threading.Thread(target=write_and_update_job)
write_and_update_thread.start()
# On the main thread, do concurrent random read.
while True:
digests = reader.graph_execution_traces(digest=True)
if digests:
trace_0 = reader.read_graph_execution_trace(digests[0])
self.assertEqual(trace_0.op_name, "Op0")
writer_state["done"] = True
break
else:
time.sleep(0.1)
continue
write_and_update_thread.join()
def testConcurrentGraphExecutionTraceRandomReads(self):
circular_buffer_size = -1
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
self.tfdbg_run_id,
circular_buffer_size)
debugged_graph = debug_event_pb2.DebuggedGraph(graph_id="graph1",
graph_name="graph1")
writer.WriteDebuggedGraph(debugged_graph)
for i in range(100):
op_name = "Op%d" % i
graph_op_creation = debug_event_pb2.GraphOpCreation(
op_type="FooOp", op_name=op_name, graph_id="graph1")
writer.WriteGraphOpCreation(graph_op_creation)
trace = debug_event_pb2.GraphExecutionTrace(
op_name=op_name, tfdbg_context_id="graph1")
writer.WriteGraphExecutionTrace(trace)
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
reader = debug_events_reader.DebugDataReader(self.dump_root)
reader.update()
traces = [None] * 100
def read_job_1():
digests = reader.graph_execution_traces(digest=True)
for i in range(49, -1, -1):
traces[i] = reader.read_graph_execution_trace(digests[i])
def read_job_2():
digests = reader.graph_execution_traces(digest=True)
for i in range(99, 49, -1):
traces[i] = reader.read_graph_execution_trace(digests[i])
thread_1 = threading.Thread(target=read_job_1)
thread_2 = threading.Thread(target=read_job_2)
thread_1.start()
thread_2.start()
thread_1.join()
thread_2.join()
for i in range(100):
self.assertEqual(traces[i].op_name, "Op%d" % i)
@parameterized.named_parameters(
("Begin1End3", 1, 3, 1, 3),
("Begin0End3", 0, 3, 0, 3),
("Begin0EndNeg1", 0, -1, 0, 4),
("BeginNoneEnd3", None, 3, 0, 3),
("Begin2EndNone", 2, None, 2, 5),
("BeginNoneEndNone", None, None, 0, 5),
)
def testRangeReadingExecutions(self, begin, end, expected_begin,
expected_end):
writer = debug_events_writer.DebugEventsWriter(
self.dump_root, self.tfdbg_run_id, circular_buffer_size=-1)
for i in range(5):
execution = debug_event_pb2.Execution(op_type="OpType%d" % i)
writer.WriteExecution(execution)
writer.FlushExecutionFiles()
writer.Close()
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
reader.update()
executions = reader.executions(begin=begin, end=end)
self.assertLen(executions, expected_end - expected_begin)
self.assertEqual(executions[0].op_type, "OpType%d" % expected_begin)
self.assertEqual(executions[-1].op_type, "OpType%d" % (expected_end - 1))
@parameterized.named_parameters(
("Begin1End3", 1, 3, 1, 3),
("Begin0End3", 0, 3, 0, 3),
("Begin0EndNeg1", 0, -1, 0, 4),
("BeginNoneEnd3", None, 3, 0, 3),
("Begin2EndNone", 2, None, 2, 5),
("BeginNoneEndNone", None, None, 0, 5),
)
def testRangeReadingGraphExecutionTraces(self, begin, end, expected_begin,
expected_end):
writer = debug_events_writer.DebugEventsWriter(
self.dump_root, self.tfdbg_run_id, circular_buffer_size=-1)
debugged_graph = debug_event_pb2.DebuggedGraph(
graph_id="graph1", graph_name="graph1")
writer.WriteDebuggedGraph(debugged_graph)
for i in range(5):
op_name = "Op_%d" % i
graph_op_creation = debug_event_pb2.GraphOpCreation(
op_name=op_name, graph_id="graph1")
writer.WriteGraphOpCreation(graph_op_creation)
trace = debug_event_pb2.GraphExecutionTrace(
op_name=op_name, tfdbg_context_id="graph1")
writer.WriteGraphExecutionTrace(trace)
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
writer.Close()
with debug_events_reader.DebugDataReader(self.dump_root) as reader:
reader.update()
traces = reader.graph_execution_traces(begin=begin, end=end)
self.assertLen(traces, expected_end - expected_begin)
self.assertEqual(traces[0].op_name, "Op_%d" % expected_begin)
self.assertEqual(traces[-1].op_name, "Op_%d" % (expected_end - 1))
class MultiSetReaderTest(dumping_callback_test_lib.DumpingCallbackTestBase):
"""Test for DebugDataReader for multiple file sets under a dump root."""
def testReadingTwoFileSetsWithTheSameDumpRootSucceeds(self):
# To simulate a multi-host data dump, we first generate file sets in two
# different directories, with the same tfdbg_run_id, and then combine them.
tfdbg_run_id = "foo"
for i in range(2):
writer = debug_events_writer.DebugEventsWriter(
os.path.join(self.dump_root, str(i)),
tfdbg_run_id,
circular_buffer_size=-1)
if i == 0:
debugged_graph = debug_event_pb2.DebuggedGraph(
graph_id="graph1", graph_name="graph1")
writer.WriteDebuggedGraph(debugged_graph)
op_name = "Op_0"
graph_op_creation = debug_event_pb2.GraphOpCreation(
op_type="FooOp", op_name=op_name, graph_id="graph1")
writer.WriteGraphOpCreation(graph_op_creation)
op_name = "Op_1"
graph_op_creation = debug_event_pb2.GraphOpCreation(
op_type="FooOp", op_name=op_name, graph_id="graph1")
writer.WriteGraphOpCreation(graph_op_creation)
for _ in range(10):
trace = debug_event_pb2.GraphExecutionTrace(
op_name="Op_%d" % i, tfdbg_context_id="graph1")
writer.WriteGraphExecutionTrace(trace)
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
# Move all files from the subdirectory /1 to subdirectory /0.
dump_root_0 = os.path.join(self.dump_root, "0")
src_paths = glob.glob(os.path.join(self.dump_root, "1", "*"))
for src_path in src_paths:
dst_path = os.path.join(
dump_root_0,
# Rename the file set to avoid file name collision.
re.sub(r"(tfdbg_events\.\d+)", r"\g<1>1", os.path.basename(src_path)))
os.rename(src_path, dst_path)
with debug_events_reader.DebugDataReader(dump_root_0) as reader:
reader.update()
# Verify the content of the .graph_execution_traces file.
trace_digests = reader.graph_execution_traces(digest=True)
self.assertLen(trace_digests, 20)
for _ in range(10):
trace = reader.read_graph_execution_trace(trace_digests[i])
self.assertEqual(trace.op_name, "Op_0")
for _ in range(10):
trace = reader.read_graph_execution_trace(trace_digests[i + 10])
self.assertEqual(trace.op_name, "Op_1")
def testReadingTwoFileSetsWithTheDifferentRootsLeadsToError(self):
# To simulate a multi-host data dump, we first generate file sets in two
# different directories, with different tfdbg_run_ids, and then combine
# them.
for i in range(2):
writer = debug_events_writer.DebugEventsWriter(
os.path.join(self.dump_root, str(i)),
"run_id_%d" % i,
circular_buffer_size=-1)
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
# Move all files from the subdirectory /1 to subdirectory /0.
dump_root_0 = os.path.join(self.dump_root, "0")
src_paths = glob.glob(os.path.join(self.dump_root, "1", "*"))
for src_path in src_paths:
dst_path = os.path.join(
dump_root_0,
# Rename the file set to avoid file name collision.
re.sub(r"(tfdbg_events\.\d+)", r"\g<1>1", os.path.basename(src_path)))
os.rename(src_path, dst_path)
with self.assertRaisesRegex(ValueError,
r"Found multiple \(2\) tfdbg2 runs"):
debug_events_reader.DebugDataReader(dump_root_0)
class DataObjectsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
def jsonRoundTripCheck(self, obj):
self.assertEqual(
json_lib.dumps(json_lib.loads(json_lib.dumps(obj)), sort_keys=True),
json_lib.dumps(obj, sort_keys=True))
def testExecutionDigestWithNoOutputToJson(self):
execution_digest = debug_events_reader.ExecutionDigest(
1234, 5678, "FooOp", output_tensor_device_ids=None)
json = execution_digest.to_json()
self.jsonRoundTripCheck(json)
self.assertEqual(json["wall_time"], 1234)
self.assertEqual(json["op_type"], "FooOp")
self.assertEqual(json["output_tensor_device_ids"], None)
def testExecutionDigestWithTwoOutputsToJson(self):
execution_digest = debug_events_reader.ExecutionDigest(
1234, 5678, "FooOp", output_tensor_device_ids=[1357, 2468])
json = execution_digest.to_json()
self.jsonRoundTripCheck(json)
self.assertEqual(json["wall_time"], 1234)
self.assertEqual(json["op_type"], "FooOp")
self.assertEqual(json["output_tensor_device_ids"], (1357, 2468))
def testExecutionNoGraphNoInputToJson(self):
execution_digest = debug_events_reader.ExecutionDigest(
1234, 5678, "FooOp", output_tensor_device_ids=[1357])
execution = debug_events_reader.Execution(
execution_digest,
"localhost",
("a1", "b2"),
debug_event_pb2.TensorDebugMode.CURT_HEALTH,
graph_id=None,
input_tensor_ids=None,
output_tensor_ids=[2468],
debug_tensor_values=([1, 0],))
json = execution.to_json()
self.jsonRoundTripCheck(json)
self.assertEqual(json["wall_time"], 1234)
self.assertEqual(json["op_type"], "FooOp")
self.assertEqual(json["output_tensor_device_ids"], (1357,))
self.assertEqual(json["host_name"], "localhost")
self.assertEqual(json["stack_frame_ids"], ("a1", "b2"))
self.assertEqual(json["tensor_debug_mode"],
debug_event_pb2.TensorDebugMode.CURT_HEALTH)
self.assertIsNone(json["graph_id"])
self.assertIsNone(json["input_tensor_ids"])
self.assertEqual(json["output_tensor_ids"], (2468,))
self.assertEqual(json["debug_tensor_values"], ([1, 0],))
def testExecutionNoGraphNoInputButWithOutputToJson(self):
execution_digest = debug_events_reader.ExecutionDigest(
1234, 5678, "FooOp", output_tensor_device_ids=[1357])
execution = debug_events_reader.Execution(
execution_digest,
"localhost",
("a1", "b2"),
debug_event_pb2.TensorDebugMode.FULL_HEALTH,
graph_id="abcd",
input_tensor_ids=[13, 37],
output_tensor_ids=None,
debug_tensor_values=None)
json = execution.to_json()
self.jsonRoundTripCheck(json)
self.assertEqual(json["wall_time"], 1234)
self.assertEqual(json["op_type"], "FooOp")
self.assertEqual(json["output_tensor_device_ids"], (1357,))
self.assertEqual(json["host_name"], "localhost")
self.assertEqual(json["stack_frame_ids"], ("a1", "b2"))
self.assertEqual(json["tensor_debug_mode"],
debug_event_pb2.TensorDebugMode.FULL_HEALTH)
self.assertEqual(json["graph_id"], "abcd")
self.assertEqual(json["input_tensor_ids"], (13, 37))
self.assertIsNone(json["output_tensor_ids"])
self.assertIsNone(json["debug_tensor_values"])
@parameterized.named_parameters(
("EmptyList", []),
("None", None),
)
def testExecutionWithNoOutputTensorsReturnsZeroForNumOutputs(
self, output_tensor_ids):
execution = debug_events_reader.Execution(
debug_events_reader.ExecutionDigest(1234, 5678, "FooOp"),
"localhost", ("a1", "b2"),
debug_event_pb2.TensorDebugMode.FULL_HEALTH,
graph_id="abcd",
input_tensor_ids=[13, 37],
output_tensor_ids=output_tensor_ids,
debug_tensor_values=None)
self.assertEqual(execution.num_outputs, 0)
def testDebuggedDeviceToJons(self):
debugged_device = debug_events_reader.DebuggedDevice("/TPU:3", 4)
self.assertEqual(debugged_device.to_json(), {
"device_name": "/TPU:3",
"device_id": 4,
})
def testDebuggedGraphToJonsWitouthNameInnerOuterGraphIds(self):
debugged_graph = debug_events_reader.DebuggedGraph(
None,
"b1c2",
outer_graph_id=None,
)
self.assertEqual(
debugged_graph.to_json(), {
"name": None,
"graph_id": "b1c2",
"outer_graph_id": None,
"inner_graph_ids": [],
})
def testDebuggedGraphToJonsWithNameAndInnerOuterGraphIds(self):
debugged_graph = debug_events_reader.DebuggedGraph(
"loss_function",
"b1c2",
outer_graph_id="a0b1",
)
debugged_graph.add_inner_graph_id("c2d3")
debugged_graph.add_inner_graph_id("c2d3e4")
self.assertEqual(
debugged_graph.to_json(), {
"name": "loss_function",
"graph_id": "b1c2",
"outer_graph_id": "a0b1",
"inner_graph_ids": ["c2d3", "c2d3e4"],
})
@parameterized.named_parameters(
("EmptyList", []),
("None", None),
)
def testGraphOpDigestWithNoOutpusReturnsNumOutputsZero(
self, output_tensor_ids):
op_creation_digest = debug_events_reader.GraphOpCreationDigest(
1234,
5678,
"deadbeef",
"FooOp",
"Model_1/Foo_2",
output_tensor_ids,
"machine.cluster", ("a1", "a2"),
input_names=None,
device_name=None)
self.assertEqual(op_creation_digest.num_outputs, 0)
def testGraphOpCreationDigestNoInputNoDeviceNameToJson(self):
op_creation_digest = debug_events_reader.GraphOpCreationDigest(
1234,
5678,
"deadbeef",
"FooOp",
"Model_1/Foo_2", [135],
"machine.cluster", ("a1", "a2"),
input_names=None,
device_name=None)
json = op_creation_digest.to_json()
self.jsonRoundTripCheck(json)
self.assertEqual(json["wall_time"], 1234)
self.assertEqual(json["graph_id"], "deadbeef")
self.assertEqual(json["op_type"], "FooOp")
self.assertEqual(json["op_name"], "Model_1/Foo_2")
self.assertEqual(json["output_tensor_ids"], (135,))
self.assertEqual(json["host_name"], "machine.cluster")
self.assertEqual(json["stack_frame_ids"], ("a1", "a2"))
self.assertIsNone(json["input_names"])
self.assertIsNone(json["device_name"])
def testGraphOpCreationDigestWithInputsAndDeviceNameToJson(self):
op_creation_digest = debug_events_reader.GraphOpCreationDigest(
1234,
5678,
"deadbeef",
"FooOp",
"Model_1/Foo_2", [135],
"machine.cluster", ("a1", "a2"),
input_names=["Bar_1", "Qux_2"],
device_name="/device:GPU:0")
json = op_creation_digest.to_json()
self.jsonRoundTripCheck(json)
self.assertEqual(json["wall_time"], 1234)
self.assertEqual(json["graph_id"], "deadbeef")
self.assertEqual(json["op_type"], "FooOp")
self.assertEqual(json["op_name"], "Model_1/Foo_2")
self.assertEqual(json["output_tensor_ids"], (135,))
self.assertEqual(json["host_name"], "machine.cluster")
self.assertEqual(json["stack_frame_ids"], ("a1", "a2"))
self.assertEqual(json["input_names"], ("Bar_1", "Qux_2"))
self.assertEqual(json["device_name"], "/device:GPU:0")
def testGraphExecutionTraceDigestToJson(self):
trace_digest = debug_events_reader.GraphExecutionTraceDigest(
1234, 5678, "FooOp", "Model_1/Foo_2", 1, "deadbeef")
json = trace_digest.to_json()
self.assertEqual(json["wall_time"], 1234)
self.assertEqual(json["op_type"], "FooOp")
self.assertEqual(json["op_name"], "Model_1/Foo_2")
self.assertEqual(json["output_slot"], 1)
self.assertEqual(json["graph_id"], "deadbeef")
def testGraphExecutionTraceWithTensorDebugValueAndDeviceNameToJson(self):
trace_digest = debug_events_reader.GraphExecutionTraceDigest(
1234, 5678, "FooOp", "Model_1/Foo_2", 1, "deadbeef")
trace = debug_events_reader.GraphExecutionTrace(
trace_digest, ["g1", "g2", "deadbeef"],
debug_event_pb2.TensorDebugMode.CURT_HEALTH,
debug_tensor_value=[3, 1], device_name="/device:GPU:0")
json = trace.to_json()
self.assertEqual(json["wall_time"], 1234)
self.assertEqual(json["op_type"], "FooOp")
self.assertEqual(json["op_name"], "Model_1/Foo_2")
self.assertEqual(json["output_slot"], 1)
self.assertEqual(json["graph_id"], "deadbeef")
self.assertEqual(json["graph_ids"], ("g1", "g2", "deadbeef"))
self.assertEqual(json["tensor_debug_mode"],
debug_event_pb2.TensorDebugMode.CURT_HEALTH)
self.assertEqual(json["debug_tensor_value"], (3, 1))
self.assertEqual(json["device_name"], "/device:GPU:0")
def testGraphExecutionTraceNoTensorDebugValueNoDeviceNameToJson(self):
trace_digest = debug_events_reader.GraphExecutionTraceDigest(
1234, 5678, "FooOp", "Model_1/Foo_2", 1, "deadbeef")
trace = debug_events_reader.GraphExecutionTrace(
trace_digest, ["g1", "g2", "deadbeef"],
debug_event_pb2.TensorDebugMode.NO_TENSOR,
debug_tensor_value=None, device_name=None)
json = trace.to_json()
self.assertEqual(json["wall_time"], 1234)
self.assertEqual(json["op_type"], "FooOp")
self.assertEqual(json["op_name"], "Model_1/Foo_2")
self.assertEqual(json["output_slot"], 1)
self.assertEqual(json["graph_id"], "deadbeef")
self.assertEqual(json["graph_ids"], ("g1", "g2", "deadbeef"))
self.assertEqual(json["tensor_debug_mode"],
debug_event_pb2.TensorDebugMode.NO_TENSOR)
self.assertIsNone(json["debug_tensor_value"])
self.assertIsNone(json["device_name"])
if __name__ == "__main__":
ops.enable_eager_execution()
googletest.main()
tensorflow/python/debug/lib/debug_gradients.py
+412
View File
@@ -0,0 +1,412 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""TensorFlow Debugger: Tools for debugging gradients."""
import re
import uuid
from tensorflow.python.debug.lib import debug_data
from tensorflow.python.debug.lib import debug_graphs
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor as tensor_lib
from tensorflow.python.ops import gen_array_ops
from tensorflow.python.ops import variables
_GRADIENT_DEBUG_TAG = "gradient_debug_"
_gradient_debuggers = {}
def _tensor_to_grad_debug_op_name(tensor, grad_debugger_uuid):
op_name, slot = debug_graphs.parse_node_or_tensor_name(tensor.name)
return "%s_%d/%s%s" % (op_name, slot, _GRADIENT_DEBUG_TAG, grad_debugger_uuid)
def _parse_grad_debug_op_name(op_name):
"""Parse the name of a debug gradient op.
Args:
op_name: the name of the debug gradient op.
Returns:
1) The UUID of the GradientsDebugger that created the debug gradient op.
2) Name of the original tensor whose gradient is debugged by the debug
gradient op.
"""
name_items = op_name.split("/")
assert len(name_items) > 1
assert name_items[-1].startswith(_GRADIENT_DEBUG_TAG)
grad_debugger_uuid = name_items[-1][len(_GRADIENT_DEBUG_TAG):]
if "_" in grad_debugger_uuid:
grad_debugger_uuid = grad_debugger_uuid[:grad_debugger_uuid.index("_")]
orig_tensor_slot = int(name_items[-2][name_items[-2].rfind("_") + 1:])
orig_base_op_name = name_items[-2][:name_items[-2].rfind("_")]
orig_tensor_name = ("/".join(name_items[:-2] + [orig_base_op_name]) +
":%d" % orig_tensor_slot)
return grad_debugger_uuid, orig_tensor_name
class GradientsDebugger:
"""Gradients Debugger.
Allows retrieval of gradient tensors created by TensorFlow's automatic
differentiation algorithm, i.e., `tf.gradients` and optimizer classes that
use it.
"""
# TODO(cais): Add examples code in the doc string?
def __init__(self, y_tensor=None):
"""Constructor of GradientsDebugger.
Args:
y_tensor: optional: the `tf.Tensor` to be differentiated, i.e., the tensor
on the numerator of the differentiation.
"""
self._uuid = uuid.uuid4().hex
_gradient_debuggers[self._uuid] = self
# A dict mapping x-tensor names to gradient tensor. x-tensor refers to the
# independent tf.Tensor, i.e., the tensor on the denominator of the
# differentiation.
self._gradient_tensors = {}
self._y_tensor = y_tensor
self._graph = None
if y_tensor:
self._graph = y_tensor.graph
self._is_active_context = False
@property
def y_tensor(self):
return self._y_tensor
@property
def graph(self):
return self._graph
def __enter__(self):
self._is_active_context = True
def __exit__(self, unused_type, unused_value, unused_traceback):
self._is_active_context = False
def identify_gradient(self, input_tensor):
"""Create a debug identity tensor that registers and forwards gradients.
The side effect of this method is that when gradient tensor(s) are created
with respect to the any paths that include the `input_tensor`, the gradient
tensor(s) with respect to `input_tensor` will be registered with this
this `GradientsDebugger` instance and can later be retrieved, with the
methods `gradient_tensor` and `gradient_tensors`.
Example:
```python
x = tf.Variable(1.0)
y = tf.add(x, x)
grad_debugger = tf_debug.GradientsDebugger()
debug_y = grad_debugger.identify_gradient(y)
z = tf.square(debug_y)
# Create a train op under the grad_debugger context.
with grad_debugger:
train_op = tf.compat.v1.train.GradientDescentOptimizer(z)
# Now we can reflect through grad_debugger to get the gradient tensor
# with respect to y.
y_grad = grad_debugger.gradient_tensor(y)
```
Args:
input_tensor: the input `tf.Tensor` object whose related gradient tensors
are to be registered with this `GradientsDebugger` instance when they
are created, e.g., during `tf.gradients` calls or the construction
of optimization (training) op that uses `tf.gradients`.
Returns:
A forwarded identity of `input_tensor`, as a `tf.Tensor`.
Raises:
ValueError: If an op with name that duplicates the gradient-debugging op
already exists in the graph (highly unlikely).
"""
# TODO(cais): Allow overriding gradient.
# TODO(cais): Implement value_stack.
grad_debug_op_name = _tensor_to_grad_debug_op_name(input_tensor, self._uuid)
# pylint: disable=protected-access
identity_op = (
gen_array_ops.debug_gradient_ref_identity
if input_tensor.dtype._is_ref_dtype else
gen_array_ops.debug_gradient_identity)
# pylint: enable=protected-access
debug_grad_identity = identity_op(input_tensor, name=grad_debug_op_name)
assert debug_grad_identity.dtype == input_tensor.dtype
if debug_grad_identity.op.name != grad_debug_op_name:
raise ValueError(
"The graph already contains an op named %s" % grad_debug_op_name)
return debug_grad_identity
def watch_gradients_by_tensors(self, graph, tensors):
"""Watch gradient tensors by x-tensor(s).
The side effect of this method is that when gradient tensor(s) are created
with respect to the any paths that include the `x_tensor`s, the gradient
tensor(s) with respect to the tensor will be registered with this
this `GradientsDebugger` instance and can later be retrieved, with the
methods `gradient_tensor` and `gradient_tensors`.
Unlike the method `identify_gradient`, this method is used to retrieve
gradient tensors after the construction of the forward subgraph has
completed (but before the construction of the backward subgraph).
This method is the same as `watch_gradients_by_x_tensor_names` except that
the tensors are specified by the Python `tf.Tensor` or `tf.Variable`
objects, instead by name patterns.
Example:
```python
x = tf.Variable(1.0)
y = tf.add(x, x, name="y")
z = tf.square(debug_y)
# Create a train op under the grad_debugger context.
grad_debugger = tf_debug.GradientsDebugger()
with grad_debugger.watch_gradients_by_tensors(y):
train_op = tf.compat.v1.train.GradientDescentOptimizer(z)
# Now we can reflect through grad_debugger to get the gradient tensor
# with respect to y.
y_grad = grad_debugger.gradient_tensor(y)
# or
y_grad = grad_debugger.gradient_tensor("y:0")
```
Args:
graph: the `tf.Graph` to watch the gradients on.
tensors: a `tf.Tensor` or `tf.Variable` object, or a list of such objects.
Returns:
The GradientsDebugger instance itself.
"""
if not isinstance(tensors, list):
tensors = [tensors]
tensor_name_regex = []
for tensor in tensors:
tensor_name_regex.append(re.escape(tensor.name) + "$")
tensor_name_regex = "(" + "|".join(tensor_name_regex) + ")"
return self.watch_gradients_by_tensor_names(graph, tensor_name_regex)
def watch_gradients_by_tensor_names(self, graph, tensor_name_regex):
"""Watch gradient tensors by name(s) of the x-tensor(s).
The side effect of this method is that when gradient tensor(s) are created
with respect to the x-tensors, the gradient tensor(s) will be registered
with this `GradientsDebugger` instance and can later be retrieved.
Unlike the `identify_gradient` method, this method is used after the
construction of the forward graph has completed. Unlike the
`watch_gradients_by_tensor` method, this method does not use handles to the
tensors of interest; it uses their names.
This method is the same as `watch_gradients_by_tensors` except that the
x-tensors are specified by name patterns, instead of `tf.Tensor` or
`tf.Variable` objects.
Example:
```python
x = tf.Variable(1.0, name="x")
y = tf.add(x, x, name="y")
z = tf.square(debug_y)
# Create a train op under the grad_debugger context.
grad_debugger = tf_debug.GradientsDebugger()
with grad_debugger.watch_gradients_by_tensor_names(r"(x|y):0$"):
train_op = tf.compat.v1.train.GradientDescentOptimizer(z)
# Now we can reflect through grad_debugger to get the gradient tensor
# with respect to x and y.
x_grad = grad_debugger.gradient_tensor("x:0")
y_grad = grad_debugger.gradient_tensor("y:0")
```
Args:
graph: the `tf.Graph` to watch the gradients on.
tensor_name_regex: the regular-expression pattern of the name(s) of the
x-tensor(s) to watch. x-tensor refers to the tensors on the denominator
of the differentiation.
Returns:
The GradientsDebugger instance itself.
"""
tensor_name_pattern = re.compile(tensor_name_regex)
with graph.as_default():
for op in graph.get_operations():
for output in op.outputs:
if tensor_name_pattern.match(output.name):
debug_op = self.identify_gradient(output)
# Make a copy of output.consumers() since we'll modify the consumers
# TODO(skyewm): this is unnecessary once the C API is enabled
for consumer in list(output.consumers()):
if consumer == debug_op.op:
continue
# Locate the slot index of the original input.
for i, consumer_input in enumerate(consumer.inputs):
if consumer_input == output:
consumer._update_input(i, debug_op) # pylint: disable=protected-access
return self
def _check_same_graph(self, tensor):
if self._graph is None:
self._graph = tensor.graph
elif self._graph != tensor.graph:
raise ValueError(
"The graph of the value (%s) is not the same as the graph %s" %
(tensor.graph, self._graph))
def register_gradient_tensor(self,
x_tensor_name,
gradient_tensor):
"""Register the gradient tensor for an x-tensor.
Args:
x_tensor_name: (`str`) the name of the independent `tf.Tensor`, i.e.,
the tensor on the denominator of the differentiation.
gradient_tensor: the gradient `tf.Tensor`.
"""
if len(_gradient_debuggers) == 1 or self._is_active_context:
self._check_same_graph(gradient_tensor)
self._gradient_tensors[x_tensor_name] = gradient_tensor
def gradient_tensor(self, x_tensor):
"""Get the gradient tensor of an x-tensor.
Args:
x_tensor: (`tf.Tensor`, `tf.Variable` or `str`) The x-tensor object or its
name. x-tensor refers to the independent `tf.Tensor`, i.e., the tensor
on the denominator of the differentiation.
Returns:
If found, the gradient tensor.
Raises:
TypeError: If `x_tensor` is not a `tf.Tensor`, `tf.Variable` or `str`.
LookupError: If the `x_tensor` has not been registered with a gradient
tensor.
"""
x_tensor_name = self._get_tensor_name(x_tensor)
if x_tensor_name not in self._gradient_tensors:
raise LookupError(
"This GradientsDebugger has not received any gradient tensor for "
"x-tensor %s" % x_tensor_name)
return self._gradient_tensors[x_tensor_name]
def gradient_tensors(self):
"""Get the gradient tensors that this object is aware of.
Returns:
A dict mapping x-tensor names to gradient tensor objects. x-tensor refers
to the tensors on the denominator of the differentation.
"""
return self._gradient_tensors
def _get_tensor_name(self, tensor):
if isinstance(tensor, (tensor_lib.Tensor, variables.Variable)):
return tensor.name
elif isinstance(tensor, str):
return tensor
else:
raise TypeError(
"x_tensor must be a str or tf.Tensor or tf.Variable, "
"but instead has type %s" % type(tensor))
def clear_gradient_debuggers():
"""Clear all globally registered gradient debuggers."""
_gradient_debuggers.clear()
@ops.RegisterGradient("DebugGradientIdentity")
def _identify_gradient_grad(op, dy):
"""Gradient function for the DebugIdentity op."""
# TODO(cais): Allow overriding gradient.
grad_debugger_uuid, orig_tensor_name = _parse_grad_debug_op_name(op.name)
grad_debugger = _gradient_debuggers[grad_debugger_uuid]
grad_debugger.register_gradient_tensor(orig_tensor_name, dy)
return dy
@ops.RegisterGradient("DebugGradientRefIdentity")
def _identify_gradient_grad_ref(op, dy):
"""Gradient function for the DebugIdentity op."""
return _identify_gradient_grad(op, dy)
def gradient_values_from_dump(grad_debugger, x_tensor, dump):
"""Find gradient values from a `DebugDumpDir` object.
Args:
grad_debugger: the `tf_debug.GradientsDebugger` instance to be used.
x_tensor: (`tf.Tensor`, `tf.Variable` or `str`) The x-tensor object or its
name. x-tensor refers to the independent `tf.Tensor`, i.e., the tensor
on the denominator of the differentiation.
dump: A `tfdbg.DebugDumpDir` object.
Returns:
If this `GradientsDebugger` instance has the gradient tensor of `x_tensor`
registered: a list of `numpy.ndarray` representing the value of the
gradient tensor from `dump`. The list could be empty, if the gradient
tensor is not executed in the `tf.Session.run()` call that generated
the `dump`. The list could also contain multiple values of the gradient
tensor, e.g., if gradient tensor is computed repeatedly in a
`tf.while_loop` during the run that generated the `dump`.
Raises:
LookupError: If this `GradientsDebugger` instance does not have the
gradient tensor of `x_tensor` registered.
ValueError: If this `GradientsDebugger` has a `tf.Graph` object that
does not match the `tf.Graph` object of the `dump`.
TypeError: If `x_tensor` is not a `tf.Tensor`, `tf.Variable` or `str`.
"""
# TODO(cais): Use this method in LocalCLIDebugWrapperSession to present the
# gradient tensors to the TFDBG CLI.
# If possible, verify that the Python graph of the dump and that of this
# GradientsDebugger match.
if (dump.python_graph and grad_debugger.graph and
dump.python_graph != grad_debugger.graph):
raise ValueError(
"This GradientsDebugger instance has a graph (%s) that differs from "
"the graph of the DebugDumpDir object (%s)." %
(grad_debugger.graph, dump.python_graph))
gradient_tensor = grad_debugger.gradient_tensor(x_tensor)
node_name, output_slot = debug_graphs.parse_node_or_tensor_name(
gradient_tensor.name)
try:
return dump.get_tensors(node_name, output_slot, "DebugIdentity")
except debug_data.WatchKeyDoesNotExistInDebugDumpDirError:
return []
tensorflow/python/debug/lib/debug_gradients_test.py
+381
View File
@@ -0,0 +1,381 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Unit tests for debug_gradients module."""
import tempfile
from tensorflow.core.protobuf import config_pb2
from tensorflow.core.protobuf import rewriter_config_pb2
from tensorflow.python.client import session
from tensorflow.python.debug.lib import debug_data
from tensorflow.python.debug.lib import debug_gradients
from tensorflow.python.debug.lib import debug_utils
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.ops import gradients_impl
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import googletest
from tensorflow.python.training import gradient_descent
@test_util.run_v1_only("Sessions are not available in TF 2.x")
class IdentifyGradientTest(test_util.TensorFlowTestCase):
def setUp(self):
rewriter_config = rewriter_config_pb2.RewriterConfig(
disable_model_pruning=True,
dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF)
graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_config)
config = config_pb2.ConfigProto(graph_options=graph_options)
self.sess = session.Session(config=config)
with self.sess.as_default():
self.u = variables.Variable(2.0, name="u")
self.v = variables.Variable(3.0, name="v")
self.w = math_ops.multiply(self.u.value(), self.v.value(), name="w")
def tearDown(self):
ops.reset_default_graph()
debug_gradients.clear_gradient_debuggers()
def testIdentifyGradientGivesCorrectTensorObjectWithoutContextManager(self):
grad_debugger = debug_gradients.GradientsDebugger()
id_grad_w = grad_debugger.identify_gradient(self.w)
y = math_ops.add(id_grad_w, -1.0, name="y")
grads = gradients_impl.gradients(y, [self.u, self.v])
self.assertEqual(2, len(grads))
u_grad = grads[0]
v_grad = grads[1]
self.sess.run(variables.global_variables_initializer())
self.assertAllClose(5.0, self.sess.run(y))
self.assertAllClose(3.0, self.sess.run(u_grad))
self.assertAllClose(2.0, self.sess.run(v_grad))
# Fetch the gradient tensor with the x-tensor object.
w_grad = grad_debugger.gradient_tensor(self.w)
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
# Fetch the gradient tensor with the x-tensor's name.
w_grad = grad_debugger.gradient_tensor(self.w.name)
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
# Fetch the gradient tensor with the x-tensor name.
w_grad = grad_debugger.gradient_tensor(self.w.name)
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
def testIdentifyGradientGivesCorrectTensorObjectWithTfGradients(self):
grad_debugger = debug_gradients.GradientsDebugger()
id_grad_w = grad_debugger.identify_gradient(self.w)
y = math_ops.add(id_grad_w, -1.0, name="y")
with grad_debugger:
grads = gradients_impl.gradients(y, [self.u, self.v])
self.assertEqual(2, len(grads))
u_grad = grads[0]
v_grad = grads[1]
self.sess.run(variables.global_variables_initializer())
self.assertAllClose(5.0, self.sess.run(y))
self.assertAllClose(3.0, self.sess.run(u_grad))
self.assertAllClose(2.0, self.sess.run(v_grad))
# Fetch the gradient tensor with the x-tensor object.
w_grad = grad_debugger.gradient_tensor(self.w)
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
# Fetch the gradient tensor with the x-tensor's name.
w_grad = grad_debugger.gradient_tensor(self.w.name)
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
# Fetch the gradient tensor with the x-tensor name.
w_grad = grad_debugger.gradient_tensor(self.w.name)
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
def testCallingIdentifyGradientTwiceWithTheSameGradientsDebuggerErrors(self):
grad_debugger = debug_gradients.GradientsDebugger()
grad_debugger.identify_gradient(self.w)
with self.assertRaisesRegex(ValueError,
"The graph already contains an op named .*"):
grad_debugger.identify_gradient(self.w)
def testIdentifyGradientWorksOnMultipleLosses(self):
grad_debugger_1 = debug_gradients.GradientsDebugger()
grad_debugger_2 = debug_gradients.GradientsDebugger()
y = math_ops.add(self.w, -1.0, name="y")
debug_y = grad_debugger_1.identify_gradient(y)
z1 = math_ops.square(debug_y, name="z1")
debug_y = grad_debugger_2.identify_gradient(y)
z2 = math_ops.sqrt(debug_y, name="z2")
with grad_debugger_1:
gradient_descent.GradientDescentOptimizer(0.1).minimize(z1)
with grad_debugger_2:
gradient_descent.GradientDescentOptimizer(0.1).minimize(z2)
dz1_dy = grad_debugger_1.gradient_tensor(y)
dz2_dy = grad_debugger_2.gradient_tensor(y)
self.assertIsInstance(dz1_dy, tensor.Tensor)
self.assertIsInstance(dz2_dy, tensor.Tensor)
self.assertIsNot(dz1_dy, dz2_dy)
self.sess.run(variables.global_variables_initializer())
self.assertAllClose(5.0**2, self.sess.run(z1))
self.assertAllClose(5.0**0.5, self.sess.run(z2))
self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy))
self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy))
def testIdentifyGradientRaisesLookupErrorForUnknownXTensor(self):
grad_debugger_1 = debug_gradients.GradientsDebugger()
grad_debugger_2 = debug_gradients.GradientsDebugger()
id_grad_w = grad_debugger_1.identify_gradient(self.w)
y = math_ops.add(id_grad_w, -1.0, name="y")
# There are >1 gradient debuggers registered, and grad_debugger is not used
# as a context manager here, so the gradient w.r.t. self.w will not be
# registered.
gradients_impl.gradients(y, [self.u, self.v])
with self.assertRaisesRegex(
LookupError,
r"This GradientsDebugger has not received any gradient tensor for "):
grad_debugger_1.gradient_tensor(self.w)
with self.assertRaisesRegex(
LookupError,
r"This GradientsDebugger has not received any gradient tensor for "):
grad_debugger_2.gradient_tensor(self.w)
def testIdentifyGradientRaisesTypeErrorForNonTensorOrTensorNameInput(self):
grad_debugger = debug_gradients.GradientsDebugger()
with self.assertRaisesRegex(
TypeError,
r"x_tensor must be a str or tf\.Tensor or tf\.Variable, but instead "
r"has type .*Operation.*"):
grad_debugger.gradient_tensor(variables.global_variables_initializer())
def testIdentifyGradientTensorWorksWithGradientDescentOptimizer(self):
grad_debugger = debug_gradients.GradientsDebugger()
id_grad_w = grad_debugger.identify_gradient(self.w)
y = math_ops.add(id_grad_w, -1.0, name="y")
with grad_debugger:
gradient_descent.GradientDescentOptimizer(0.1).minimize(y)
self.sess.run(variables.global_variables_initializer())
# Fetch the gradient tensor with the x-tensor object.
w_grad = grad_debugger.gradient_tensor(self.w)
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
def testWatchGradientsByXTensorNamesWorks(self):
y = math_ops.add(self.w, -1.0, name="y")
# The constructrion of the forward graph has completed.
# But we can still get the gradient tensors by using
# watch_gradients_by_tensor_names().
grad_debugger = debug_gradients.GradientsDebugger()
with grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, "w:0$"):
grads = gradients_impl.gradients(y, [self.u, self.v])
self.assertEqual(2, len(grads))
u_grad = grads[0]
v_grad = grads[1]
self.sess.run(variables.global_variables_initializer())
self.assertAllClose(5.0, self.sess.run(y))
self.assertAllClose(3.0, self.sess.run(u_grad))
self.assertAllClose(2.0, self.sess.run(v_grad))
w_grad = grad_debugger.gradient_tensor(self.w)
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
w_grad = grad_debugger.gradient_tensor("w:0")
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
def testWatchGradientsByXTensorNamesWorksWithoutContextManager(self):
y = math_ops.add(self.w, -1.0, name="y")
# The constructrion of the forward graph has completed.
# But we can still get the gradient tensors by using
# watch_gradients_by_tensor_names().
grad_debugger = debug_gradients.GradientsDebugger()
grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, "w:0$")
grads = gradients_impl.gradients(y, [self.u, self.v])
self.assertEqual(2, len(grads))
u_grad = grads[0]
v_grad = grads[1]
self.sess.run(variables.global_variables_initializer())
self.assertAllClose(5.0, self.sess.run(y))
self.assertAllClose(3.0, self.sess.run(u_grad))
self.assertAllClose(2.0, self.sess.run(v_grad))
w_grad = grad_debugger.gradient_tensor(self.w)
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
w_grad = grad_debugger.gradient_tensor("w:0")
self.assertIsInstance(w_grad, tensor.Tensor)
self.assertAllClose(1.0, self.sess.run(w_grad))
def testWatchGradientsWorksOnRefTensor(self):
y = math_ops.add(self.w, -1.0, name="y")
grad_debugger = debug_gradients.GradientsDebugger()
with grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, "u:0$"):
grads = gradients_impl.gradients(y, [self.u, self.v])
self.assertEqual(2, len(grads))
u_grad = grads[0]
v_grad = grads[1]
self.assertIs(u_grad, grad_debugger.gradient_tensor("u:0"))
self.sess.run(variables.global_variables_initializer())
self.assertAllClose(3.0, self.sess.run(u_grad))
self.assertAllClose(2.0, self.sess.run(v_grad))
self.assertAllClose(3.0, self.sess.run(
grad_debugger.gradient_tensor("u:0")))
def testWatchGradientsWorksOnMultipleTensors(self):
y = math_ops.add(self.w, -1.0, name="y")
grad_debugger = debug_gradients.GradientsDebugger()
with grad_debugger.watch_gradients_by_tensor_names(self.sess.graph,
"(u|w):0$"):
grads = gradients_impl.gradients(y, [self.u, self.v])
self.assertEqual(2, len(grads))
u_grad = grads[0]
self.assertEqual(2, len(grad_debugger.gradient_tensors()))
self.assertIs(u_grad, grad_debugger.gradient_tensor("u:0"))
self.assertIsInstance(grad_debugger.gradient_tensor("w:0"), tensor.Tensor)
self.sess.run(variables.global_variables_initializer())
self.assertAllClose(1.0, self.sess.run(
grad_debugger.gradient_tensor("w:0")))
self.assertAllClose(3.0, self.sess.run(
grad_debugger.gradient_tensor("u:0")))
def testWatchGradientsByXTensorsWorks(self):
y = math_ops.add(self.w, -1.0, name="foo/y")
z = math_ops.square(y, name="foo/z")
# The constructrion of the forward graph has completed.
# But we can still get the gradient tensors by using
# watch_gradients_by_x_tensors().
grad_debugger = debug_gradients.GradientsDebugger()
with grad_debugger.watch_gradients_by_tensors(self.sess.graph,
[self.w, self.u, y]):
gradient_descent.GradientDescentOptimizer(0.1).minimize(z)
self.assertEqual(3, len(grad_debugger.gradient_tensors()))
u_grad = grad_debugger.gradient_tensor(self.u)
w_grad = grad_debugger.gradient_tensor(self.w)
y_grad = grad_debugger.gradient_tensor(y)
self.sess.run(variables.global_variables_initializer())
self.assertAllClose(10.0, self.sess.run(y_grad))
self.assertAllClose(10.0, self.sess.run(w_grad))
self.assertAllClose(30.0, self.sess.run(u_grad))
def testWatchGradientsByTensorCanWorkOnMultipleLosses(self):
y = math_ops.add(self.w, -1.0, name="y")
z1 = math_ops.square(y, name="z1")
z2 = math_ops.sqrt(y, name="z2")
grad_debugger_1 = debug_gradients.GradientsDebugger()
with grad_debugger_1.watch_gradients_by_tensors(self.sess.graph, y):
gradient_descent.GradientDescentOptimizer(0.1).minimize(z1)
grad_debugger_2 = debug_gradients.GradientsDebugger()
with grad_debugger_2.watch_gradients_by_tensors(self.sess.graph, y):
gradient_descent.GradientDescentOptimizer(0.1).minimize(z2)
dz1_dy = grad_debugger_1.gradient_tensor(y)
dz2_dy = grad_debugger_2.gradient_tensor(y)
self.assertIsInstance(dz1_dy, tensor.Tensor)
self.assertIsInstance(dz2_dy, tensor.Tensor)
self.assertIsNot(dz1_dy, dz2_dy)
self.sess.run(variables.global_variables_initializer())
self.assertAllClose(5.0**2, self.sess.run(z1))
self.assertAllClose(5.0**0.5, self.sess.run(z2))
self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy))
self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy))
def testGradientsValuesFromDumpWorks(self):
y = math_ops.add(self.w, -1.0, name="y")
z = math_ops.square(y, name="z")
grad_debugger = debug_gradients.GradientsDebugger()
with grad_debugger.watch_gradients_by_tensors(self.sess.graph,
[self.w, self.u, y]):
train_op = gradient_descent.GradientDescentOptimizer(0.1).minimize(z)
self.sess.run(variables.global_variables_initializer())
run_options = config_pb2.RunOptions(output_partition_graphs=True)
dump_dir = tempfile.mkdtemp()
debug_url = "file://" + dump_dir
debug_utils.watch_graph(run_options, self.sess.graph, debug_urls=debug_url)
run_metadata = config_pb2.RunMetadata()
self.assertAllClose(2.0, self.sess.run(self.u))
self.sess.run(train_op, options=run_options, run_metadata=run_metadata)
self.assertAllClose(-1.0, self.sess.run(self.u))
dump = debug_data.DebugDumpDir(
dump_dir, partition_graphs=run_metadata.partition_graphs)
dump.set_python_graph(self.sess.graph)
y_grad_values = debug_gradients.gradient_values_from_dump(
grad_debugger, y, dump)
self.assertEqual(1, len(y_grad_values))
self.assertAllClose(10.0, y_grad_values[0])
w_grad_values = debug_gradients.gradient_values_from_dump(
grad_debugger, self.w, dump)
self.assertEqual(1, len(w_grad_values))
self.assertAllClose(10.0, w_grad_values[0])
u_grad_values = debug_gradients.gradient_values_from_dump(
grad_debugger, self.u, dump)
self.assertEqual(1, len(u_grad_values))
self.assertAllClose(30.0, u_grad_values[0])
with self.assertRaisesRegex(
LookupError,
r"This GradientsDebugger has not received any gradient tensor for "
r"x-tensor v:0"):
debug_gradients.gradient_values_from_dump(grad_debugger, self.v, dump)
# Cleanup.
file_io.delete_recursively(dump_dir)
if __name__ == "__main__":
googletest.main()
tensorflow/python/debug/lib/debug_graph_reconstruction_test.py
+179
View File
@@ -0,0 +1,179 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for the reconstruction of non-debugger-decorated GraphDefs."""
import tempfile
from tensorflow.core.framework import graph_pb2
from tensorflow.core.protobuf import config_pb2
from tensorflow.core.protobuf import rewriter_config_pb2
from tensorflow.python.client import session
from tensorflow.python.debug.lib import debug_data
from tensorflow.python.debug.lib import debug_graphs
from tensorflow.python.debug.lib import debug_utils
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.ops import cond as tf_cond
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variables
from tensorflow.python.ops import while_loop
from tensorflow.python.platform import test
from tensorflow.python.training import gradient_descent
class ReconstructNonDebugGraphTest(test_util.TensorFlowTestCase):
_OP_TYPE_DENYLIST = ("_Send", "_Recv", "_HostSend", "_HostRecv", "_Retval")
def _no_rewrite_session_config(self):
rewriter_config = rewriter_config_pb2.RewriterConfig(
dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF,
pin_to_host_optimization=rewriter_config_pb2.RewriterConfig.OFF,
min_graph_nodes=-1)
graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_config)
return config_pb2.ConfigProto(graph_options=graph_options)
def setUp(self):
super(ReconstructNonDebugGraphTest, self).setUp()
self._dump_dir = tempfile.mkdtemp()
self._debug_url = "file://" + self._dump_dir
ops.reset_default_graph()
def tearDown(self):
file_io.delete_recursively(self._dump_dir)
super(ReconstructNonDebugGraphTest, self).tearDown()
def _graphDefWithoutDenylistedNodes(self, graph_def):
output_graph_def = graph_pb2.GraphDef()
for node in graph_def.node:
if node.op not in self._OP_TYPE_DENYLIST:
new_node = output_graph_def.node.add()
new_node.CopyFrom(node)
if new_node.op == "Enter":
# The debugger sets parallel_iterations attribute of while-loop Enter
# nodes to 1 for debugging.
for attr_key in new_node.attr:
if attr_key == "parallel_iterations":
new_node.attr[attr_key].i = 1
elif new_node.op == "Switch" or new_node.op == "Identity":
# We don't check the inputs to Switch or Identity ops as their inputs
# may be Send/Recv nodes.
del new_node.input[:]
return output_graph_def
def _compareOriginalAndReconstructedGraphDefs(self,
sess,
fetches,
feed_dict=None,
expected_output=None):
run_options = config_pb2.RunOptions(output_partition_graphs=True)
run_metadata = config_pb2.RunMetadata()
output = sess.run(fetches, feed_dict=feed_dict, options=run_options,
run_metadata=run_metadata)
if expected_output is not None:
self.assertAllClose(expected_output, output)
non_debug_graph_defs = run_metadata.partition_graphs
debug_utils.watch_graph(
run_options, sess.graph, debug_urls=self._debug_url)
run_metadata = config_pb2.RunMetadata()
output = sess.run(fetches, feed_dict=feed_dict, options=run_options,
run_metadata=run_metadata)
if expected_output is not None:
self.assertAllClose(expected_output, output)
dump = debug_data.DebugDumpDir(
self._dump_dir, partition_graphs=run_metadata.partition_graphs,
validate=True)
reconstructed = dump.reconstructed_non_debug_partition_graphs()
self.assertEqual(len(non_debug_graph_defs), len(reconstructed))
for i, non_debug_graph_def in enumerate(non_debug_graph_defs):
device_name = debug_graphs._infer_device_name(non_debug_graph_def)
test_util.assert_equal_graph_def(
self._graphDefWithoutDenylistedNodes(reconstructed[device_name]),
self._graphDefWithoutDenylistedNodes(non_debug_graph_def))
# Test debug_graphs.reconstruct_non_debug_graph_def.
reconstructed_again = (
debug_graphs.reconstruct_non_debug_graph_def(
run_metadata.partition_graphs[i]))
test_util.assert_equal_graph_def(
self._graphDefWithoutDenylistedNodes(reconstructed_again),
self._graphDefWithoutDenylistedNodes(non_debug_graph_def))
def testReconstructSimpleGraph(self):
with session.Session() as sess:
u = variables.Variable([12.0], name="u")
v = variables.Variable([30.0], name="v")
w = math_ops.add(u, v, name="w")
self.evaluate(u.initializer)
self.evaluate(v.initializer)
self._compareOriginalAndReconstructedGraphDefs(
sess, w, expected_output=[42.0])
def testReconstructGraphWithControlEdge(self):
with session.Session() as sess:
a = variables.Variable(10.0, name="a")
with ops.control_dependencies([a]):
b = math_ops.add(a, a, name="b")
with ops.control_dependencies([a, b]):
c = math_ops.multiply(b, b, name="c")
self.evaluate(a.initializer)
self._compareOriginalAndReconstructedGraphDefs(
sess, c, expected_output=400.0)
def testReconstructGraphWithCond(self):
with session.Session(config=self._no_rewrite_session_config()) as sess:
x = variables.Variable(10.0, name="x")
y = variables.Variable(20.0, name="y")
cond = tf_cond.cond(
x > y, lambda: math_ops.add(x, 1), lambda: math_ops.add(y, 1))
self.evaluate(x.initializer)
self.evaluate(y.initializer)
self._compareOriginalAndReconstructedGraphDefs(
sess, cond, expected_output=21.0)
def testReconstructGraphWithWhileLoop(self):
with session.Session(config=self._no_rewrite_session_config()) as sess:
loop_body = lambda i: math_ops.add(i, 2)
loop_cond = lambda i: math_ops.less(i, 16)
i = constant_op.constant(10, name="i")
loop = while_loop.while_loop(loop_cond, loop_body, [i])
self._compareOriginalAndReconstructedGraphDefs(sess, loop)
def testReconstructGraphWithGradients(self):
with session.Session(config=self._no_rewrite_session_config()) as sess:
u = variables.Variable(12.0, name="u")
v = variables.Variable(30.0, name="v")
x = constant_op.constant(1.1, name="x")
toy_loss = x * (u - v)
train_op = gradient_descent.GradientDescentOptimizer(
learning_rate=0.1).minimize(toy_loss, name="train_op")
self.evaluate(u.initializer)
self.evaluate(v.initializer)
self._compareOriginalAndReconstructedGraphDefs(sess, train_op)
if __name__ == "__main__":
test.main()
tensorflow/python/debug/lib/debug_graphs.py
+499
View File
@@ -0,0 +1,499 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Classes and methods for processing debugger-decorated graphs."""
from tensorflow.core.framework import graph_pb2
from tensorflow.python.framework import op_def_registry
from tensorflow.python.platform import tf_logging as logging
def parse_node_or_tensor_name(name):
"""Get the node name from a string that can be node or tensor name.
Args:
name: An input node name (e.g., "node_a") or tensor name (e.g.,
"node_a:0"), as a str.
Returns:
1) The node name, as a str. If the input name is a tensor name, i.e.,
consists of a colon, the final colon and the following output slot
will be stripped.
2) If the input name is a tensor name, the output slot, as an int. If
the input name is not a tensor name, None.
"""
if ":" in name and not name.endswith(":"):
node_name = name[:name.rfind(":")]
output_slot = int(name[name.rfind(":") + 1:])
return node_name, output_slot
else:
return name, None
def get_node_name(element_name):
node_name, _ = parse_node_or_tensor_name(element_name)
return node_name
def get_output_slot(element_name):
"""Get the output slot number from the name of a graph element.
If element_name is a node name without output slot at the end, 0 will be
assumed.
Args:
element_name: (`str`) name of the graph element in question.
Returns:
(`int`) output slot number.
"""
_, output_slot = parse_node_or_tensor_name(element_name)
return output_slot if output_slot is not None else 0
def is_copy_node(node_name):
"""Determine whether a node name is that of a debug Copy node.
Such nodes are inserted by TensorFlow core upon request in
RunOptions.debug_options.debug_tensor_watch_opts.
Args:
node_name: Name of the node.
Returns:
A bool indicating whether the input argument is the name of a debug Copy
node.
"""
return node_name.startswith("__copy_")
def is_debug_node(node_name):
"""Determine whether a node name is that of a debug node.
Such nodes are inserted by TensorFlow core upon request in
RunOptions.debug_options.debug_tensor_watch_opts.
Args:
node_name: Name of the node.
Returns:
A bool indicating whether the input argument is the name of a debug node.
"""
return node_name.startswith("__dbg_")
def parse_debug_node_name(node_name):
"""Parse the name of a debug node.
Args:
node_name: Name of the debug node.
Returns:
1. Name of the watched node, as a str.
2. Output slot index of the watched tensor, as an int.
3. Index of the debug node, as an int.
4. Name of the debug op, as a str, e.g, "DebugIdentity".
Raises:
ValueError: If the input node name is not a valid debug node name.
"""
prefix = "__dbg_"
name = node_name
if not name.startswith(prefix):
raise ValueError("Invalid prefix in debug node name: '%s'" % node_name)
name = name[len(prefix):]
if name.count("_") < 2:
raise ValueError("Invalid debug node name: '%s'" % node_name)
debug_op = name[name.rindex("_") + 1:]
name = name[:name.rindex("_")]
debug_op_index = int(name[name.rindex("_") + 1:])
name = name[:name.rindex("_")]
if name.count(":") != 1:
raise ValueError("Invalid tensor name in debug node name: '%s'" % node_name)
watched_node_name = name[:name.index(":")]
watched_output_slot = int(name[name.index(":") + 1:])
return watched_node_name, watched_output_slot, debug_op_index, debug_op
class GraphTracingReachedDestination(Exception):
pass
class DFSGraphTracer(object):
"""Graph input tracer using depth-first search."""
def __init__(self,
input_lists,
skip_node_names=None,
destination_node_name=None):
"""Constructor of _DFSGraphTracer.
Args:
input_lists: A list of dicts. Each dict is an adjacency (input) map from
the recipient node name as the key and the list of input node names
as the value.
skip_node_names: Optional: a list of node names to skip tracing.
destination_node_name: Optional: destination node name. If not `None`, it
should be the name of a destination not as a str and the graph tracing
will raise GraphTracingReachedDestination as soon as the node has been
reached.
Raises:
GraphTracingReachedDestination: if stop_at_node_name is not None and
the specified node is reached.
"""
self._input_lists = input_lists
self._skip_node_names = skip_node_names
self._inputs = []
self._visited_nodes = []
self._depth_count = 0
self._depth_list = []
self._destination_node_name = destination_node_name
def trace(self, graph_element_name):
"""Trace inputs.
Args:
graph_element_name: Name of the node or an output tensor of the node, as a
str.
Raises:
GraphTracingReachedDestination: if destination_node_name of this tracer
object is not None and the specified node is reached.
"""
self._depth_count += 1
node_name = get_node_name(graph_element_name)
if node_name == self._destination_node_name:
raise GraphTracingReachedDestination()
if node_name in self._skip_node_names:
return
if node_name in self._visited_nodes:
return
self._visited_nodes.append(node_name)
for input_list in self._input_lists:
if node_name not in input_list:
continue
for inp in input_list[node_name]:
if get_node_name(inp) in self._visited_nodes:
continue
self._inputs.append(inp)
self._depth_list.append(self._depth_count)
self.trace(inp)
self._depth_count -= 1
def inputs(self):
return self._inputs
def depth_list(self):
return self._depth_list
def _infer_device_name(graph_def):
"""Infer device name from a partition GraphDef."""
device_name = None
for node in graph_def.node:
if node.device:
device_name = node.device
break
if device_name is None:
logging.warn(
"Failed to infer device name from partition GraphDef: none of the "
"nodes of the GraphDef has a non-empty device name.")
return device_name
class DebugGraph(object):
"""Represents a debugger-decorated graph."""
def __init__(self, debug_graph_def, device_name=None):
self._debug_graph_def = debug_graph_def
self._non_debug_graph_def = None
self._node_attributes = {}
self._node_inputs = {}
self._node_reversed_ref_inputs = {}
self._node_ctrl_inputs = {}
self._node_recipients = {}
self._node_ctrl_recipients = {}
self._node_devices = {}
self._node_op_types = {}
self._copy_send_nodes = []
self._ref_args = {}
self._device_name = device_name
if not self._device_name:
self._device_name = _infer_device_name(debug_graph_def)
for node in debug_graph_def.node:
self._process_debug_graph_node(node)
self._prune_non_control_edges_of_debug_ops()
self._prune_control_edges_of_debug_ops()
self._prune_nodes_from_input_and_recipient_maps(self._get_copy_nodes())
self._populate_recipient_maps()
def _process_debug_graph_node(self, node):
"""Process a node from the debug GraphDef.
Args:
node: (NodeDef) A partition-graph node to be processed.
Raises:
ValueError: If duplicate node names are encountered.
"""
if is_debug_node(node.name):
# This is a debug node. Parse the node name and retrieve the
# information about debug watches on tensors. But do not include
# the node in the graph.
return
if node.name in self._node_inputs:
raise ValueError("Duplicate node name on device %s: '%s'" %
(self._device_name, node.name))
self._node_attributes[node.name] = node.attr
self._node_inputs[node.name] = []
self._node_ctrl_inputs[node.name] = []
self._node_recipients[node.name] = []
self._node_ctrl_recipients[node.name] = []
if node.name not in self._node_devices:
self._node_devices[node.name] = set()
self._node_devices[node.name].add(
node.device if node.device else self._device_name)
self._node_op_types[node.name] = node.op
self._ref_args[node.name] = self._get_ref_args(node)
for inp in node.input:
if is_copy_node(inp) and (node.op == "_Send" or node.op == "_Retval"):
self._copy_send_nodes.append(node.name)
if inp.startswith("^"):
cinp = inp[1:]
self._node_ctrl_inputs[node.name].append(cinp)
else:
self._node_inputs[node.name].append(inp)
def _get_ref_args(self, node):
"""Determine whether an input of an op is ref-type.
Args:
node: A `NodeDef`.
Returns:
A list of the arg names (as strs) that are ref-type.
"""
op_def = op_def_registry.get(node.op)
if op_def is None:
return []
ref_args = []
for i, output_arg in enumerate(op_def.output_arg):
if output_arg.is_ref:
arg_name = node.name if i == 0 else ("%s:%d" % (node.name, i))
ref_args.append(arg_name)
return ref_args
def _get_copy_nodes(self):
"""Find all Copy nodes in the loaded graph."""
copy_nodes = []
for node in self._node_inputs:
if is_copy_node(node):
copy_nodes.append(node)
return copy_nodes
def _prune_non_control_edges_of_debug_ops(self):
"""Prune (non-control) edges related to debug ops.
Prune the Copy ops and associated _Send ops inserted by the debugger out
from the non-control inputs and output recipients map. Replace the inputs
and recipients with original ones.
"""
for node in self._node_inputs:
inputs = self._node_inputs[node]
for i, inp in enumerate(inputs):
if is_copy_node(inp):
# Find the input to the Copy node, which should be the original
# input to the node.
orig_inp = self._node_inputs[inp][0]
inputs[i] = orig_inp
def _prune_control_edges_of_debug_ops(self):
"""Prune control edges related to the debug ops."""
for node in self._node_ctrl_inputs:
ctrl_inputs = self._node_ctrl_inputs[node]
debug_op_inputs = []
for ctrl_inp in ctrl_inputs:
if is_debug_node(ctrl_inp):
debug_op_inputs.append(ctrl_inp)
for debug_op_inp in debug_op_inputs:
ctrl_inputs.remove(debug_op_inp)
def _populate_recipient_maps(self):
"""Populate the map from node name to recipient(s) of its output(s).
This method also populates the input map based on reversed ref edges.
"""
for node in self._node_inputs:
inputs = self._node_inputs[node]
for inp in inputs:
inp = get_node_name(inp)
if inp not in self._node_recipients:
self._node_recipients[inp] = []
self._node_recipients[inp].append(node)
if inp in self._ref_args:
if inp not in self._node_reversed_ref_inputs:
self._node_reversed_ref_inputs[inp] = []
self._node_reversed_ref_inputs[inp].append(node)
for node in self._node_ctrl_inputs:
ctrl_inputs = self._node_ctrl_inputs[node]
for ctrl_inp in ctrl_inputs:
if ctrl_inp in self._copy_send_nodes:
continue
if ctrl_inp not in self._node_ctrl_recipients:
self._node_ctrl_recipients[ctrl_inp] = []
self._node_ctrl_recipients[ctrl_inp].append(node)
def _prune_nodes_from_input_and_recipient_maps(self, nodes_to_prune):
"""Prune nodes out of input and recipient maps.
Args:
nodes_to_prune: (`list` of `str`) Names of the nodes to be pruned.
"""
for node in nodes_to_prune:
del self._node_inputs[node]
del self._node_ctrl_inputs[node]
del self._node_recipients[node]
del self._node_ctrl_recipients[node]
def _reconstruct_non_debug_graph_def(self):
"""Reconstruct non-debug GraphDef.
Non-debug GraphDef means the original GraphDef without the Copy* and Debug
nodes inserted by the debugger.
"""
if self._non_debug_graph_def:
return
self._non_debug_graph_def = graph_pb2.GraphDef()
for node in self._debug_graph_def.node:
if is_copy_node(node.name) or is_debug_node(node.name):
continue
new_node = self._non_debug_graph_def.node.add()
new_node.CopyFrom(node)
# Redo the list of inputs, because in _debug_graph_def, the list can
# consist of Copy* and Debug* nodes inserted by the debugger. Those will
# be replaced with the original inputs here.
del new_node.input[:]
for inp in self._node_inputs[node.name]:
new_node.input.append(inp)
for ctrl_inp in self._node_ctrl_inputs[node.name]:
new_node.input.append("^" + ctrl_inp)
@property
def device_name(self):
return self._device_name
@property
def debug_graph_def(self):
"""The debugger-decorated GraphDef."""
return self._debug_graph_def
@property
def non_debug_graph_def(self):
"""The GraphDef without the Copy* and Debug* nodes added by the debugger."""
self._reconstruct_non_debug_graph_def()
return self._non_debug_graph_def
@property
def node_devices(self):
return self._node_devices
@property
def node_op_types(self):
return self._node_op_types
@property
def node_attributes(self):
return self._node_attributes
@property
def node_inputs(self):
return self._node_inputs
@property
def node_ctrl_inputs(self):
return self._node_ctrl_inputs
@property
def node_reversed_ref_inputs(self):
return self._node_reversed_ref_inputs
@property
def node_recipients(self):
return self._node_recipients
@property
def node_ctrl_recipients(self):
return self._node_ctrl_recipients
def reconstruct_non_debug_graph_def(debug_graph_def):
"""Reconstruct original (non-debugger-decorated) partition GraphDef.
This method strips the input `tf.compat.v1.GraphDef` of the Copy* and
Debug*-type nodes inserted by the debugger.
The reconstructed partition graph is identical to the original (i.e.,
non-debugger-decorated) partition graph except in the following respects:
1) The exact names of the runtime-inserted internal nodes may differ.
These include _Send, _Recv, _HostSend, _HostRecv, _Retval ops.
2) As a consequence of 1, the nodes that receive input directly from such
send- and recv-type ops will have different input names.
3) The parallel_iteration attribute of while-loop Enter ops are set to 1.
Args:
debug_graph_def: The debugger-decorated `tf.compat.v1.GraphDef`, with the
debugger-inserted Copy* and Debug* nodes.
Returns:
The reconstructed `tf.compat.v1.GraphDef` stripped of the debugger-inserted
nodes.
"""
return DebugGraph(debug_graph_def).non_debug_graph_def
tensorflow/python/debug/lib/debug_graphs_test.py
+108
View File
@@ -0,0 +1,108 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tfdbg module debug_data."""
from tensorflow.python.debug.lib import debug_graphs
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test
class ParseNodeOrTensorNameTest(test_util.TensorFlowTestCase):
def testParseNodeName(self):
node_name, slot = debug_graphs.parse_node_or_tensor_name(
"namespace1/node_1")
self.assertEqual("namespace1/node_1", node_name)
self.assertIsNone(slot)
def testParseTensorName(self):
node_name, slot = debug_graphs.parse_node_or_tensor_name(
"namespace1/node_2:3")
self.assertEqual("namespace1/node_2", node_name)
self.assertEqual(3, slot)
class GetNodeNameAndOutputSlotTest(test_util.TensorFlowTestCase):
def testParseTensorNameInputWorks(self):
self.assertEqual("a", debug_graphs.get_node_name("a:0"))
self.assertEqual(0, debug_graphs.get_output_slot("a:0"))
self.assertEqual("_b", debug_graphs.get_node_name("_b:1"))
self.assertEqual(1, debug_graphs.get_output_slot("_b:1"))
def testParseNodeNameInputWorks(self):
self.assertEqual("a", debug_graphs.get_node_name("a"))
self.assertEqual(0, debug_graphs.get_output_slot("a"))
class NodeNameChecksTest(test_util.TensorFlowTestCase):
def testIsCopyNode(self):
self.assertTrue(debug_graphs.is_copy_node("__copy_ns1/ns2/node3_0"))
self.assertFalse(debug_graphs.is_copy_node("copy_ns1/ns2/node3_0"))
self.assertFalse(debug_graphs.is_copy_node("_copy_ns1/ns2/node3_0"))
self.assertFalse(debug_graphs.is_copy_node("_copyns1/ns2/node3_0"))
self.assertFalse(debug_graphs.is_copy_node("__dbg_ns1/ns2/node3_0"))
def testIsDebugNode(self):
self.assertTrue(
debug_graphs.is_debug_node("__dbg_ns1/ns2/node3:0_0_DebugIdentity"))
self.assertFalse(
debug_graphs.is_debug_node("dbg_ns1/ns2/node3:0_0_DebugIdentity"))
self.assertFalse(
debug_graphs.is_debug_node("_dbg_ns1/ns2/node3:0_0_DebugIdentity"))
self.assertFalse(
debug_graphs.is_debug_node("_dbgns1/ns2/node3:0_0_DebugIdentity"))
self.assertFalse(debug_graphs.is_debug_node("__copy_ns1/ns2/node3_0"))
class ParseDebugNodeNameTest(test_util.TensorFlowTestCase):
def testParseDebugNodeName_valid(self):
debug_node_name_1 = "__dbg_ns_a/ns_b/node_c:1_0_DebugIdentity"
(watched_node, watched_output_slot, debug_op_index,
debug_op) = debug_graphs.parse_debug_node_name(debug_node_name_1)
self.assertEqual("ns_a/ns_b/node_c", watched_node)
self.assertEqual(1, watched_output_slot)
self.assertEqual(0, debug_op_index)
self.assertEqual("DebugIdentity", debug_op)
def testParseDebugNodeName_invalidPrefix(self):
invalid_debug_node_name_1 = "__copy_ns_a/ns_b/node_c:1_0_DebugIdentity"
with self.assertRaisesRegex(ValueError, "Invalid prefix"):
debug_graphs.parse_debug_node_name(invalid_debug_node_name_1)
def testParseDebugNodeName_missingDebugOpIndex(self):
invalid_debug_node_name_1 = "__dbg_node1:0_DebugIdentity"
with self.assertRaisesRegex(ValueError, "Invalid debug node name"):
debug_graphs.parse_debug_node_name(invalid_debug_node_name_1)
def testParseDebugNodeName_invalidWatchedTensorName(self):
invalid_debug_node_name_1 = "__dbg_node1_0_DebugIdentity"
with self.assertRaisesRegex(ValueError,
"Invalid tensor name in debug node name"):
debug_graphs.parse_debug_node_name(invalid_debug_node_name_1)
if __name__ == "__main__":
test.main()
tensorflow/python/debug/lib/debug_grappler_test.py
+121
View File
@@ -0,0 +1,121 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for debugger functionalities in tf.Session."""
import os
import tempfile
from tensorflow.core.protobuf import config_pb2
from tensorflow.core.protobuf import rewriter_config_pb2
from tensorflow.python.client import session
from tensorflow.python.debug.lib import debug_data
from tensorflow.python.debug.lib import debug_utils
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variable_v1
from tensorflow.python.ops import variables
from tensorflow.python.platform import googletest
def _grappler_enabled_session_config():
"""Constructs a Session config proto that explicitly enables Grappler.
Returns:
A config proto that obtains extra safety for the unit tests in this
file by ensuring that the relevant Grappler rewrites are always enabled.
"""
rewriter_config = rewriter_config_pb2.RewriterConfig(
disable_model_pruning=False,
arithmetic_optimization=rewriter_config_pb2.RewriterConfig.ON)
graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_config)
return config_pb2.ConfigProto(graph_options=graph_options)
class SessionDebugGrapplerInteractionTest(test_util.TensorFlowTestCase):
def setUp(self):
super(SessionDebugGrapplerInteractionTest, self).setUp()
self._dump_root = tempfile.mkdtemp()
self._debug_url = "file://%s" % self._dump_root
def tearDown(self):
ops.reset_default_graph()
if os.path.isdir(self._dump_root):
file_io.delete_recursively(self._dump_root)
super(SessionDebugGrapplerInteractionTest, self).tearDown()
def testArithmeticOptimizationActive(self):
"""Tests that tfdbg can dump the tensor from nodes created by Grappler."""
with session.Session(config=_grappler_enabled_session_config()) as sess:
u = variable_v1.VariableV1([[1, 2], [3, 4]],
name="u",
dtype=dtypes.float32)
# The next two ops should be optimized by Grappler into a single op:
# either an AddN op or a Mul op.
x = math_ops.add(u, u)
x = math_ops.add(x, u)
y = math_ops.multiply(x, u)
sess.run(variables.global_variables_initializer())
run_options = config_pb2.RunOptions(output_partition_graphs=True)
debug_utils.watch_graph(
run_options,
sess.graph,
debug_ops=["DebugIdentity"],
debug_urls=[self._debug_url])
run_metadata = config_pb2.RunMetadata()
run_result = sess.run(y, options=run_options, run_metadata=run_metadata)
self.assertAllClose(run_result, [[3, 12], [27, 48]])
dump_data = debug_data.DebugDumpDir(
self._dump_root, partition_graphs=run_metadata.partition_graphs,
validate=True)
original_node_names = set(op.name for op in sess.graph.get_operations())
dumped_node_names = set(dump_data.nodes())
grappler_created_node_names = dumped_node_names - original_node_names
grappler_removed_node_names = original_node_names - dumped_node_names
# Assert that Grappler should have replaced some of the nodes from the
# original graph with new nodes.
self.assertTrue(grappler_created_node_names)
self.assertTrue(grappler_removed_node_names)
# Iterate through the nodes created by Grappler. One of them should be
# be the result of replacing the original add ops with an AddN op or a
# Mul op.
found_optimized_node = False
for grappler_node_name in grappler_created_node_names:
node_op_type = dump_data.node_op_type(grappler_node_name)
# Look for the node created by Grappler's arithmetic optimization.
if ((test_util.IsMklEnabled() and node_op_type in ("_MklAddN", "Mul"))
or (node_op_type in ("AddN", "Mul"))):
datum = dump_data.get_tensors(grappler_node_name, 0, "DebugIdentity")
self.assertEqual(1, len(datum))
self.assertAllClose(datum[0], [[3, 6], [9, 12]])
found_optimized_node = True
break
self.assertTrue(
found_optimized_node,
"Failed to find optimized node created by Grappler's arithmetic "
"optimization.")
if __name__ == "__main__":
googletest.main()
tensorflow/python/debug/lib/debug_service_pb2_grpc.py
+107
View File
@@ -0,0 +1,107 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
#
# Do not use pylint on generated code.
# pylint: disable=missing-docstring,g-short-docstring-punctuation,g-no-space-after-docstring-summary,invalid-name,line-too-long,unused-argument,g-doc-args
import grpc
from tensorflow.core.debug import debug_service_pb2 as tensorflow_dot_core_dot_debug_dot_debug__service__pb2
from tensorflow.core.protobuf import debug_pb2 as tensorflow_dot_core_dot_protobuf_dot_debug__pb2
from tensorflow.core.util import event_pb2 as tensorflow_dot_core_dot_util_dot_event__pb2
class EventListenerStub(object):
"""EventListener: Receives Event protos, e.g., from debugged TensorFlow
runtime(s).
"""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.SendEvents = channel.stream_stream(
'/tensorflow.EventListener/SendEvents',
request_serializer=tensorflow_dot_core_dot_util_dot_event__pb2.Event.SerializeToString,
response_deserializer=tensorflow_dot_core_dot_debug_dot_debug__service__pb2.EventReply.FromString,
)
self.SendTracebacks = channel.unary_unary(
'/tensorflow.EventListener/SendTracebacks',
request_serializer=tensorflow_dot_core_dot_debug_dot_debug__service__pb2.CallTraceback.SerializeToString,
response_deserializer=tensorflow_dot_core_dot_debug_dot_debug__service__pb2.EventReply.FromString,
)
self.SendSourceFiles = channel.unary_unary(
'/tensorflow.EventListener/SendSourceFiles',
request_serializer=tensorflow_dot_core_dot_protobuf_dot_debug__pb2.DebuggedSourceFiles.SerializeToString,
response_deserializer=tensorflow_dot_core_dot_debug_dot_debug__service__pb2.EventReply.FromString,
)
class EventListenerServicer(object):
"""EventListener: Receives Event protos, e.g., from debugged TensorFlow
runtime(s).
"""
def SendEvents(self, request_iterator, context):
"""Client(s) can use this RPC method to send the EventListener Event protos.
The Event protos can hold information such as:
1) intermediate tensors from a debugged graph being executed, which can
be sent from DebugIdentity ops configured with grpc URLs.
2) GraphDefs of partition graphs, which can be sent from special debug
ops that get executed immediately after the beginning of the graph
execution.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def SendTracebacks(self, request, context):
"""Send the tracebacks of ops in a Python graph definition.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def SendSourceFiles(self, request, context):
"""Send a collection of source code files being debugged.
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_EventListenerServicer_to_server(servicer, server):
rpc_method_handlers = {
'SendEvents': grpc.stream_stream_rpc_method_handler(
servicer.SendEvents,
request_deserializer=tensorflow_dot_core_dot_util_dot_event__pb2.Event.FromString,
response_serializer=tensorflow_dot_core_dot_debug_dot_debug__service__pb2.EventReply.SerializeToString,
),
'SendTracebacks': grpc.unary_unary_rpc_method_handler(
servicer.SendTracebacks,
request_deserializer=tensorflow_dot_core_dot_debug_dot_debug__service__pb2.CallTraceback.FromString,
response_serializer=tensorflow_dot_core_dot_debug_dot_debug__service__pb2.EventReply.SerializeToString,
),
'SendSourceFiles': grpc.unary_unary_rpc_method_handler(
servicer.SendSourceFiles,
request_deserializer=tensorflow_dot_core_dot_protobuf_dot_debug__pb2.DebuggedSourceFiles.FromString,
response_serializer=tensorflow_dot_core_dot_debug_dot_debug__service__pb2.EventReply.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'tensorflow.EventListener', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
tensorflow/python/debug/lib/debug_utils.py
+286
View File
@@ -0,0 +1,286 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""TensorFlow Debugger (tfdbg) Utilities."""
import re
def add_debug_tensor_watch(run_options,
node_name,
output_slot=0,
debug_ops="DebugIdentity",
debug_urls=None,
tolerate_debug_op_creation_failures=False,
global_step=-1):
"""Add watch on a `Tensor` to `RunOptions`.
N.B.:
1. Under certain circumstances, the `Tensor` may not get actually watched
(e.g., if the node of the `Tensor` is constant-folded during runtime).
2. For debugging purposes, the `parallel_iteration` attribute of all
`tf.while_loop`s in the graph are set to 1 to prevent any node from
being executed multiple times concurrently. This change does not affect
subsequent non-debugged runs of the same `tf.while_loop`s.
Args:
run_options: An instance of `config_pb2.RunOptions` to be modified.
node_name: (`str`) name of the node to watch.
output_slot: (`int`) output slot index of the tensor from the watched node.
debug_ops: (`str` or `list` of `str`) name(s) of the debug op(s). Can be a
`list` of `str` or a single `str`. The latter case is equivalent to a
`list` of `str` with only one element.
For debug op types with customizable attributes, each debug op string can
optionally contain a list of attribute names, in the syntax of:
debug_op_name(attr_name_1=attr_value_1;attr_name_2=attr_value_2;...)
debug_urls: (`str` or `list` of `str`) URL(s) to send debug values to,
e.g., `file:///tmp/tfdbg_dump_1`, `grpc://localhost:12345`.
tolerate_debug_op_creation_failures: (`bool`) Whether to tolerate debug op
creation failures by not throwing exceptions.
global_step: (`int`) Optional global_step count for this debug tensor
watch.
"""
watch_opts = run_options.debug_options.debug_tensor_watch_opts
run_options.debug_options.global_step = global_step
watch = watch_opts.add()
watch.tolerate_debug_op_creation_failures = (
tolerate_debug_op_creation_failures)
watch.node_name = node_name
watch.output_slot = output_slot
if isinstance(debug_ops, str):
debug_ops = [debug_ops]
watch.debug_ops.extend(debug_ops)
if debug_urls:
if isinstance(debug_urls, str):
debug_urls = [debug_urls]
watch.debug_urls.extend(debug_urls)
def watch_graph(run_options,
graph,
debug_ops="DebugIdentity",
debug_urls=None,
node_name_regex_allowlist=None,
op_type_regex_allowlist=None,
tensor_dtype_regex_allowlist=None,
tolerate_debug_op_creation_failures=False,
global_step=-1,
reset_disk_byte_usage=False):
"""Add debug watches to `RunOptions` for a TensorFlow graph.
To watch all `Tensor`s on the graph, let both `node_name_regex_allowlist`
and `op_type_regex_allowlist` be the default (`None`).
N.B.:
1. Under certain circumstances, the `Tensor` may not get actually watched
(e.g., if the node of the `Tensor` is constant-folded during runtime).
2. For debugging purposes, the `parallel_iteration` attribute of all
`tf.while_loop`s in the graph are set to 1 to prevent any node from
being executed multiple times concurrently. This change does not affect
subsequent non-debugged runs of the same `tf.while_loop`s.
Args:
run_options: An instance of `config_pb2.RunOptions` to be modified.
graph: An instance of `ops.Graph`.
debug_ops: (`str` or `list` of `str`) name(s) of the debug op(s) to use.
debug_urls: URLs to send debug values to. Can be a list of strings,
a single string, or None. The case of a single string is equivalent to
a list consisting of a single string, e.g., `file:///tmp/tfdbg_dump_1`,
`grpc://localhost:12345`.
For debug op types with customizable attributes, each debug op name string
can optionally contain a list of attribute names, in the syntax of:
debug_op_name(attr_name_1=attr_value_1;attr_name_2=attr_value_2;...)
node_name_regex_allowlist: Regular-expression allowlist for node_name,
e.g., `"(weight_[0-9]+|bias_.*)"`
op_type_regex_allowlist: Regular-expression allowlist for the op type of
nodes, e.g., `"(Variable|Add)"`.
If both `node_name_regex_allowlist` and `op_type_regex_allowlist`
are set, the two filtering operations will occur in a logical `AND`
relation. In other words, a node will be included if and only if it
hits both allowlists.
tensor_dtype_regex_allowlist: Regular-expression allowlist for Tensor
data type, e.g., `"^int.*"`.
This allowlist operates in logical `AND` relations to the two allowlists
above.
tolerate_debug_op_creation_failures: (`bool`) whether debug op creation
failures (e.g., due to dtype incompatibility) are to be tolerated by not
throwing exceptions.
global_step: (`int`) Optional global_step count for this debug tensor
watch.
reset_disk_byte_usage: (`bool`) whether to reset the tracked disk byte
usage to zero (default: `False`).
"""
if not debug_ops:
raise ValueError("debug_ops must not be empty or None.")
if not debug_urls:
raise ValueError("debug_urls must not be empty or None.")
if isinstance(debug_ops, str):
debug_ops = [debug_ops]
node_name_pattern = (
re.compile(node_name_regex_allowlist)
if node_name_regex_allowlist else None)
op_type_pattern = (
re.compile(op_type_regex_allowlist) if op_type_regex_allowlist else None)
tensor_dtype_pattern = (
re.compile(tensor_dtype_regex_allowlist)
if tensor_dtype_regex_allowlist else None)
ops = graph.get_operations()
for op in ops:
# Skip nodes without any output tensors.
if not op.outputs:
continue
node_name = op.name
op_type = op.type
if node_name_pattern and not node_name_pattern.match(node_name):
continue
if op_type_pattern and not op_type_pattern.match(op_type):
continue
for slot in range(len(op.outputs)):
if (tensor_dtype_pattern and
not tensor_dtype_pattern.match(op.outputs[slot].dtype.name)):
continue
add_debug_tensor_watch(
run_options,
node_name,
output_slot=slot,
debug_ops=debug_ops,
debug_urls=debug_urls,
tolerate_debug_op_creation_failures=(
tolerate_debug_op_creation_failures),
global_step=global_step)
# If no filter for node or tensor is used, will add a wildcard node name, so
# that all nodes, including the ones created internally by TensorFlow itself
# (e.g., by Grappler), can be watched during debugging.
use_node_name_wildcard = (not node_name_pattern and
not op_type_pattern and
not tensor_dtype_pattern)
if use_node_name_wildcard:
add_debug_tensor_watch(
run_options,
"*",
output_slot=-1,
debug_ops=debug_ops,
debug_urls=debug_urls,
tolerate_debug_op_creation_failures=tolerate_debug_op_creation_failures,
global_step=global_step)
run_options.debug_options.reset_disk_byte_usage = reset_disk_byte_usage
def watch_graph_with_denylists(run_options,
graph,
debug_ops="DebugIdentity",
debug_urls=None,
node_name_regex_denylist=None,
op_type_regex_denylist=None,
tensor_dtype_regex_denylist=None,
tolerate_debug_op_creation_failures=False,
global_step=-1,
reset_disk_byte_usage=False):
"""Add debug tensor watches, denylisting nodes and op types.
This is similar to `watch_graph()`, but the node names and op types are
denylisted, instead of allowlisted.
N.B.:
1. Under certain circumstances, the `Tensor` may not get actually watched
(e.g., if the node of the `Tensor` is constant-folded during runtime).
2. For debugging purposes, the `parallel_iteration` attribute of all
`tf.while_loop`s in the graph are set to 1 to prevent any node from
being executed multiple times concurrently. This change does not affect
subsequent non-debugged runs of the same `tf.while_loop`s.
Args:
run_options: An instance of `config_pb2.RunOptions` to be modified.
graph: An instance of `ops.Graph`.
debug_ops: (`str` or `list` of `str`) name(s) of the debug op(s) to use. See
the documentation of `watch_graph` for more details.
debug_urls: URL(s) to send debug values to, e.g.,
`file:///tmp/tfdbg_dump_1`, `grpc://localhost:12345`.
node_name_regex_denylist: Regular-expression denylist for node_name. This
should be a string, e.g., `"(weight_[0-9]+|bias_.*)"`.
op_type_regex_denylist: Regular-expression denylist for the op type of
nodes, e.g., `"(Variable|Add)"`. If both node_name_regex_denylist and
op_type_regex_denylist are set, the two filtering operations will occur in
a logical `OR` relation. In other words, a node will be excluded if it
hits either of the two denylists; a node will be included if and only if
it hits neither of the denylists.
tensor_dtype_regex_denylist: Regular-expression denylist for Tensor data
type, e.g., `"^int.*"`. This denylist operates in logical `OR` relations
to the two allowlists above.
tolerate_debug_op_creation_failures: (`bool`) whether debug op creation
failures (e.g., due to dtype incompatibility) are to be tolerated by not
throwing exceptions.
global_step: (`int`) Optional global_step count for this debug tensor watch.
reset_disk_byte_usage: (`bool`) whether to reset the tracked disk byte
usage to zero (default: `False`).
"""
if isinstance(debug_ops, str):
debug_ops = [debug_ops]
node_name_pattern = (
re.compile(node_name_regex_denylist)
if node_name_regex_denylist else None)
op_type_pattern = (
re.compile(op_type_regex_denylist) if op_type_regex_denylist else None)
tensor_dtype_pattern = (
re.compile(tensor_dtype_regex_denylist)
if tensor_dtype_regex_denylist else None)
ops = graph.get_operations()
for op in ops:
# Skip nodes without any output tensors.
if not op.outputs:
continue
node_name = op.name
op_type = op.type
if node_name_pattern and node_name_pattern.match(node_name):
continue
if op_type_pattern and op_type_pattern.match(op_type):
continue
for slot in range(len(op.outputs)):
if (tensor_dtype_pattern and
tensor_dtype_pattern.match(op.outputs[slot].dtype.name)):
continue
add_debug_tensor_watch(
run_options,
node_name,
output_slot=slot,
debug_ops=debug_ops,
debug_urls=debug_urls,
tolerate_debug_op_creation_failures=(
tolerate_debug_op_creation_failures),
global_step=global_step)
run_options.debug_options.reset_disk_byte_usage = reset_disk_byte_usage
tensorflow/python/debug/lib/debug_utils_test.py
+364
View File
@@ -0,0 +1,364 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for TensorFlow Debugger (tfdbg) Utilities."""
import numpy as np
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.client import session
from tensorflow.python.debug.lib import debug_utils
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import test_util
from tensorflow.python.ops import math_ops
# Import resource_variable_ops for the variables-to-tensor implicit conversion.
from tensorflow.python.ops import resource_variable_ops # pylint: disable=unused-import
from tensorflow.python.ops import variable_v1
from tensorflow.python.platform import googletest
@test_util.run_v1_only("Requires tf.Session")
class DebugUtilsTest(test_util.TensorFlowTestCase):
@classmethod
def setUpClass(cls):
cls._sess = session.Session()
with cls._sess:
cls._a_init_val = np.array([[5.0, 3.0], [-1.0, 0.0]])
cls._b_init_val = np.array([[2.0], [-1.0]])
cls._c_val = np.array([[-4.0], [np.nan]])
cls._a_init = constant_op.constant(
cls._a_init_val, shape=[2, 2], name="a1_init")
cls._b_init = constant_op.constant(
cls._b_init_val, shape=[2, 1], name="b_init")
cls._a = variable_v1.VariableV1(cls._a_init, name="a1")
cls._b = variable_v1.VariableV1(cls._b_init, name="b")
cls._c = constant_op.constant(cls._c_val, shape=[2, 1], name="c")
# Matrix product of a and b.
cls._p = math_ops.matmul(cls._a, cls._b, name="p1")
# Sum of two vectors.
cls._s = math_ops.add(cls._p, cls._c, name="s")
cls._graph = cls._sess.graph
# These are all the expected nodes in the graph:
# - Two variables (a, b), each with four nodes (Variable, init, Assign,
# read).
# - One constant (c).
# - One add operation and one matmul operation.
# - One wildcard node name ("*") that covers nodes created internally
# by TensorFlow itself (e.g., Grappler).
cls._expected_num_nodes = 4 * 2 + 1 + 1 + 1 + 1
def setUp(self):
self._run_options = config_pb2.RunOptions()
def _verify_watches(self, watch_opts, expected_output_slot,
expected_debug_ops, expected_debug_urls):
"""Verify a list of debug tensor watches.
This requires all watches in the watch list have exactly the same
output_slot, debug_ops and debug_urls.
Args:
watch_opts: Repeated protobuf field of DebugTensorWatch.
expected_output_slot: Expected output slot index, as an integer.
expected_debug_ops: Expected debug ops, as a list of strings.
expected_debug_urls: Expected debug URLs, as a list of strings.
Returns:
List of node names from the list of debug tensor watches.
"""
node_names = []
for watch in watch_opts:
node_names.append(watch.node_name)
if watch.node_name == "*":
self.assertEqual(-1, watch.output_slot)
self.assertEqual(expected_debug_ops, watch.debug_ops)
self.assertEqual(expected_debug_urls, watch.debug_urls)
else:
self.assertEqual(expected_output_slot, watch.output_slot)
self.assertEqual(expected_debug_ops, watch.debug_ops)
self.assertEqual(expected_debug_urls, watch.debug_urls)
return node_names
def testAddDebugTensorWatches_defaultDebugOp(self):
debug_utils.add_debug_tensor_watch(
self._run_options, "foo/node_a", 1, debug_urls="file:///tmp/tfdbg_1")
debug_utils.add_debug_tensor_watch(
self._run_options, "foo/node_b", 0, debug_urls="file:///tmp/tfdbg_2")
debug_watch_opts = self._run_options.debug_options.debug_tensor_watch_opts
self.assertEqual(2, len(debug_watch_opts))
watch_0 = debug_watch_opts[0]
watch_1 = debug_watch_opts[1]
self.assertEqual("foo/node_a", watch_0.node_name)
self.assertEqual(1, watch_0.output_slot)
self.assertEqual("foo/node_b", watch_1.node_name)
self.assertEqual(0, watch_1.output_slot)
# Verify default debug op name.
self.assertEqual(["DebugIdentity"], watch_0.debug_ops)
self.assertEqual(["DebugIdentity"], watch_1.debug_ops)
# Verify debug URLs.
self.assertEqual(["file:///tmp/tfdbg_1"], watch_0.debug_urls)
self.assertEqual(["file:///tmp/tfdbg_2"], watch_1.debug_urls)
def testAddDebugTensorWatches_explicitDebugOp(self):
debug_utils.add_debug_tensor_watch(
self._run_options,
"foo/node_a",
0,
debug_ops="DebugNanCount",
debug_urls="file:///tmp/tfdbg_1")
debug_watch_opts = self._run_options.debug_options.debug_tensor_watch_opts
self.assertEqual(1, len(debug_watch_opts))
watch_0 = debug_watch_opts[0]
self.assertEqual("foo/node_a", watch_0.node_name)
self.assertEqual(0, watch_0.output_slot)
# Verify default debug op name.
self.assertEqual(["DebugNanCount"], watch_0.debug_ops)
# Verify debug URLs.
self.assertEqual(["file:///tmp/tfdbg_1"], watch_0.debug_urls)
def testAddDebugTensorWatches_multipleDebugOps(self):
debug_utils.add_debug_tensor_watch(
self._run_options,
"foo/node_a",
0,
debug_ops=["DebugNanCount", "DebugIdentity"],
debug_urls="file:///tmp/tfdbg_1")
debug_watch_opts = self._run_options.debug_options.debug_tensor_watch_opts
self.assertEqual(1, len(debug_watch_opts))
watch_0 = debug_watch_opts[0]
self.assertEqual("foo/node_a", watch_0.node_name)
self.assertEqual(0, watch_0.output_slot)
# Verify default debug op name.
self.assertEqual(["DebugNanCount", "DebugIdentity"], watch_0.debug_ops)
# Verify debug URLs.
self.assertEqual(["file:///tmp/tfdbg_1"], watch_0.debug_urls)
def testAddDebugTensorWatches_multipleURLs(self):
debug_utils.add_debug_tensor_watch(
self._run_options,
"foo/node_a",
0,
debug_ops="DebugNanCount",
debug_urls=["file:///tmp/tfdbg_1", "file:///tmp/tfdbg_2"])
debug_watch_opts = self._run_options.debug_options.debug_tensor_watch_opts
self.assertEqual(1, len(debug_watch_opts))
watch_0 = debug_watch_opts[0]
self.assertEqual("foo/node_a", watch_0.node_name)
self.assertEqual(0, watch_0.output_slot)
# Verify default debug op name.
self.assertEqual(["DebugNanCount"], watch_0.debug_ops)
# Verify debug URLs.
self.assertEqual(["file:///tmp/tfdbg_1", "file:///tmp/tfdbg_2"],
watch_0.debug_urls)
def testWatchGraph_allNodes(self):
debug_utils.watch_graph(
self._run_options,
self._graph,
debug_ops=["DebugIdentity", "DebugNanCount"],
debug_urls="file:///tmp/tfdbg_1")
debug_watch_opts = self._run_options.debug_options.debug_tensor_watch_opts
self.assertEqual(self._expected_num_nodes, len(debug_watch_opts))
# Verify that each of the nodes in the graph with output tensors in the
# graph have debug tensor watch.
node_names = self._verify_watches(debug_watch_opts, 0,
["DebugIdentity", "DebugNanCount"],
["file:///tmp/tfdbg_1"])
# Verify the node names.
self.assertIn("a1_init", node_names)
self.assertIn("a1", node_names)
self.assertIn("a1/Assign", node_names)
self.assertIn("a1/read", node_names)
self.assertIn("b_init", node_names)
self.assertIn("b", node_names)
self.assertIn("b/Assign", node_names)
self.assertIn("b/read", node_names)
self.assertIn("c", node_names)
self.assertIn("p1", node_names)
self.assertIn("s", node_names)
# Assert that the wildcard node name has been created.
self.assertIn("*", node_names)
def testWatchGraph_nodeNameAllowlist(self):
debug_utils.watch_graph(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
node_name_regex_allowlist="(a1$|a1_init$|a1/.*|p1$)")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertEqual(
sorted(["a1_init", "a1", "a1/Assign", "a1/read", "p1"]),
sorted(node_names))
def testWatchGraph_opTypeAllowlist(self):
debug_utils.watch_graph(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
op_type_regex_allowlist="(Variable|MatMul)")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertEqual(sorted(["a1", "b", "p1"]), sorted(node_names))
def testWatchGraph_nodeNameAndOpTypeAllowlists(self):
debug_utils.watch_graph(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
node_name_regex_allowlist="([a-z]+1$)",
op_type_regex_allowlist="(MatMul)")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertEqual(["p1"], node_names)
def testWatchGraph_tensorDTypeAllowlist(self):
debug_utils.watch_graph(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
tensor_dtype_regex_allowlist=".*_ref")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertItemsEqual(["a1", "a1/Assign", "b", "b/Assign"], node_names)
def testWatchGraph_nodeNameAndTensorDTypeAllowlists(self):
debug_utils.watch_graph(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
node_name_regex_allowlist="^a.*",
tensor_dtype_regex_allowlist=".*_ref")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertItemsEqual(["a1", "a1/Assign"], node_names)
def testWatchGraph_nodeNameDenylist(self):
debug_utils.watch_graph_with_denylists(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
node_name_regex_denylist="(a1$|a1_init$|a1/.*|p1$)")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertEqual(
sorted(["b_init", "b", "b/Assign", "b/read", "c", "s"]),
sorted(node_names))
def testWatchGraph_opTypeDenylist(self):
debug_utils.watch_graph_with_denylists(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
op_type_regex_denylist="(Variable|Identity|Assign|Const)")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertEqual(sorted(["p1", "s"]), sorted(node_names))
def testWatchGraph_nodeNameAndOpTypeDenylists(self):
debug_utils.watch_graph_with_denylists(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
node_name_regex_denylist="p1$",
op_type_regex_denylist="(Variable|Identity|Assign|Const)")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertEqual(["s"], node_names)
def testWatchGraph_tensorDTypeDenylists(self):
debug_utils.watch_graph_with_denylists(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
tensor_dtype_regex_denylist=".*_ref")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertNotIn("a1", node_names)
self.assertNotIn("a1/Assign", node_names)
self.assertNotIn("b", node_names)
self.assertNotIn("b/Assign", node_names)
self.assertIn("s", node_names)
def testWatchGraph_nodeNameAndTensorDTypeDenylists(self):
debug_utils.watch_graph_with_denylists(
self._run_options,
self._graph,
debug_urls="file:///tmp/tfdbg_1",
node_name_regex_denylist="^s$",
tensor_dtype_regex_denylist=".*_ref")
node_names = self._verify_watches(
self._run_options.debug_options.debug_tensor_watch_opts, 0,
["DebugIdentity"], ["file:///tmp/tfdbg_1"])
self.assertNotIn("a1", node_names)
self.assertNotIn("a1/Assign", node_names)
self.assertNotIn("b", node_names)
self.assertNotIn("b/Assign", node_names)
self.assertNotIn("s", node_names)
if __name__ == "__main__":
googletest.main()
tensorflow/python/debug/lib/debug_v2_ops_test.py
+806
View File
@@ -0,0 +1,806 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Test for the internal ops used by tfdbg v2."""
import os
import numpy as np
from tensorflow.core.protobuf import debug_event_pb2
from tensorflow.python.debug.lib import debug_events_reader
from tensorflow.python.debug.lib import debug_events_writer
from tensorflow.python.debug.lib import dumping_callback_test_lib
from tensorflow.python.eager import def_function
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_util
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_debug_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.platform import googletest
class DebugIdentityV2OpTest(dumping_callback_test_lib.DumpingCallbackTestBase):
"""Tests for DebugIdentityV2Op: when DebugEventsWriter is initialized.
DebugEventsWriter being initialized prior to DebugIdentityV2 ops being invoked
for the first time is the typical case (e.g., tfdbg2 running on a local
machine with only local devices.)
"""
def setUp(self):
super(DebugIdentityV2OpTest, self).setUp()
# Testing using a small circular-buffer size.
self.circular_buffer_size = 4
self.tfdbg_run_id = "test_tfdbg_run"
self.writer = debug_events_writer.DebugEventsWriter(
self.dump_root, self.tfdbg_run_id, self.circular_buffer_size)
def tearDown(self):
self.writer.Close()
super(DebugIdentityV2OpTest, self).tearDown()
@test_util.run_in_graph_and_eager_modes
def testSingleTensorFullTensorDebugModeWithCircularBufferBehavior(self):
@def_function.function
def write_debug_trace(x):
square = math_ops.square(x)
gen_debug_ops.debug_identity_v2(
square,
tfdbg_context_id="deadbeaf",
op_name="Square",
output_slot=0,
tensor_debug_mode=debug_event_pb2.TensorDebugMode.FULL_TENSOR,
debug_urls=["file://%s" % self.dump_root])
sqrt = math_ops.sqrt(x)
gen_debug_ops.debug_identity_v2(
sqrt,
tfdbg_context_id="beafdead",
op_name="Sqrt",
output_slot=0,
tensor_debug_mode=debug_event_pb2.TensorDebugMode.FULL_TENSOR,
debug_urls=["file://%s" % self.dump_root])
return square + sqrt
x = np.array([3.0, 4.0])
# Only the graph-execution trace of the last iteration should be written
# to self.dump_root.
for _ in range(self.circular_buffer_size // 2 + 1):
self.assertAllClose(
write_debug_trace(x), [9.0 + np.sqrt(3.0), 16.0 + 2.0])
with debug_events_reader.DebugEventsReader(self.dump_root) as reader:
# Check that the .metadata DebugEvents data file has been created, even
# before FlushExecutionFiles() is called.
self.assertGreater(reader.starting_wall_time(), 0)
self.assertTrue(reader.tensorflow_version())
self.assertTrue(reader.tfdbg_file_version().startswith("debug.Event"))
graph_trace_iter = reader.graph_execution_traces_iterators()[0]
# Before FlushExecutionFiles() is called, the .graph_execution_traces file
# ought to be empty.
with self.assertRaises(StopIteration):
next(graph_trace_iter)
# Flush the circular buffer.
self.writer.FlushExecutionFiles()
graph_trace_iter = reader.graph_execution_traces_iterators()[0]
# The circular buffer has a size of 4. So only the data from the
# last two iterations should have been written to self.dump_root.
for _ in range(2):
debug_event = next(graph_trace_iter).debug_event
self.assertGreater(debug_event.wall_time, 0)
trace = debug_event.graph_execution_trace
self.assertEqual(trace.tfdbg_context_id, "deadbeaf")
self.assertEqual(trace.op_name, "Square")
self.assertEqual(trace.output_slot, 0)
self.assertEqual(trace.tensor_debug_mode,
debug_event_pb2.TensorDebugMode.FULL_TENSOR)
tensor_value = tensor_util.MakeNdarray(trace.tensor_proto)
self.assertAllClose(tensor_value, [9.0, 16.0])
debug_event = next(graph_trace_iter).debug_event
self.assertGreater(debug_event.wall_time, 0)
trace = debug_event.graph_execution_trace
self.assertEqual(trace.tfdbg_context_id, "beafdead")
self.assertEqual(trace.op_name, "Sqrt")
self.assertEqual(trace.output_slot, 0)
self.assertEqual(trace.tensor_debug_mode,
debug_event_pb2.TensorDebugMode.FULL_TENSOR)
tensor_value = tensor_util.MakeNdarray(trace.tensor_proto)
self.assertAllClose(tensor_value, [np.sqrt(3.0), 2.0])
# Only the graph-execution trace of the last iteration should be written
# to self.dump_root.
with self.assertRaises(StopIteration):
next(graph_trace_iter)
@test_util.run_in_graph_and_eager_modes
def testControlFlow(self):
@def_function.function
def collatz(x):
counter = constant_op.constant(0, dtype=dtypes.int32)
while math_ops.greater(x, 1):
counter = counter + 1
gen_debug_ops.debug_identity_v2(
x,
tfdbg_context_id="deadbeaf",
op_name="x",
output_slot=0,
tensor_debug_mode=debug_event_pb2.TensorDebugMode.FULL_TENSOR,
debug_urls=["file://%s" % self.dump_root])
if math_ops.equal(x % 2, 0):
x = math_ops.div(x, 2)
else:
x = x * 3 + 1
return counter
x = constant_op.constant(10, dtype=dtypes.int32)
self.evaluate(collatz(x))
self.writer.FlushExecutionFiles()
with debug_events_reader.DebugEventsReader(self.dump_root) as reader:
graph_trace_iter = reader.graph_execution_traces_iterators()[0]
try:
x_values = []
timestamp = 0
while True:
debug_event = next(graph_trace_iter).debug_event
self.assertGreater(debug_event.wall_time, timestamp)
timestamp = debug_event.wall_time
trace = debug_event.graph_execution_trace
self.assertEqual(trace.tfdbg_context_id, "deadbeaf")
self.assertEqual(trace.op_name, "x")
self.assertEqual(trace.output_slot, 0)
self.assertEqual(trace.tensor_debug_mode,
debug_event_pb2.TensorDebugMode.FULL_TENSOR)
x_values.append(int(tensor_util.MakeNdarray(trace.tensor_proto)))
except StopIteration:
pass
# Due to the circular buffer, only the last 4 iterations of
# [10, 5, 16, 8, 4, 2] should have been written.
self.assertAllEqual(x_values, [16, 8, 4, 2])
@test_util.run_in_graph_and_eager_modes
def testTwoDumpRoots(self):
another_dump_root = os.path.join(self.dump_root, "another")
another_debug_url = "file://%s" % another_dump_root
another_writer = debug_events_writer.DebugEventsWriter(
another_dump_root, "test_tfdbg_run")
@def_function.function
def write_debug_trace(x):
# DebugIdentityV2 is a stateful op. It ought to be included by auto
# control dependency.
square = math_ops.square(x)
gen_debug_ops.debug_identity_v2(
square,
tfdbg_context_id="deadbeaf",
tensor_debug_mode=debug_event_pb2.TensorDebugMode.FULL_TENSOR,
debug_urls=["file://%s" % self.dump_root, another_debug_url])
return square + 1.0
x = np.array([3.0, 4.0])
self.assertAllClose(write_debug_trace(x), np.array([10.0, 17.0]))
self.writer.FlushExecutionFiles()
another_writer.FlushExecutionFiles()
another_writer.Close()
for debug_root in (self.dump_root, another_dump_root):
with debug_events_reader.DebugEventsReader(debug_root) as reader:
graph_trace_iter = reader.graph_execution_traces_iterators()[0]
debug_event = next(graph_trace_iter).debug_event
trace = debug_event.graph_execution_trace
self.assertEqual(trace.tfdbg_context_id, "deadbeaf")
self.assertEqual(trace.op_name, "")
self.assertEqual(trace.tensor_debug_mode,
debug_event_pb2.TensorDebugMode.FULL_TENSOR)
tensor_value = tensor_util.MakeNdarray(trace.tensor_proto)
self.assertAllClose(tensor_value, [9.0, 16.0])
with self.assertRaises(StopIteration):
next(graph_trace_iter)
class DebugIdentityV2OpUninitializedWriterTest(
dumping_callback_test_lib.DumpingCallbackTestBase):
"""Tests for DebugIdentityV2Op: when DebugEventsWriter is not initialized.
This case can occur when DebugIdentityV2Ops are running on a remote
TensorFlow server (e.g., a TPU worker).
"""
@test_util.run_in_graph_and_eager_modes
def testInvokingDebugIdentityV2OpBeforeCreatingDebugEventsWriterWorks(self):
circular_buffer_size = 3
@def_function.function
def write_debug_trace(x):
# DebugIdentityV2 is a stateful op. It ought to be included by auto
# control dependency.
square = math_ops.square(x)
gen_debug_ops.debug_identity_v2(
square,
tfdbg_context_id="deadbeaf",
op_name="Square",
output_slot=0,
tensor_debug_mode=debug_event_pb2.TensorDebugMode.FULL_TENSOR,
debug_urls=["file://%s" % self.dump_root],
circular_buffer_size=circular_buffer_size)
return square
# The DebugIdentityV2 ops are invokes *before* a DebugEventsWriter at the
# same dump root is created.
for i in range(circular_buffer_size * 2):
self.assertAllClose(
write_debug_trace(np.array([i]).astype(np.float32)), [i**2.0])
writer = debug_events_writer.DebugEventsWriter(self.dump_root,
"test_tfdbg_run",
circular_buffer_size)
writer.FlushNonExecutionFiles()
writer.FlushExecutionFiles()
with debug_events_reader.DebugEventsReader(self.dump_root) as reader:
graph_trace_iter = reader.graph_execution_traces_iterators()[0]
graph_execution_traces = []
while True:
try:
graph_execution_traces.append(
next(graph_trace_iter).debug_event.graph_execution_trace)
except StopIteration:
break
self.assertLen(graph_execution_traces, circular_buffer_size)
for i in range(circular_buffer_size):
self.assertAllClose(
tensor_util.MakeNdarray(graph_execution_traces[i].tensor_proto),
[(i + circular_buffer_size)**2.0])
class DebugNumericSummaryV2Test(test_util.TensorFlowTestCase):
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpReduceInfNanThreeSlots(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(
debug_event_pb2.TensorDebugMode.REDUCE_INF_NAN_THREE_SLOTS)))
self.assertAllEqual(
debug_summary(constant_op.constant([])), [0.0, 0.0, 0.0])
self.assertAllEqual(
debug_summary(constant_op.constant(42.0)), [0.0, 0.0, 0.0])
self.assertAllEqual(
debug_summary(constant_op.constant([3.0, 4.0])), [0.0, 0.0, 0.0])
self.assertAllEqual(
debug_summary(constant_op.constant(np.array([3.0, -np.inf]))),
[-np.inf, 0.0, 0.0])
self.assertAllEqual(
debug_summary(constant_op.constant(np.array([[0, 0], [np.nan, 0]]))),
[0.0, 0.0, np.nan])
self.assertAllEqual(
debug_summary(
constant_op.constant(np.array([[0, 0], [np.nan, np.inf]]))),
[0.0, np.inf, np.nan])
self.assertAllEqual(
debug_summary(
constant_op.constant(np.array([[0, np.inf], [np.nan, -np.inf]]))),
[-np.inf, np.inf, np.nan])
x = np.zeros([100, 100], dtype=np.float16)
x[32, 47] = np.nan
self.assertAllEqual(
debug_summary(constant_op.constant(x)), [0.0, 0.0, np.nan])
x = np.zeros([97, 97], dtype=np.float32)
x[50, 83] = -np.inf
self.assertAllEqual(
debug_summary(constant_op.constant(x)), [-np.inf, 0.0, 0.0])
x[1, 41] = np.nan
self.assertAllEqual(
debug_summary(constant_op.constant(x)), [-np.inf, 0.0, np.nan])
x = np.zeros([9701], dtype=np.float64)
x[9700] = np.nan
self.assertAllEqual(
debug_summary(constant_op.constant(x)), [0.0, 0.0, np.nan])
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpLargeTensorIDError(self):
modes = [
debug_event_pb2.TensorDebugMode.CURT_HEALTH,
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
debug_event_pb2.TensorDebugMode.SHAPE,
]
# Maximum allowed tensor_id
tensor_id = np.power(2, 53, dtype=np.int64)
for mode in modes:
self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
constant_op.constant(42.0),
tensor_debug_mode=mode,
tensor_id=tensor_id,
output_dtype=dtypes.float64))
# Incrementing by one should error
tensor_id += 1
for mode in modes:
with self.assertRaises(errors.InvalidArgumentError):
self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
constant_op.constant(42.0),
tensor_debug_mode=mode,
tensor_id=tensor_id,
output_dtype=dtypes.float64))
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpCurtHealthValuesSmall(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(debug_event_pb2.TensorDebugMode.CURT_HEALTH),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
tensor, tensor_id = debug_summary(constant_op.constant([]))
self.assertAllEqual(tensor, [tensor_id, 0.0])
tensor, tensor_id = debug_summary(constant_op.constant(42.0))
self.assertAllEqual(tensor, [tensor_id, 0.0])
tensor, tensor_id = debug_summary(constant_op.constant([3.0, 4.0]))
self.assertAllEqual(tensor, [tensor_id, 0.0])
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([3.0, -np.inf])))
self.assertAllEqual(tensor, [tensor_id, 1.0])
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([[0, 0], [np.nan, 0]])))
self.assertAllEqual(tensor, [tensor_id, 1.0])
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([[0, 0], [np.nan, np.inf]])))
self.assertAllEqual(tensor, [tensor_id, 1.0])
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([[0, np.inf], [np.nan, -np.inf]])))
self.assertAllEqual(tensor, [tensor_id, 1.0])
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpCurtHealthValuesLarge(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(debug_event_pb2.TensorDebugMode.CURT_HEALTH),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
x = np.zeros([100, 100], dtype=np.float16)
x[32, 47] = np.nan
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(tensor, [tensor_id, 1.0])
x = np.zeros([97, 97], dtype=np.float32)
x[50, 83] = -np.inf
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(tensor, [tensor_id, 1.0])
x[1, 41] = np.nan
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(tensor, [tensor_id, 1.0])
x = np.zeros([9701], dtype=np.float64)
x[9700] = np.nan
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(tensor, [tensor_id, 1.0])
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpCurtHealthConsistency(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(debug_event_pb2.TensorDebugMode.CURT_HEALTH),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
x = np.zeros([100, 100], dtype=np.float16)
x[43, 99] = np.nan
c = constant_op.constant(x)
tensor_1, tensor_id_1 = debug_summary(c)
tensor_2, tensor_id_2 = debug_summary(c)
self.assertAllEqual(tensor_1, tensor_2)
self.assertEqual(tensor_id_1, tensor_id_2)
x = np.zeros([100, 100, 50], dtype=np.float64)
x[0, 0, 1] = np.inf
c = constant_op.constant(x)
tensor_1, tensor_id_1 = debug_summary(c)
tensor_2, tensor_id_2 = debug_summary(c)
self.assertAllEqual(tensor_1, tensor_2)
self.assertEqual(tensor_id_1, tensor_id_2)
c = constant_op.constant(np.ones((100, 200), np.double))
tensor_1, tensor_id_1 = debug_summary(c)
tensor_2, tensor_id_2 = debug_summary(c)
self.assertAllEqual(tensor_1, tensor_2)
self.assertEqual(tensor_id_1, tensor_id_2)
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpDeterminism(self):
x = np.zeros([100, 100, 50], dtype=np.float64)
x = constant_op.constant(x)
modes = (
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
debug_event_pb2.TensorDebugMode.FULL_HEALTH,
)
for mode in modes:
debug_mode = debug_event_pb2.TensorDebugMode.Name(mode)
with test_util.deterministic_ops():
if test_util.config.list_physical_devices("GPU"):
with self.assertRaisesRegex(
errors_impl.UnimplementedError, "Determinism is not yet "
"supported for DebugNumericSummaryV2 when tensor_debug_mode is "
+ debug_mode + "."):
self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=mode,
tensor_id=x._id,
output_dtype=dtypes.float64))
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpConciseHealthSmall(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
tensor, tensor_id = debug_summary(constant_op.constant([]))
self.assertAllEqual(tensor, [tensor_id, 0.0, 0.0, 0.0, 0.0])
tensor, tensor_id = debug_summary(constant_op.constant(42.0))
self.assertAllEqual(tensor, [tensor_id, 1.0, 0.0, 0.0, 0.0])
tensor, tensor_id = debug_summary(constant_op.constant([3.0, 4.0]))
self.assertAllEqual(tensor, [tensor_id, 2.0, 0.0, 0.0, 0.0])
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([3.0, -np.inf])))
self.assertAllEqual(tensor, [tensor_id, 2.0, 1.0, 0.0, 0.0])
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([[0, 0], [np.nan, 0]])))
self.assertAllEqual(tensor, [tensor_id, 4.0, 0.0, 0.0, 1.0])
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([[0, 0], [np.nan, np.inf]])))
self.assertAllEqual(tensor, [tensor_id, 4.0, 0.0, 1.0, 1.0])
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([[0, np.inf], [np.nan, -np.inf]])))
self.assertAllEqual(tensor, [tensor_id, 4.0, 1.0, 1.0, 1.0])
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpConciseHealthLarge(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
x = np.zeros([100, 100], dtype=np.float16)
x[32, :] = np.nan
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(tensor, [tensor_id, 10000.0, 0.0, 0.0, 100.0])
x = np.zeros([97, 97], dtype=np.float32)
x[50, 83:85] = -np.inf
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(tensor, [tensor_id, 97 * 97, 2.0, 0.0, 0.0])
x[1:9, 41] = np.nan
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(tensor, [tensor_id, 97 * 97, 2.0, 0.0, 8.0])
x = np.zeros([9701], dtype=np.float64)
x[9700] = np.nan
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(tensor, [tensor_id, 9701, 0.0, 0.0, 1.0])
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpConciseHealthConsistency(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
# Assert the same op is returns a consistent value
x = np.zeros([100, 100], dtype=np.float16)
x[3, 4] = -np.inf
c = constant_op.constant(x)
tensor_1, tensor_id_1 = debug_summary(c)
tensor_2, tensor_id_2 = debug_summary(c)
self.assertAllEqual(tensor_1, tensor_2)
self.assertEqual(tensor_id_1, tensor_id_2)
c = constant_op.constant(np.ones((100, 200), np.double))
tensor_1, tensor_id_1 = debug_summary(c)
tensor_2, tensor_id_2 = debug_summary(c)
self.assertAllEqual(tensor_1, tensor_2)
self.assertEqual(tensor_id_1, tensor_id_2)
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpShapeEmpty(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(debug_event_pb2.TensorDebugMode.SHAPE),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
tensor, tensor_id = debug_summary(constant_op.constant(0.0))
self.assertAllEqual(
tensor, [tensor_id, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpShapeSmall(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(debug_event_pb2.TensorDebugMode.SHAPE),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
x = np.zeros([3, 4], dtype=np.float32)
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(
tensor, [tensor_id, 1.0, 2.0, 12.0, 3.0, 4.0, 0.0, 0.0, 0.0, 0.0])
x = np.ones([1, 2, 3, 4, 5, 6], dtype=np.float16)
x[0, 1, 2, 2, 2, 2] = np.nan
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(
tensor,
[tensor_id, 19, 6.0, 2 * 3 * 4 * 5 * 6, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
x = np.zeros([2], dtype=np.float32)
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(
tensor, [tensor_id, 1.0, 1.0, 2.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0])
tensor, tensor_id = debug_summary(constant_op.constant([]))
self.assertAllEqual(
tensor, [tensor_id, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpShapeLarge(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(debug_event_pb2.TensorDebugMode.SHAPE),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
x = np.ones([1, 2, 3, 4, 5, 6, 7], dtype=np.double)
tensor, tensor_id = debug_summary(constant_op.constant(x))
self.assertAllEqual(tensor, [
tensor_id, 2.0, 7.0, 2 * 3 * 4 * 5 * 6 * 7, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0
])
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpFullHealthSmall(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(debug_event_pb2.TensorDebugMode.FULL_HEALTH),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
tensor, tensor_id = debug_summary(constant_op.constant([]))
expected = [tensor_id, -1, 1, 1, 0, 0, 0, 0, 0, 0, 0]
self.assertAllEqual(tensor, expected)
tensor, tensor_id = debug_summary(constant_op.constant(42.0))
expected = [tensor_id, -1, 1, 0, 1, 0, 0, 0, 0, 0, 1]
self.assertAllEqual(tensor, expected)
tensor, tensor_id = debug_summary(constant_op.constant([3.0, 4.0]))
expected = [tensor_id, -1, 1, 1, 2, 0, 0, 0, 0, 0, 2]
self.assertAllEqual(tensor, expected)
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([3, -np.inf], dtype=np.float32)))
expected = [tensor_id, -1, 1, 1, 2, 1, 0, 0, 0, 0, 1]
self.assertAllEqual(tensor, expected)
tensor, tensor_id = debug_summary(
constant_op.constant(np.array([[0, 0], [np.nan, 0]], dtype=np.float64)))
expected = [tensor_id, -1, 2, 2, 4, 0, 0, 1, 0, 3, 0]
self.assertAllEqual(tensor, expected)
tensor, tensor_id = debug_summary(
constant_op.constant(
np.array([[0, 0], [np.nan, np.inf]], dtype=np.float16)))
expected = [tensor_id, -1, 19, 2, 4, 0, 1, 1, 0, 2, 0]
self.assertAllEqual(tensor, expected)
tensor, tensor_id = debug_summary(
constant_op.constant(
np.array([[0, np.inf], [np.nan, -np.inf]], dtype=np.float32)))
expected = [tensor_id, -1, 1, 2, 4, 1, 1, 1, 0, 1, 0]
self.assertAllEqual(tensor, expected)
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpFullHealthLarge(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(debug_event_pb2.TensorDebugMode.FULL_HEALTH),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
def tensor_counts(arr):
counts = [len(np.shape(arr)), np.size(arr), 0, 0, 0, 0, 0, 0]
for n in np.ravel(arr):
if np.isneginf(n):
counts[2] += 1
elif np.isposinf(n):
counts[3] += 1
elif np.isnan(n):
counts[4] += 1
elif n < 0.:
counts[5] += 1
elif n == 0.:
counts[6] += 1
else:
counts[7] += 1
return counts
x = np.zeros([50, 50], dtype=np.float16)
x[32, 47] = np.nan
x[0:4, 3] = np.inf
x[40:50, 40:50] = 10
x[3, 20] = -10
tensor, tensor_id = debug_summary(constant_op.constant(x))
expected = [tensor_id, -1, 19] + tensor_counts(x)
self.assertAllEqual(tensor, expected)
x = np.ones([25, 25, 50], dtype=np.float32) * np.inf
x[:, :, 1] = np.nan
x[:, :, 2] = -np.inf
x[:, :, 3] = -1
x[:, :, 4] = 0
x[:, :, 5] = 1
tensor, tensor_id = debug_summary(constant_op.constant(x))
expected = [tensor_id, -1, 1] + tensor_counts(x)
self.assertAllEqual(tensor, expected)
x[0, 0, 0] = np.nan
tensor, tensor_id = debug_summary(constant_op.constant(x))
expected = [
tensor_id,
-1,
1,
] + tensor_counts(x)
self.assertAllEqual(tensor, expected)
x = np.zeros([9701], dtype=np.float64)
x[9700] = np.nan
tensor, tensor_id = debug_summary(constant_op.constant(x))
expected = [tensor_id, -1, 2] + tensor_counts(x)
self.assertAllEqual(tensor, expected)
@test_util.run_in_graph_and_eager_modes
def testDebugNumericSummaryV2OpFullHealthConsistency(self):
def debug_summary(x):
return self.evaluate(
gen_debug_ops.debug_numeric_summary_v2(
x,
tensor_debug_mode=(debug_event_pb2.TensorDebugMode.FULL_HEALTH),
tensor_id=x._id,
output_dtype=dtypes.float64)), x._id
# Assert the same op is returns a consistent value
x = np.zeros([100, 100], dtype=np.float16)
x[32, 47] = np.nan
x[0:4, 3] = np.inf
x[90:100, 90:100] = 10
x[3, 20] = -10
c = constant_op.constant(x)
tensor_1, tensor_id_1 = debug_summary(c)
tensor_2, tensor_id_2 = debug_summary(c)
self.assertAllEqual(tensor_1, tensor_2)
self.assertEqual(tensor_id_1, tensor_id_2)
x = np.ones((100, 200, 3, 10), np.double)
x[1, 30, 2] = 10
x[5, :, 0, 1] = np.nan
x[90:100, 150, :, :] = np.inf
c = constant_op.constant(x)
tensor_1, tensor_id_1 = debug_summary(c)
tensor_2, tensor_id_2 = debug_summary(c)
self.assertAllEqual(tensor_1, tensor_2)
self.assertEqual(tensor_id_1, tensor_id_2)
def testCheckNumericsV2OpNegativeAndPositiveInf(self):
"""Test that CheckNumericsV2 op distinguishes negative and positive infs."""
with self.session(graph=ops.Graph()):
t1 = constant_op.constant([-1.0, 1.0])
t2 = constant_op.constant([0.0, 0.0])
with self.assertRaisesRegex(
errors.InvalidArgumentError,
r"pass through test.*had -Inf and \+Inf values"):
self.evaluate(
array_ops.check_numerics_v2(t1 / t2, message="pass through test"))
def testCheckNumericsV2OpNegativeAndPositiveInfAndNaN(self):
"""CheckNumericsV2 op distinguishes - & + infs when nan is present."""
with self.session(graph=ops.Graph()):
t1 = constant_op.constant([-1.0, 1.0, 0.0])
t2 = constant_op.constant([0.0, 0.0, 0.0])
with self.assertRaisesRegex(
errors.InvalidArgumentError,
r"pass through test.*had -Inf, \+Inf, and NaN values"):
self.evaluate(
array_ops.check_numerics_v2(t1 / t2, message="pass through test"))
def testCheckNumericsV2PositiveInfAndNaN(self):
"""Test that CheckNumericsV2 op shows sign of inf when nan is present."""
with self.session(graph=ops.Graph()):
t1 = constant_op.constant([0.0, 1.0])
t2 = constant_op.constant([0.0, 0.0])
with self.assertRaisesRegex(
errors.InvalidArgumentError,
r"pass through test.*had \+Inf and NaN values"):
self.evaluate(
array_ops.check_numerics_v2(t1 / t2, message="pass through test"))
if __name__ == "__main__":
ops.enable_eager_execution()
googletest.main()
tensorflow/python/debug/lib/dumping_callback.py
+886
View File
@@ -0,0 +1,886 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Dumping op callbacks: Enables dump-based features in tfdbg v2."""
import atexit
import os
import re
import socket
import threading
import uuid
from tensorflow.core.framework import graph_debug_info_pb2
from tensorflow.core.framework import tensor_pb2
from tensorflow.core.protobuf import debug_event_pb2
from tensorflow.python.debug.lib import debug_events_writer
from tensorflow.python.debug.lib import op_callbacks_common
from tensorflow.python.debug.lib import source_utils
from tensorflow.python.eager import function as function_lib
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import op_callbacks
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_debug_ops
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import compat
from tensorflow.python.util import object_identity
from tensorflow.python.util import tf_stack
from tensorflow.python.util.tf_export import tf_export
_state = threading.local()
DEFAULT_TENSOR_DEBUG_MODE = "NO_TENSOR"
# pylint:disable=protected-access
_FUNCTION_PREFIXES = (
compat.as_bytes(function_lib._FORWARD_PREFIX),
compat.as_bytes(function_lib._BACKWARD_PREFIX),
compat.as_bytes(function_lib._INFERENCE_PREFIX))
# pylint:enable=protected-access
def is_op_type_function(op_type):
return compat.as_bytes(op_type).startswith(_FUNCTION_PREFIXES)
@ops.RegisterGradient("DebugIdentityV2")
def _debug_identity_v2_grad(op, dy):
"""Gradient function for the DebugIdentityV2 op."""
del op # Unused
return dy
def _get_tfdbg_run_id():
return str(uuid.uuid4())[:8]
def _get_id():
"""Get a short unique ID."""
return str(uuid.uuid4())
def _concrete_tensor_to_proto(tensor):
return tensor_util.make_tensor_proto(tensor.numpy())
class _DumpingCallback(object):
"""An object holding the states surrounding the dumping callback."""
def __init__(self,
dump_root,
tensor_debug_mode,
circular_buffer_size,
op_regex,
tensor_dtypes):
self._dump_root = dump_root
self._tfdbg_run_id = _get_tfdbg_run_id()
self._tensor_debug_mode = tensor_debug_mode
self._circular_buffer_size = circular_buffer_size
self._op_regex = op_regex
self._tensor_dtypes = tensor_dtypes
self._hostname = socket.gethostname()
# A list of source-file paths.
self._source_file_paths = []
# A map from stack frame (FileLineCol) to unique ID.
self._stack_frame_to_id = dict()
# Mapping op context to unique ID.
self._context_to_id = dict()
self._function_to_graph_id = dict()
self._op_type_to_context_id = dict()
# Keeps track of counter for symbolic tensors output by in-graph ops.
# It is used to make unique names for debugger-generated tensors.
self._symbolic_tensor_counter = 0
# A map from the names of debugger-generated Identity and DebugIdentityV2
# tensors to the names of the original insrumented graph tensors. This is
# applicable to v1 graph mode only.
self._tensor_aliases = dict()
self._source_file_paths_lock = threading.Lock()
self._stack_frame_to_id_lock = threading.Lock()
self._context_lock = threading.Lock()
self._symbolic_tensor_counter_lock = threading.Lock()
# A dict mapping Placeholder tensors to their instrumenting debug tensors.
# Used only under V1 graph mode, where we can't rely on auto control
# dependency to execute the debug tensors and hence need to attach the debug
# tensors as control dependencies of the ops that consume the Placeholder.
self._placeholder_to_debug_tensor = (
object_identity.ObjectIdentityDictionary())
self._writer = None
def function_callback(self, function):
"""A callback to be called on creation of ConcreteFunctions."""
graph_id = self._get_context_id(function.graph)
with self._context_lock:
# NOTE(cais): We currently store the function (ConcreteFunction)
# as keys of this dict, because weakrefs to them sometimes become
# unreferenceable by the time the op callback is called. This approach
# may cause memory leaks due to the holding of the functions. If that's
# the case, calling `tf.debugging.disable_dump_debug_info()` should
# cause GC of this object and this dict.
self._function_to_graph_id[function] = graph_id
@property
def dump_root(self):
return self._dump_root
@dump_root.setter
def dump_root(self, dump_root):
if self._dump_root != dump_root:
self._dump_root = dump_root
self._writer = None
@property
def tfdbg_run_id(self):
return self._tfdbg_run_id
@property
def tensor_debug_mode(self):
return self._tensor_debug_mode
@property
def circular_buffer_size(self):
return self._circular_buffer_size
def get_writer(self):
"""Get the debug events writer for the currently configured dump root."""
if not self._writer:
self._writer = debug_events_writer.DebugEventsWriter(
self._dump_root,
self._tfdbg_run_id,
circular_buffer_size=self._circular_buffer_size)
return self._writer
def _get_context_id(self, context):
"""Get a unique ID for an op-construction context (e.g., a graph).
If the graph has been encountered before, reuse the same unique ID.
When encountering a new context (graph), this methods writes a DebugEvent
proto with the debugged_graph field to the proper DebugEvent file.
Args:
context: A context to get the unique ID for. Must be hashable. E.g., a
Graph object.
Returns:
A unique ID for the context.
"""
# Use the double-checked lock pattern to optimize the common case.
if context in self._context_to_id: # 1st check, without lock.
return self._context_to_id[context]
graph_is_new = False
with self._context_lock:
if context not in self._context_to_id: # 2nd check, with lock.
graph_is_new = True
context_id = _get_id()
self._context_to_id[context] = context_id
if graph_is_new:
self.get_writer().WriteDebuggedGraph(debug_event_pb2.DebuggedGraph(
graph_id=context_id,
graph_name=getattr(context, "name", None),
outer_context_id=self._get_outer_context_id(context)))
return self._context_to_id[context]
def _get_outer_context_id(self, graph):
"""Get the ID of the immediate outer context of the input graph.
Args:
graph: The graph (context) in question.
Returns:
If an outer context exists, the immediate outer context name as a string.
If such as outer context does not exist (i.e., `graph` is itself
outermost), `None`.
"""
if hasattr(graph, "outer_graph") and graph.outer_graph:
return self._get_context_id(graph.outer_graph)
else:
return None
def _write_source_file_content(self, file_path):
"""Send the content of a source file via debug-events writer.
Args:
file_path: Path to the source file.
Returns:
An int index for the file.
"""
if file_path in self._source_file_paths:
return self._source_file_paths.index(file_path)
with self._source_file_paths_lock:
if file_path not in self._source_file_paths:
lines = None
if source_utils.is_extension_uncompiled_python_source(file_path):
try:
lines, _ = source_utils.load_source(file_path)
except IOError as e:
logging.warn(
"Failed to read source code from path: %s. Reason: %s",
file_path, e)
writer = self.get_writer()
writer.WriteSourceFile(debug_event_pb2.SourceFile(
file_path=file_path, host_name=self._hostname, lines=lines))
self._source_file_paths.append(file_path)
return self._source_file_paths.index(file_path)
def _process_stack_frames(self):
"""Process stack frames.
Send the content of source-files, on a best-effort basis.
Returns:
A list of stack frame IDs.
"""
stack_frames = tf_stack.extract_stack()
stack_frame_ids = []
writer = None
for file_path, lineno, func, _ in stack_frames:
abs_path = os.path.abspath(file_path)
if (abs_path, lineno, func) in self._stack_frame_to_id:
stack_frame_ids.append(
self._stack_frame_to_id[(abs_path, lineno, func)])
continue
with self._stack_frame_to_id_lock:
if (abs_path, lineno, func) not in self._stack_frame_to_id:
stack_frame_id = _get_id()
self._stack_frame_to_id[(abs_path, lineno, func)] = stack_frame_id
file_index = self._write_source_file_content(abs_path)
file_line_col = graph_debug_info_pb2.GraphDebugInfo.FileLineCol(
file_index=file_index, line=lineno, func=func)
stack_frame_with_id = debug_event_pb2.StackFrameWithId(
id=stack_frame_id, file_line_col=file_line_col)
writer = self.get_writer()
writer.WriteStackFrameWithId(stack_frame_with_id)
stack_frame_ids.append(
self._stack_frame_to_id[(abs_path, lineno, func)])
code_location = debug_event_pb2.CodeLocation(
host_name=self._hostname, stack_frame_ids=stack_frame_ids)
return code_location
def _process_v1_graph_mode_tensor(self,
op_type,
tensor,
debug_tensor,
tensor_debug_mode):
"""For V1 graph mode, determine what tensor to output from callback.
Args:
op_type: Type of the op that outputs the original symbolic tensor.
tensor: The original output symbolic tensor.
debug_tensor: The debugger-instrumented tensor.
tensor_debug_mode: Debug mode used, a tfdbg TensorDebugMode enum.
Returns:
A symbolic tensor to be returned by the dumping op_callback.
"""
# Placeholders need special treatment under V1 graph mode. The
# callback can't simply override the Placeholder tensor to a debug tensor,
# as that would cause the Placeholder op to lack a value.
if op_type in ("Placeholder", "PlaceholderWithDefault"):
self._placeholder_to_debug_tensor[tensor] = debug_tensor
return tensor
else:
# TODO(cais): Evaluate performance optimization options. For the
# `NO_TENSOR` debug mode, an alternative is to add `debug_tensor` as a
# control dependency of `tensor.op` without an additional identity op.
if (tensor_debug_mode == debug_event_pb2.TensorDebugMode.FULL_TENSOR and
op_type != "Const"):
# NOTE(b/153716279): Under v1 graph mode, overriding the output tensor
# of Const ops can lead to downstream errors related to shapes. We opt
# to use an identity op to avoid this issue at the cost of slightly
# larger graph size.
self._tensor_aliases[debug_tensor.name] = tensor.name
return debug_tensor
else:
with self._symbolic_tensor_counter_lock:
identity_name = "tfdbg_identity_%d" % self._symbolic_tensor_counter
identity = array_ops.identity(tensor, name=identity_name)
identity.op._add_control_input( # pylint: disable=protected-access
debug_tensor.op)
self._tensor_aliases[identity.name] = tensor.name
return identity
def _instrument_symbolic_tensors(self,
tensors,
op_type,
op_name,
tfdbg_context_id,
tensor_ids):
"""Add debugging instrumentation for symbolic (i.e., non-eager) tensors.
The detailed fashion in which the tensors are instrumented is determined
by the tensor_debug_mode configured for the currently enabled dumping
callback.
Args:
tensors: A tuple of Tensors to instrument. It is assumed that their
ordering corresponds to the ordering of output tensors of an original
op. Output slot indices (0-based) will be generated based on the
ordering.
op_type: Type name of the op that emits the Tensors (e.g., "MatMul").
op_name: Name of the op that emits the Tensors (e.g., "dense_1/MatMul").
tfdbg_context_id: A unique ID for the context that the op belongs to
(e.g., a graph).
tensor_ids: A list of unique ID numbers for the tensors, for tfdbg's
internal use.
Returns:
Non-eager Tensors that override the `tensors` as the output of the op
that originally generated `tensors`. In some cases (e.g., non-V1 graph
mode), this may be `None`, as the instrumentation can simply rely on
automatic control dependencies (see `auto_control_deps.py`) instead of
tensor overriding.
"""
tensor_debug_mode = self._tensor_debug_mode
debug_urls = ["file://%s" % self._dump_root]
is_v1_graph_mode = not ops.executing_eagerly_outside_functions()
instrumented_tensors = [] if is_v1_graph_mode else None
for output_slot, tensor in enumerate(tensors):
with self._symbolic_tensor_counter_lock:
debug_identity_name = ("DebugIdentityV2_%d" %
self._symbolic_tensor_counter)
debug_identity_op_kwargs = {
"tfdbg_context_id": tfdbg_context_id,
"op_name": op_name,
"output_slot": output_slot,
"tensor_debug_mode": self._tensor_debug_mode,
"debug_urls": debug_urls,
"name": debug_identity_name,
"circular_buffer_size": self._circular_buffer_size,
"tfdbg_run_id": self._tfdbg_run_id,
}
if tensor_debug_mode == debug_event_pb2.TensorDebugMode.NO_TENSOR:
if (not self._should_dump_tensor(op_type, tensor.dtype) or
not tensor.dtype.is_numpy_compatible):
if is_v1_graph_mode:
instrumented_tensors.append(tensor)
continue
if is_v1_graph_mode and not tensor.dtype.is_numpy_compatible:
# Avoid instrumenting Placeholder under is_v1_graph_mode. Doing that
# would cause runtime complaint about Placeholders not being fed.
instrumented_tensors.append(tensor)
continue
# Except in V1 graph mode + control flow, debug_identity_v2 triggers
# auto control dependency because it's a stateful op.
debug_tensor = gen_debug_ops.debug_identity_v2(
# Use an empty (shape=[0]) float32 tensor for the NO_TENSOR mode
# as a low-overhead placeholder, since no actual tensor value is
# traced.
constant_op.constant([], dtype=dtypes.float32),
**debug_identity_op_kwargs)
if is_v1_graph_mode:
instrumented_tensors.append(self._process_v1_graph_mode_tensor(
op_type, tensor, debug_tensor, tensor_debug_mode))
elif tensor_debug_mode in (debug_event_pb2.TensorDebugMode.CURT_HEALTH,
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
debug_event_pb2.TensorDebugMode.FULL_HEALTH,
debug_event_pb2.TensorDebugMode.SHAPE):
dtype = tensor.dtype
dtype_is_dumpable = (
tensor_debug_mode in (
debug_event_pb2.TensorDebugMode.CURT_HEALTH,
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
debug_event_pb2.TensorDebugMode.FULL_HEALTH) and
dtype.is_floating or
tensor_debug_mode == debug_event_pb2.TensorDebugMode.SHAPE and
(dtype.is_floating or dtype.is_integer or dtype.is_bool))
if (not self._should_dump_tensor(op_type, tensor.dtype) or
not dtype_is_dumpable):
if is_v1_graph_mode:
instrumented_tensors.append(tensor)
continue
debug_tensor = gen_debug_ops.debug_identity_v2(
gen_debug_ops.debug_numeric_summary_v2(
tensor,
tensor_id=tensor_ids[output_slot],
tensor_debug_mode=self._tensor_debug_mode,
output_dtype=dtypes.float64), **debug_identity_op_kwargs)
if is_v1_graph_mode:
instrumented_tensors.append(self._process_v1_graph_mode_tensor(
op_type, tensor, debug_tensor, tensor_debug_mode))
elif tensor_debug_mode == debug_event_pb2.TensorDebugMode.FULL_TENSOR:
if (not self._should_dump_tensor(op_type, tensor.dtype) or
not tensor.dtype.is_numpy_compatible):
# Instrumenting DT_VARIANT and DT_RESOURCE type tensors under
# V1 graph mode is known to have issues. TODO(cais): Investigate.
if is_v1_graph_mode:
instrumented_tensors.append(tensor)
continue
debug_tensor = gen_debug_ops.debug_identity_v2(
tensor, **debug_identity_op_kwargs)
if is_v1_graph_mode:
instrumented_tensors.append(self._process_v1_graph_mode_tensor(
op_type, tensor, debug_tensor, tensor_debug_mode))
else:
raise NotImplementedError(
"Symbolic tensor instrumentation is not implemented for debug mode "
"%s" % self._tensor_debug_mode)
return instrumented_tensors
def _dump_eager_tensors(self,
tensors,
op_type,
input_tensor_ids,
output_tensor_device_ids,
graph_id=None):
"""Dump the value of eager tensors.
The destination of the dumping is determined by the dump_root of the
currently enabled dumping callback. The tensors may be transformed prior to
dumping (e.g., reduced as summary statistics such as minimum, maximum and
arithmetic mean). The details of this transformation (if any) depends on
the tensor_debug_mode of the currently enabled dumping callback.
Args:
tensors: The EagerTensors whose values are to be dumped, with or without
value transform.
op_type: Type of the op that generates the tensors, as a string.
input_tensor_ids: IDs of the input EagerTensors to the op.
output_tensor_device_ids: Debugged-generated IDs for the devices on which
the output tensors are allocated, as a `list` of `int`s. Must match
`tensors` in length.
graph_id: ID of the executed graph, applicable only to eager execution of
a FuncGraph.
Returns:
A tfdbg Execution protocol buffer.
"""
tensor_debug_mode = self._tensor_debug_mode
output_tensor_ids = [
t._id for t in tensors] # pylint:disable=protected-access
assert len(tensors) == len(output_tensor_device_ids)
if tensor_debug_mode == debug_event_pb2.TensorDebugMode.NO_TENSOR:
return debug_event_pb2.Execution(
op_type=op_type,
graph_id=graph_id,
num_outputs=len(tensors),
input_tensor_ids=input_tensor_ids,
output_tensor_ids=output_tensor_ids,
output_tensor_device_ids=output_tensor_device_ids,
tensor_debug_mode=tensor_debug_mode,
code_location=self._process_stack_frames())
elif tensor_debug_mode in (debug_event_pb2.TensorDebugMode.CURT_HEALTH,
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
debug_event_pb2.TensorDebugMode.FULL_HEALTH,
debug_event_pb2.TensorDebugMode.SHAPE,
debug_event_pb2.TensorDebugMode.FULL_TENSOR):
execution_proto = debug_event_pb2.Execution(
op_type=op_type,
num_outputs=len(tensors),
graph_id=graph_id,
input_tensor_ids=input_tensor_ids,
output_tensor_ids=output_tensor_ids,
output_tensor_device_ids=output_tensor_device_ids,
tensor_debug_mode=tensor_debug_mode,
code_location=self._process_stack_frames())
for tensor in tensors:
if (self._should_dump_tensor(op_type, tensor.dtype) and
tensor.dtype.is_numpy_compatible):
if tensor_debug_mode in (
debug_event_pb2.TensorDebugMode.CURT_HEALTH,
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
debug_event_pb2.TensorDebugMode.FULL_HEALTH):
if tensor.dtype.is_floating:
tensor_proto = _concrete_tensor_to_proto(
gen_debug_ops.debug_numeric_summary_v2(
tensor,
tensor_debug_mode=tensor_debug_mode,
output_dtype=dtypes.float64))
else:
# A placeholder for non-floating-type output tensors.
tensor_proto = tensor_pb2.TensorProto()
elif tensor_debug_mode == debug_event_pb2.TensorDebugMode.SHAPE:
if (tensor.dtype.is_floating or tensor.dtype.is_integer or
tensor.dtype.is_bool):
tensor_proto = _concrete_tensor_to_proto(
gen_debug_ops.debug_numeric_summary_v2(
tensor,
tensor_debug_mode=tensor_debug_mode,
output_dtype=dtypes.float64))
else:
# A placeholder for non-floating-type output tensors.
tensor_proto = tensor_pb2.TensorProto()
elif tensor_debug_mode == debug_event_pb2.TensorDebugMode.FULL_TENSOR:
tensor_proto = _concrete_tensor_to_proto(tensor)
if tensor_proto:
execution_proto.tensor_protos.append(tensor_proto)
return execution_proto
else:
raise NotImplementedError(
"Tensor instrumentation is not implemented for debug mode %s yet " %
self._tensor_debug_mode)
def callback(self,
op_type,
inputs,
attrs,
outputs,
op_name=None,
graph=None):
"""Op callback for tracing (dumping) a TF program's execution."""
del attrs # Unused
writer = self.get_writer()
if graph:
is_v1_graph_mode = not ops.executing_eagerly_outside_functions()
context_id = self._get_context_id(graph) # Innermost context ID.
output_tensor_ids = self._get_symbolic_tensor_ids(len(outputs))
if op_type in ("Const", "Placeholder", "PlaceholderWithDefault"):
# In some cases, the op name of a Const or Placeholder op in a graph
# can be duplicate (e.g., `None` or "resource").
# When this happens, we use the output tensor name to infer
# the non-duplicated tensor name.
op_name = outputs[0].name.split(":")[0]
if is_v1_graph_mode:
for input_tensor in inputs:
if input_tensor in self._placeholder_to_debug_tensor and outputs:
outputs[0].op._add_control_input( # pylint: disable=protected-access
self._placeholder_to_debug_tensor[input_tensor].op)
graph_op_creation = debug_event_pb2.GraphOpCreation(
op_type=op_type,
op_name=op_name,
graph_name=graph.name if hasattr(graph, "name") else None,
graph_id=context_id,
input_names=[
self._lookup_tensor_name(input_tensor) for input_tensor in inputs
],
num_outputs=len(outputs),
output_tensor_ids=output_tensor_ids,
code_location=self._process_stack_frames())
writer.WriteGraphOpCreation(graph_op_creation)
if outputs and compat.as_bytes(
op_type) not in op_callbacks_common.OP_CALLBACK_SKIP_OPS:
return self._instrument_symbolic_tensors(
outputs, op_type, op_name, context_id, output_tensor_ids)
else:
op_type_bytes = compat.as_bytes(op_type)
if op_type_bytes == b"DebugNumericSummaryV2":
# TODO(b/140334369): Remove this special casing logic once op_callback.
# automatically prevents infinite recursion in eager mode.
return None
if op_type_bytes in op_callbacks_common.OP_CALLBACK_SKIP_OPS:
return None
context_id = self._func_graph_id_from_func_name(op_type)
input_ids = [t._id for t in inputs] # pylint:disable=protected-access
output_tensor_device_ids = [writer.RegisterDeviceAndGetId(output.device)
for output in outputs] if outputs else []
writer.WriteExecution(self._dump_eager_tensors(
outputs, op_type, input_ids, output_tensor_device_ids,
graph_id=context_id))
def _lookup_tensor_name(self, tensor):
"""Look up the name of a graph tensor.
This method maps the name of a debugger-generated Identity or
DebugIdentityV2 tensor to the name of the original instrumented tensor,
if `tensor` is such a debugger-created tensor.
Otherwise, it returns the name of `tensor` as is.
Args:
tensor: The graph tensor to look up the name for.
Returns:
Name of the original instrumented tensor as known to the debugger.
"""
return self._tensor_aliases.get(tensor.name, tensor.name)
def _func_graph_id_from_func_name(self, op_type):
"""Attempt to get the ID of a FuncGraph based on an op type name.
Also caches the ID for faster access later.
Args:
op_type: Op type string, which may be the name of a function.
Returns:
If the op_type name does not fit the pattern of a function name (e.g.,
one that starts with "__inference_"), `None` is returned immediately.
Else, if the FuncGraph is found, ID of the underlying FuncGraph is
returned as a string.
Else, `None` is returned.
"""
op_type = compat.as_bytes(op_type)
if is_op_type_function(op_type):
# op_type for eagerly-executed FuncGraphs have the prefixed and suffixed
# form such as "__inference_my_function_13579", wherein the middle part
# "my_function" is the name of the Python function from which the
# FuncGraph is compiled. Due to the suffix, the op_type is unique for
# - duplicate Python function names
# - multiple compilation of the same Python function
if op_type in self._op_type_to_context_id:
return self._op_type_to_context_id[op_type]
with self._context_lock:
for function in self._function_to_graph_id:
if function.name == op_type:
graph_id = self._function_to_graph_id[function]
self._op_type_to_context_id[op_type] = graph_id
return graph_id
return None
else:
return None
def _get_symbolic_tensor_ids(self, num_tensors):
tensor_ids = []
if num_tensors:
with self._symbolic_tensor_counter_lock:
for _ in range(num_tensors):
self._symbolic_tensor_counter += 1
tensor_ids.append(self._symbolic_tensor_counter)
return tensor_ids
def _should_dump_tensor(self, op_type, dtype):
"""Determine if the given tensor's value will be dumped.
The determination is made given the configurations such as `op_regex`,
`tensor_dtypes`.
Args:
op_type: Name of the op's type, as a string (e.g., "MatMul").
dtype: The dtype of the tensor, as a `dtypes.DType` object.
Returns:
A bool indicating whether the tensor's value will be dumped.
"""
should_dump = True
if self._op_regex:
should_dump = (should_dump and
re.match(self._op_regex, op_type))
if self._tensor_dtypes:
if isinstance(self._tensor_dtypes, (list, tuple)):
should_dump = (should_dump and
any(dtype == dtype_item for dtype_item
in self._tensor_dtypes))
else: # A callable that takes a DType argument and return a boolean.
should_dump = should_dump and self._tensor_dtypes(dtype)
return should_dump
@tf_export("debugging.experimental.enable_dump_debug_info")
def enable_dump_debug_info(dump_root,
tensor_debug_mode=DEFAULT_TENSOR_DEBUG_MODE,
circular_buffer_size=1000,
op_regex=None,
tensor_dtypes=None):
"""Enable dumping debugging information from a TensorFlow program.
The debugging information is dumped to a directory on the file system
specified as `dump_root`.
The dumped debugging information can be ingested by debugger UIs.
The files in the dump directory contain the following information:
- TensorFlow Function construction (e.g., compilation of Python functions
decorated with @tf.function), the op types, names (if available), context,
the input and output tensors, and the associated stack traces.
- Execution of TensorFlow operations (ops) and Functions and their stack
traces, op types, names (if available) and contexts. In addition,
depending on the value of the `tensor_debug_mode` argument (see Args
section below), the value(s) of the output tensors or more concise
summaries of the tensor values will be dumped.
- A snapshot of Python source files involved in the execution of the
TensorFlow program.
Once enabled, the dumping can be disabled with the corresponding
`disable_dump_debug_info()` method under the same Python namespace.
Calling this method more than once with the same `dump_root` is idempotent.
Calling this method more than once with different `tensor_debug_mode`s
leads to a `ValueError`.
Calling this method more than once with different `circular_buffer_size`s
leads to a `ValueError`.
Calling this method with a different `dump_root` abolishes the
previously-enabled `dump_root`.
Usage example:
```py
tf.debugging.experimental.enable_dump_debug_info('/tmp/my-tfdbg-dumps')
# Code to build, train and run your TensorFlow model...
```
NOTE: If your code is running on TPUs, be sure to call
`tf.config.set_soft_device_placement(True)` before calling
`tf.debugging.experimental.enable_dump_debug_info()` as this API uses
automatic outside compilation on TPUs. For example:
```py
tf.config.set_soft_device_placement(True)
tf.debugging.experimental.enable_dump_debug_info(
logdir, tensor_debug_mode="FULL_HEALTH")
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')
strategy = tf.distribute.TPUStrategy(resolver)
with strategy.scope():
# ...
```
Args:
dump_root: The directory path where the dumping information will be written.
tensor_debug_mode: Debug mode for tensor values, as a string.
The currently supported options are:
- "NO_TENSOR": (Default) Only traces the output tensors of all executed
ops (including those executed eagerly at the Python level or as a part
of a TensorFlow graph) and functions, while not extracting any
information from the values of the tensors.
- "CURT_HEALTH": For each floating-dtype tensor (e.g., tensors of dtypes
such as `float32`, `float64` and `bfloat16`), extracts a binary bit
indicating whether it contains any -infinity, +infinity or NaN.
- "CONCISE_HEALTH": For each floating-dtype tensor, extract total
element count, and counts of -infinity, +infinity and NaN elements.
- "FULL_HEALTH": For each floating-dtype tensor, extracts the dtype,
rank (number of dimensions), total element count, and counts of
-infinity, +infinity and NaN elements.
- "SHAPE": For each tensor (regardless of dtype), extracts its dtype,
rank, total element count and shape.
circular_buffer_size: Size of the circular buffers for execution events.
These circular buffers are designed to reduce the overhead of debugging
dumping. They hold the most recent debug events concerning eager execution
of ops and `tf.function`s and traces of tensor values computed inside
`tf.function`s. They are written to the file system only when the proper
flushing method is called (see description of return values below).
Expected to be an integer. If <= 0, the circular-buffer behavior will be
disabled, i.e., the execution debug events will be written to the file
writers in the same way as non-execution events such as op creations and
source-file snapshots.
op_regex: Dump data from only the tensors from op types that matches to the
regular expression (through Python's `re.match()`).
"Op type" refers to the names of the TensorFlow operations (e.g.,
"MatMul", "LogSoftmax"), which may repeat in a TensorFlow
function. It does *not* refer to the names of nodes (e.g.,
"dense/MatMul", "dense_1/MatMul_1") which are unique within a function.
- Example 1: Dump tensor data from only MatMul and Relu ops
`op_regex="^(MatMul|Relu)$"`.
- Example 2: Dump tensors from all ops *except* Relu:
`op_regex="(?!^Relu$)"`.
This filter operates in a logical AND relation with `tensor_dtypes`.
tensor_dtypes: Dump data from only the tensors of which the specified
dtypes. This optional argument can be in any of the following format:
- a list or tuple of `DType` objects or strings that can be converted
to `DType` objects via `tf.as_dtype()`. Examples:
- `tensor_dtype=[tf.float32, tf.float64]`,
- `tensor_dtype=["float32", "float64"]`,
- `tensor_dtypes=(tf.int32, tf.bool)`,
- `tensor_dtypes=("int32", "bool")`
- a callable that takes a single `DType` argument and returns a Python
`boolean` indicating whether the dtype is to be included in the data
dumping. Examples:
- `tensor_dtype=lambda dtype: dtype.is_integer`.
This filter operates in a logical AND relation with `op_regex`.
Returns:
A DebugEventsWriter instance used by the dumping callback. The caller
may use its flushing methods, including `FlushNonExecutionFiles()` and
`FlushExecutionFiles()`.
"""
# TODO(cais): Revise the "UIs (currently under construction)" part of the doc
# string above.
# TODO(cais): Add Python code example to the doc string above.
global _state
tensor_debug_mode_keys = debug_event_pb2.TensorDebugMode.keys()
if tensor_debug_mode not in tensor_debug_mode_keys:
raise ValueError(
"Invalid value in tensor_debug_mode ('%s'). Valid options are: %s" %
(tensor_debug_mode, tensor_debug_mode_keys))
tensor_debug_mode = debug_event_pb2.TensorDebugMode.Value(tensor_debug_mode)
if tensor_debug_mode not in (debug_event_pb2.TensorDebugMode.NO_TENSOR,
debug_event_pb2.TensorDebugMode.CURT_HEALTH,
debug_event_pb2.TensorDebugMode.CONCISE_HEALTH,
debug_event_pb2.TensorDebugMode.FULL_HEALTH,
debug_event_pb2.TensorDebugMode.SHAPE,
debug_event_pb2.TensorDebugMode.FULL_TENSOR):
raise NotImplementedError(
"tfdbg dumping: support for tensor debug mode %s is not "
"implemented yet" %
debug_event_pb2.TensorDebugMode.Name(tensor_debug_mode))
# Validate the types of tensor_dtypes.
if tensor_dtypes is not None:
if (not isinstance(tensor_dtypes, (list, tuple)) and
not callable(tensor_dtypes)):
raise ValueError(
"If specified, tensor_dtypes is expected to be a list, a tuple, or "
"a callable that takes a DType argument and returns a boolean, "
"but received %s" % (tensor_dtypes,))
if isinstance(tensor_dtypes, (list, tuple)):
tensor_dtypes = [
dtypes.as_dtype(dtype_item) for dtype_item in tensor_dtypes]
if hasattr(_state, "dumping_callback"):
if _state.dumping_callback.circular_buffer_size != circular_buffer_size:
raise ValueError(
"There is already a dumping callback configured with a different "
"circular-buffer size (%d). Therefore the newly request "
"circular-buffer size (%d) will not be honored." %
(_state.dumping_callback.circular_buffer_size, circular_buffer_size))
if _state.dumping_callback.tensor_debug_mode != tensor_debug_mode:
raise ValueError(
"There is already a dumping callback configured for dump root "
"%s with a different "
"tensor-debug mode (%s). Therefore the newly request "
"tensor-debug mode (%s) size will not be honored." %
(_state.dumping_callback.dump_root,
tensor_debug_mode_keys[_state.dumping_callback.tensor_debug_mode],
tensor_debug_mode_keys[tensor_debug_mode]))
else:
_state.dumping_callback = _DumpingCallback(dump_root,
tensor_debug_mode,
circular_buffer_size,
op_regex,
tensor_dtypes)
op_callbacks.add_op_callback(_state.dumping_callback.callback)
function_lib.CONCRETE_FUNCTION_CALLBACKS.append(
_state.dumping_callback.function_callback)
if _state.dumping_callback.dump_root != dump_root:
_state.dumping_callback.dump_root = dump_root
logging.info(
"Enabled dumping callback in thread %s "
"(dump root: %s, tensor debug mode: %s)",
threading.current_thread().name,
_state.dumping_callback.dump_root,
debug_event_pb2.TensorDebugMode.Name(tensor_debug_mode))
atexit.register(disable_dump_debug_info)
return _state.dumping_callback.get_writer()
@tf_export("debugging.experimental.disable_dump_debug_info")
def disable_dump_debug_info():
"""Disable the currently-enabled debugging dumping.
If the `enable_dump_debug_info()` method under the same Python namespace
has been invoked before, calling this method disables it. If no call to
`enable_dump_debug_info()` has been made, calling this method is a no-op.
Calling this method more than once is idempotent.
"""
if hasattr(_state, "dumping_callback"):
dump_root = _state.dumping_callback.dump_root
tfdbg_run_id = _state.dumping_callback.tfdbg_run_id
debug_events_writer.DebugEventsWriter(dump_root, tfdbg_run_id).Close()
op_callbacks.remove_op_callback(_state.dumping_callback.callback)
if (
_state.dumping_callback.function_callback
in function_lib.CONCRETE_FUNCTION_CALLBACKS
):
function_lib.CONCRETE_FUNCTION_CALLBACKS.remove(
_state.dumping_callback.function_callback
)
delattr(_state, "dumping_callback")
logging.info("Disabled dumping callback in thread %s (dump root: %s)",
threading.current_thread().name, dump_root)
tensorflow/python/debug/lib/dumping_callback_test.py
+1397
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/debug/lib/dumping_callback_test_lib.py
+49
View File
@@ -0,0 +1,49 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Shared library for testing tfdbg v2 dumping callback."""
import os
import shutil
import tempfile
import uuid
from tensorflow.python.debug.lib import check_numerics_callback
from tensorflow.python.debug.lib import debug_events_reader
from tensorflow.python.debug.lib import dumping_callback
from tensorflow.python.framework import test_util
from tensorflow.python.framework import versions
class DumpingCallbackTestBase(test_util.TensorFlowTestCase):
"""Base test-case class for tfdbg v2 callbacks."""
def setUp(self):
super(DumpingCallbackTestBase, self).setUp()
self.dump_root = tempfile.mkdtemp()
self.tfdbg_run_id = str(uuid.uuid4())
def tearDown(self):
if os.path.isdir(self.dump_root):
shutil.rmtree(self.dump_root, ignore_errors=True)
check_numerics_callback.disable_check_numerics()
dumping_callback.disable_dump_debug_info()
super(DumpingCallbackTestBase, self).tearDown()
def _readAndCheckMetadataFile(self):
"""Read and check the .metadata debug-events file."""
with debug_events_reader.DebugEventsReader(self.dump_root) as reader:
self.assertTrue(reader.tfdbg_run_id())
self.assertEqual(reader.tensorflow_version(), versions.__version__)
self.assertTrue(reader.tfdbg_file_version().startswith("debug.Event"))
tensorflow/python/debug/lib/grpc_debug_server.py
+492
View File
@@ -0,0 +1,492 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""gRPC debug server in Python."""
# pylint: disable=g-bad-import-order
import collections
import json
import queue
import threading
import time
from concurrent import futures
import grpc
from tensorflow.core.debug import debug_service_pb2
from tensorflow.core.framework import graph_pb2
from tensorflow.python.debug.lib import debug_graphs
from tensorflow.python.debug.lib import debug_service_pb2_grpc
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import compat
DebugWatch = collections.namedtuple("DebugWatch",
["node_name", "output_slot", "debug_op"])
def _state_change(new_state, node_name, output_slot, debug_op):
state_change = debug_service_pb2.EventReply.DebugOpStateChange()
state_change.state = new_state
state_change.node_name = node_name
state_change.output_slot = output_slot
state_change.debug_op = debug_op
return state_change
class EventListenerBaseStreamHandler:
"""Per-stream handler of EventListener gRPC streams."""
def __init__(self):
"""Constructor of EventListenerBaseStreamHandler."""
def on_core_metadata_event(self, event):
"""Callback for core metadata.
Args:
event: The Event proto that carries a JSON string in its
`log_message.message` field.
Returns:
`None` or an `EventReply` proto to be sent back to the client. If `None`,
an `EventReply` proto construct with the default no-arg constructor will
be sent back to the client.
"""
raise NotImplementedError(
"on_core_metadata_event() is not implemented in the base servicer "
"class")
def on_graph_def(self, graph_def, device_name, wall_time):
"""Callback for Event proto received through the gRPC stream.
This Event proto carries a GraphDef, encoded as bytes, in its graph_def
field.
Args:
graph_def: A GraphDef object.
device_name: Name of the device on which the graph was created.
wall_time: An epoch timestamp (in microseconds) for the graph.
Returns:
`None` or an `EventReply` proto to be sent back to the client. If `None`,
an `EventReply` proto construct with the default no-arg constructor will
be sent back to the client.
"""
raise NotImplementedError(
"on_graph_def() is not implemented in the base servicer class")
def on_value_event(self, event):
"""Callback for Event proto received through the gRPC stream.
This Event proto carries a Tensor in its summary.value[0] field.
Args:
event: The Event proto from the stream to be processed.
"""
raise NotImplementedError(
"on_value_event() is not implemented in the base servicer class")
class EventListenerBaseServicer(debug_service_pb2_grpc.EventListenerServicer):
"""Base Python class for gRPC debug server."""
def __init__(self, server_port, stream_handler_class):
"""Constructor.
Args:
server_port: (int) Port number to bind to.
stream_handler_class: A class of the base class
`EventListenerBaseStreamHandler` that will be used to constructor
stream handler objects during `SendEvents` calls.
"""
self._server_port = server_port
self._stream_handler_class = stream_handler_class
self._server_lock = threading.Lock()
self._server_started = False
self._stop_requested = False
self._debug_ops_state_change_queue = queue.Queue()
self._gated_grpc_debug_watches = set()
self._breakpoints = set()
def SendEvents(self, request_iterator, context):
"""Implementation of the SendEvents service method.
This method receives streams of Event protos from the client, and processes
them in ways specified in the on_event() callback. The stream is
bi-directional, but currently only the client-to-server stream (i.e., the
stream from the debug ops to the server) is used.
Args:
request_iterator: The incoming stream of Event protos.
context: Server context.
Raises:
ValueError: If there are more than one core metadata events.
Yields:
An empty stream of responses.
"""
core_metadata_count = 0
# A map from GraphDef hash to a list of received chunks.
graph_def_chunks = {}
tensor_chunks = {}
stream_handler = None
for event in request_iterator:
if not stream_handler:
stream_handler = self._stream_handler_class()
if event.summary and event.summary.value:
# An Event proto carrying a tensor value.
maybe_tensor_event = self._process_tensor_event_in_chunks(
event, tensor_chunks)
if maybe_tensor_event:
event_reply = stream_handler.on_value_event(maybe_tensor_event)
if event_reply is not None:
yield self._process_debug_op_state_changes(event_reply)
else:
# Non-tensor-value Event.
if event.graph_def:
# GraphDef-carrying Event.
maybe_graph_def, maybe_device_name, maybe_wall_time = (
self._process_encoded_graph_def_in_chunks(
event, graph_def_chunks))
if maybe_graph_def:
reply = stream_handler.on_graph_def(
maybe_graph_def, maybe_device_name, maybe_wall_time)
yield self._process_debug_op_state_changes(reply)
elif event.log_message.message:
# Core metadata-carrying Event.
core_metadata_count += 1
if core_metadata_count > 1:
raise ValueError(
"Expected one core metadata event; received multiple")
reply = stream_handler.on_core_metadata_event(event)
yield self._process_debug_op_state_changes(reply)
def _process_debug_op_state_changes(self, event_reply=None):
"""Dequeue and process all the queued debug-op state change protos.
Include all the debug-op state change protos in a `EventReply` proto.
Args:
event_reply: An `EventReply` to add the `DebugOpStateChange` protos to,
or `None`.
Returns:
An `EventReply` proto with the dequeued `DebugOpStateChange` protos (if
any) added.
"""
if event_reply is None:
event_reply = debug_service_pb2.EventReply()
while not self._debug_ops_state_change_queue.empty():
state_change = self._debug_ops_state_change_queue.get()
debug_node_key = (state_change.node_name, state_change.output_slot,
state_change.debug_op)
if (state_change.state ==
debug_service_pb2.EventReply.DebugOpStateChange.READ_WRITE):
logging.info("Adding breakpoint %s:%d:%s", state_change.node_name,
state_change.output_slot, state_change.debug_op)
self._breakpoints.add(debug_node_key)
elif (state_change.state ==
debug_service_pb2.EventReply.DebugOpStateChange.READ_ONLY):
logging.info("Adding watchpoint %s:%d:%s", state_change.node_name,
state_change.output_slot, state_change.debug_op)
if debug_node_key in self._breakpoints:
self._breakpoints.discard(debug_node_key)
elif (state_change.state ==
debug_service_pb2.EventReply.DebugOpStateChange.DISABLED):
logging.info("Removing watchpoint or breakpoint: %s:%d:%s",
state_change.node_name, state_change.output_slot,
state_change.debug_op)
if debug_node_key in self._breakpoints:
self._breakpoints.discard(debug_node_key)
else:
logging.warn(
"Attempting to remove a non-existent debug node key: %s",
debug_node_key)
new_state_change = event_reply.debug_op_state_changes.add()
new_state_change.CopyFrom(state_change)
return event_reply
def _process_tensor_event_in_chunks(self, event, tensor_chunks):
"""Possibly reassemble event chunks.
Due to gRPC's message size limit, a large tensor can be encapsulated in
multiple Event proto chunks to be sent through the debugger stream. This
method keeps track of the chunks that have arrived, reassemble all chunks
corresponding to a tensor when they have arrived and return the reassembled
Event proto.
Args:
event: The single Event proto that has arrived.
tensor_chunks: A dict used to keep track of the Event protos that have
arrived but haven't been reassembled.
Returns:
If all Event protos corresponding to a tensor have arrived, returns the
reassembled Event proto. Otherwise, return None.
"""
value = event.summary.value[0]
debugger_plugin_metadata = json.loads(
compat.as_text(value.metadata.plugin_data.content))
device_name = debugger_plugin_metadata["device"]
num_chunks = debugger_plugin_metadata["numChunks"]
chunk_index = debugger_plugin_metadata["chunkIndex"]
if num_chunks <= 1:
return event
debug_node_name = value.node_name
timestamp = int(event.wall_time)
tensor_key = "%s_%s_%d" % (device_name, debug_node_name, timestamp)
if tensor_key not in tensor_chunks:
tensor_chunks[tensor_key] = [None] * num_chunks
chunks = tensor_chunks[tensor_key]
if value.tensor.tensor_content:
chunks[chunk_index] = value.tensor
elif value.tensor.string_val:
chunks[chunk_index] = event
if None not in chunks:
if value.tensor.tensor_content:
event.summary.value[0].tensor.tensor_content = b"".join(
chunk.tensor_content for chunk in chunks)
del tensor_chunks[tensor_key]
return event
elif value.tensor.string_val:
merged_event = chunks[0]
for chunk in chunks[1:]:
merged_event.summary.value[0].tensor.string_val.extend(
list(chunk.summary.value[0].tensor.string_val))
return merged_event
def _process_encoded_graph_def_in_chunks(self,
event,
graph_def_chunks):
"""Process an Event proto containing a chunk of encoded GraphDef.
Args:
event: the Event proto containing the chunk of encoded GraphDef.
graph_def_chunks: A dict mapping keys for GraphDefs (i.e.,
"<graph_def_hash>,<device_name>,<wall_time>") to a list of chunks of
encoded GraphDefs.
Returns:
If all chunks of the GraphDef have arrived,
return decoded GraphDef proto, device name, wall_time.
Otherwise,
return None, None, None.
"""
graph_def = graph_pb2.GraphDef()
index_bar_0 = event.graph_def.find(b"|")
index_bar_1 = event.graph_def.find(b"|", index_bar_0 + 1)
index_bar_2 = event.graph_def.find(b"|", index_bar_1 + 1)
graph_def_hash_device_timestamp = event.graph_def[:index_bar_0]
chunk_index = int(event.graph_def[index_bar_0 + 1 : index_bar_1])
num_chunks = int(event.graph_def[index_bar_1 + 1 : index_bar_2])
if graph_def_hash_device_timestamp not in graph_def_chunks:
graph_def_chunks[graph_def_hash_device_timestamp] = [None] * num_chunks
graph_def_chunks[graph_def_hash_device_timestamp][
chunk_index] = event.graph_def[index_bar_2 + 1:]
if all(graph_def_chunks[graph_def_hash_device_timestamp]):
device_name = graph_def_hash_device_timestamp.split(b",")[1]
wall_time = int(graph_def_hash_device_timestamp.split(b",")[2])
graph_def.ParseFromString(
b"".join(graph_def_chunks[graph_def_hash_device_timestamp]))
del graph_def_chunks[graph_def_hash_device_timestamp]
self._process_graph_def(graph_def)
return graph_def, device_name, wall_time
else:
return None, None, None
def _process_graph_def(self, graph_def):
for node_def in graph_def.node:
if (debug_graphs.is_debug_node(node_def.name) and
node_def.attr["gated_grpc"].b):
node_name, output_slot, _, debug_op = (
debug_graphs.parse_debug_node_name(node_def.name))
self._gated_grpc_debug_watches.add(
DebugWatch(node_name, output_slot, debug_op))
def run_server(self, blocking=True):
"""Start running the server.
Args:
blocking: If `True`, block until `stop_server()` is invoked.
Raises:
ValueError: If server stop has already been requested, or if the server
has already started running.
"""
self._server_lock.acquire()
try:
if self._stop_requested:
raise ValueError("Server has already stopped")
if self._server_started:
raise ValueError("Server has already started running")
no_max_message_sizes = [("grpc.max_receive_message_length", -1),
("grpc.max_send_message_length", -1)]
self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10),
options=no_max_message_sizes)
debug_service_pb2_grpc.add_EventListenerServicer_to_server(self,
self.server)
self.server.add_insecure_port("[::]:%d" % self._server_port)
self.server.start()
self._server_started = True
finally:
self._server_lock.release()
if blocking:
while not self._stop_requested:
time.sleep(1.0)
def stop_server(self, grace=1.0):
"""Request server stopping.
Once stopped, server cannot be stopped or started again. This method is
non-blocking. Call `wait()` on the returned event to block until the server
has completely stopped.
Args:
grace: Grace period in seconds to be used when calling `server.stop()`.
Raises:
ValueError: If server stop has already been requested, or if the server
has not started running yet.
Returns:
A threading.Event that will be set when the server has completely stopped.
"""
self._server_lock.acquire()
try:
if not self._server_started:
raise ValueError("Server has not started running")
if self._stop_requested:
raise ValueError("Server has already stopped")
self._stop_requested = True
return self.server.stop(grace=grace)
finally:
self._server_lock.release()
def request_watch(self, node_name, output_slot, debug_op, breakpoint=False): # pylint: disable=redefined-builtin
"""Request enabling a debug tensor watchpoint or breakpoint.
This will let the server send a EventReply to the client side
(i.e., the debugged TensorFlow runtime process) to request adding a watch
key (i.e., <node_name>:<output_slot>:<debug_op>) to the list of enabled
watch keys. The list applies only to debug ops with the attribute
gated_grpc=True.
To disable the watch, use `request_unwatch()`.
Args:
node_name: (`str`) name of the node that the to-be-watched tensor belongs
to, e.g., "hidden/Weights".
output_slot: (`int`) output slot index of the tensor to watch.
debug_op: (`str`) name of the debug op to enable. This should not include
any attribute substrings.
breakpoint: (`bool`) Iff `True`, the debug op will block and wait until it
receives an `EventReply` response from the server. The `EventReply`
proto may carry a TensorProto that modifies the value of the debug op's
output tensor.
"""
self._debug_ops_state_change_queue.put(
_state_change(
debug_service_pb2.EventReply.DebugOpStateChange.READ_WRITE
if breakpoint
else debug_service_pb2.EventReply.DebugOpStateChange.READ_ONLY,
node_name, output_slot, debug_op))
def request_unwatch(self, node_name, output_slot, debug_op):
"""Request disabling a debug tensor watchpoint or breakpoint.
This is the opposite of `request_watch()`.
Args:
node_name: (`str`) name of the node that the to-be-watched tensor belongs
to, e.g., "hidden/Weights".
output_slot: (`int`) output slot index of the tensor to watch.
debug_op: (`str`) name of the debug op to enable. This should not include
any attribute substrings.
"""
self._debug_ops_state_change_queue.put(
_state_change(
debug_service_pb2.EventReply.DebugOpStateChange.DISABLED, node_name,
output_slot, debug_op))
@property
def breakpoints(self):
"""Get a set of the currently-activated breakpoints.
Returns:
A `set` of 3-tuples: (node_name, output_slot, debug_op), e.g.,
{("MatMul", 0, "DebugIdentity")}.
"""
return self._breakpoints
def gated_grpc_debug_watches(self):
"""Get the list of debug watches with attribute gated_grpc=True.
Since the server receives `GraphDef` from the debugged runtime, it can only
return such debug watches that it has received so far.
Returns:
A `list` of `DebugWatch` `namedtuples` representing the debug watches with
gated_grpc=True. Each `namedtuple` element has the attributes:
`node_name` as a `str`,
`output_slot` as an `int`,
`debug_op` as a `str`.
"""
return list(self._gated_grpc_debug_watches)
def SendTracebacks(self, request, context):
"""Base implementation of the handling of SendTracebacks calls.
The base implementation does nothing with the incoming request.
Override in an implementation of the server if necessary.
Args:
request: A `CallTraceback` proto, containing information about the
type (e.g., graph vs. eager execution) and source-code traceback of the
call and (any) associated `tf.Graph`s.
context: Server context.
Returns:
A `EventReply` proto.
"""
return debug_service_pb2.EventReply()
def SendSourceFiles(self, request, context):
"""Base implementation of the handling of SendSourceFiles calls.
The base implementation does nothing with the incoming request.
Override in an implementation of the server if necessary.
Args:
request: A `DebuggedSourceFiles` proto, containing the path, content, size
and last-modified timestamp of source files.
context: Server context.
Returns:
A `EventReply` proto.
"""
return debug_service_pb2.EventReply()
tensorflow/python/debug/lib/grpc_debug_test_server.py
+484
View File
@@ -0,0 +1,484 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""GRPC debug server for testing."""
import collections
import errno
import functools
import hashlib
import json
import os
import re
import tempfile
import threading
import time
import portpicker
from tensorflow.core.debug import debug_service_pb2
from tensorflow.core.protobuf import config_pb2
from tensorflow.core.util import event_pb2
from tensorflow.python.client import session
from tensorflow.python.debug.lib import debug_data
from tensorflow.python.debug.lib import debug_utils
from tensorflow.python.debug.lib import grpc_debug_server
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import errors
from tensorflow.python.lib.io import file_io
from tensorflow.python.ops import variables
from tensorflow.python.util import compat
def _get_dump_file_path(dump_root, device_name, debug_node_name):
"""Get the file path of the dump file for a debug node.
Args:
dump_root: (str) Root dump directory.
device_name: (str) Name of the device that the debug node resides on.
debug_node_name: (str) Name of the debug node, e.g.,
cross_entropy/Log:0:DebugIdentity.
Returns:
(str) Full path of the dump file.
"""
dump_root = os.path.join(
dump_root, debug_data.device_name_to_device_path(device_name))
if "/" in debug_node_name:
dump_dir = os.path.join(dump_root, os.path.dirname(debug_node_name))
dump_file_name = re.sub(":", "_", os.path.basename(debug_node_name))
else:
dump_dir = dump_root
dump_file_name = re.sub(":", "_", debug_node_name)
now_microsec = int(round(time.time() * 1000 * 1000))
dump_file_name += "_%d" % now_microsec
return os.path.join(dump_dir, dump_file_name)
class EventListenerTestStreamHandler(
grpc_debug_server.EventListenerBaseStreamHandler):
"""Implementation of EventListenerBaseStreamHandler that dumps to file."""
def __init__(self, dump_dir, event_listener_servicer):
super(EventListenerTestStreamHandler, self).__init__()
self._dump_dir = dump_dir
self._event_listener_servicer = event_listener_servicer
if self._dump_dir:
self._try_makedirs(self._dump_dir)
self._grpc_path = None
self._cached_graph_defs = []
self._cached_graph_def_device_names = []
self._cached_graph_def_wall_times = []
def on_core_metadata_event(self, event):
self._event_listener_servicer.toggle_watch()
core_metadata = json.loads(event.log_message.message)
if not self._grpc_path:
grpc_path = core_metadata["grpc_path"]
if grpc_path:
if grpc_path.startswith("/"):
grpc_path = grpc_path[1:]
if self._dump_dir:
self._dump_dir = os.path.join(self._dump_dir, grpc_path)
# Write cached graph defs to filesystem.
for graph_def, device_name, wall_time in zip(
self._cached_graph_defs,
self._cached_graph_def_device_names,
self._cached_graph_def_wall_times):
self._write_graph_def(graph_def, device_name, wall_time)
if self._dump_dir:
self._write_core_metadata_event(event)
else:
self._event_listener_servicer.core_metadata_json_strings.append(
event.log_message.message)
def on_graph_def(self, graph_def, device_name, wall_time):
"""Implementation of the tensor value-carrying Event proto callback.
Args:
graph_def: A GraphDef object.
device_name: Name of the device on which the graph was created.
wall_time: An epoch timestamp (in microseconds) for the graph.
"""
if self._dump_dir:
if self._grpc_path:
self._write_graph_def(graph_def, device_name, wall_time)
else:
self._cached_graph_defs.append(graph_def)
self._cached_graph_def_device_names.append(device_name)
self._cached_graph_def_wall_times.append(wall_time)
else:
self._event_listener_servicer.partition_graph_defs.append(graph_def)
def on_value_event(self, event):
"""Implementation of the tensor value-carrying Event proto callback.
Writes the Event proto to the file system for testing. The path written to
follows the same pattern as the file:// debug URLs of tfdbg, i.e., the
name scope of the op becomes the directory structure under the dump root
directory.
Args:
event: The Event proto carrying a tensor value.
Returns:
If the debug node belongs to the set of currently activated breakpoints,
a `EventReply` proto will be returned.
"""
if self._dump_dir:
self._write_value_event(event)
else:
value = event.summary.value[0]
tensor_value = debug_data.load_tensor_from_event(event)
self._event_listener_servicer.debug_tensor_values[value.node_name].append(
tensor_value)
items = event.summary.value[0].node_name.split(":")
node_name = items[0]
output_slot = int(items[1])
debug_op = items[2]
if ((node_name, output_slot, debug_op) in
self._event_listener_servicer.breakpoints):
return debug_service_pb2.EventReply()
def _try_makedirs(self, dir_path):
if not os.path.isdir(dir_path):
try:
os.makedirs(dir_path)
except OSError as error:
if error.errno != errno.EEXIST:
raise
def _write_core_metadata_event(self, event):
core_metadata_path = os.path.join(
self._dump_dir,
debug_data.METADATA_FILE_PREFIX + debug_data.CORE_METADATA_TAG +
"_%d" % event.wall_time)
self._try_makedirs(self._dump_dir)
with open(core_metadata_path, "wb") as f:
f.write(event.SerializeToString())
def _write_graph_def(self, graph_def, device_name, wall_time):
encoded_graph_def = graph_def.SerializeToString()
graph_hash = int(hashlib.sha1(encoded_graph_def).hexdigest(), 16)
event = event_pb2.Event(graph_def=encoded_graph_def, wall_time=wall_time)
graph_file_path = os.path.join(
self._dump_dir,
debug_data.device_name_to_device_path(device_name),
debug_data.METADATA_FILE_PREFIX + debug_data.GRAPH_FILE_TAG +
debug_data.HASH_TAG + "%d_%d" % (graph_hash, wall_time))
self._try_makedirs(os.path.dirname(graph_file_path))
with open(graph_file_path, "wb") as f:
f.write(event.SerializeToString())
def _write_value_event(self, event):
value = event.summary.value[0]
# Obtain the device name from the metadata.
summary_metadata = event.summary.value[0].metadata
if not summary_metadata.plugin_data:
raise ValueError("The value lacks plugin data.")
try:
content = json.loads(compat.as_text(summary_metadata.plugin_data.content))
except ValueError as err:
raise ValueError("Could not parse content into JSON: %r, %r" % (content,
err))
device_name = content["device"]
dump_full_path = _get_dump_file_path(
self._dump_dir, device_name, value.node_name)
self._try_makedirs(os.path.dirname(dump_full_path))
with open(dump_full_path, "wb") as f:
f.write(event.SerializeToString())
class EventListenerTestServicer(grpc_debug_server.EventListenerBaseServicer):
"""An implementation of EventListenerBaseServicer for testing."""
def __init__(self, server_port, dump_dir, toggle_watch_on_core_metadata=None):
"""Constructor of EventListenerTestServicer.
Args:
server_port: (int) The server port number.
dump_dir: (str) The root directory to which the data files will be
dumped. If empty or None, the received debug data will not be dumped
to the file system: they will be stored in memory instead.
toggle_watch_on_core_metadata: A list of
(node_name, output_slot, debug_op) tuples to toggle the
watchpoint status during the on_core_metadata calls (optional).
"""
self.core_metadata_json_strings = []
self.partition_graph_defs = []
self.debug_tensor_values = collections.defaultdict(list)
self._initialize_toggle_watch_state(toggle_watch_on_core_metadata)
grpc_debug_server.EventListenerBaseServicer.__init__(
self, server_port,
functools.partial(EventListenerTestStreamHandler, dump_dir, self))
# Members for storing the graph ops traceback and source files.
self._call_types = []
self._call_keys = []
self._origin_stacks = []
self._origin_id_to_strings = []
self._graph_tracebacks = []
self._graph_versions = []
self._source_files = []
def _initialize_toggle_watch_state(self, toggle_watches):
self._toggle_watches = toggle_watches
self._toggle_watch_state = {}
if self._toggle_watches:
for watch_key in self._toggle_watches:
self._toggle_watch_state[watch_key] = False
def toggle_watch(self):
for watch_key in self._toggle_watch_state:
node_name, output_slot, debug_op = watch_key
if self._toggle_watch_state[watch_key]:
self.request_unwatch(node_name, output_slot, debug_op)
else:
self.request_watch(node_name, output_slot, debug_op)
self._toggle_watch_state[watch_key] = (
not self._toggle_watch_state[watch_key])
def clear_data(self):
self.core_metadata_json_strings = []
self.partition_graph_defs = []
self.debug_tensor_values = collections.defaultdict(list)
self._call_types = []
self._call_keys = []
self._origin_stacks = []
self._origin_id_to_strings = []
self._graph_tracebacks = []
self._graph_versions = []
self._source_files = []
def SendTracebacks(self, request, context):
self._call_types.append(request.call_type)
self._call_keys.append(request.call_key)
self._origin_stacks.append(request.origin_stack)
self._origin_id_to_strings.append(request.origin_id_to_string)
self._graph_tracebacks.append(request.graph_traceback)
self._graph_versions.append(request.graph_version)
return debug_service_pb2.EventReply()
def SendSourceFiles(self, request, context):
self._source_files.append(request)
return debug_service_pb2.EventReply()
def query_op_traceback(self, op_name):
"""Query the traceback of an op.
Args:
op_name: Name of the op to query.
Returns:
The traceback of the op, as a list of 3-tuples:
(filename, lineno, function_name)
Raises:
ValueError: If the op cannot be found in the tracebacks received by the
server so far.
"""
for op_log_proto in self._graph_tracebacks:
for log_entry in op_log_proto.log_entries:
if log_entry.name == op_name:
return self._code_def_to_traceback(log_entry.code_def,
op_log_proto.id_to_string)
raise ValueError(
"Op '%s' does not exist in the tracebacks received by the debug "
"server." % op_name)
def query_origin_stack(self):
"""Query the stack of the origin of the execution call.
Returns:
A `list` of all tracebacks. Each item corresponds to an execution call,
i.e., a `SendTracebacks` request. Each item is a `list` of 3-tuples:
(filename, lineno, function_name).
"""
ret = []
for stack, id_to_string in zip(
self._origin_stacks, self._origin_id_to_strings):
ret.append(self._code_def_to_traceback(stack, id_to_string))
return ret
def query_call_types(self):
return self._call_types
def query_call_keys(self):
return self._call_keys
def query_graph_versions(self):
return self._graph_versions
def query_source_file_line(self, file_path, lineno):
"""Query the content of a given line in a source file.
Args:
file_path: Path to the source file.
lineno: Line number as an `int`.
Returns:
Content of the line as a string.
Raises:
ValueError: If no source file is found at the given file_path.
"""
if not self._source_files:
raise ValueError(
"This debug server has not received any source file contents yet.")
for source_files in self._source_files:
for source_file_proto in source_files.source_files:
if source_file_proto.file_path == file_path:
return source_file_proto.lines[lineno - 1]
raise ValueError(
"Source file at path %s has not been received by the debug server",
file_path)
def _code_def_to_traceback(self, code_def, id_to_string):
return [(id_to_string[trace.file_id],
trace.lineno,
id_to_string[trace.function_id]) for trace in code_def.traces]
def start_server_on_separate_thread(dump_to_filesystem=True,
server_start_delay_sec=0.0,
poll_server=False,
blocking=True,
toggle_watch_on_core_metadata=None):
"""Create a test gRPC debug server and run on a separate thread.
Args:
dump_to_filesystem: (bool) whether the debug server will dump debug data
to the filesystem.
server_start_delay_sec: (float) amount of time (in sec) to delay the server
start up for.
poll_server: (bool) whether the server will be polled till success on
startup.
blocking: (bool) whether the server should be started in a blocking mode.
toggle_watch_on_core_metadata: A list of
(node_name, output_slot, debug_op) tuples to toggle the
watchpoint status during the on_core_metadata calls (optional).
Returns:
server_port: (int) Port on which the server runs.
debug_server_url: (str) grpc:// URL to the server.
server_dump_dir: (str) The debug server's dump directory.
server_thread: The server Thread object.
server: The `EventListenerTestServicer` object.
Raises:
ValueError: If polling the server process for ready state is not successful
within maximum polling count.
"""
server_port = portpicker.pick_unused_port()
debug_server_url = "grpc://localhost:%d" % server_port
server_dump_dir = tempfile.mkdtemp() if dump_to_filesystem else None
server = EventListenerTestServicer(
server_port=server_port,
dump_dir=server_dump_dir,
toggle_watch_on_core_metadata=toggle_watch_on_core_metadata)
def delay_then_run_server():
time.sleep(server_start_delay_sec)
server.run_server(blocking=blocking)
server_thread = threading.Thread(target=delay_then_run_server)
server_thread.start()
if poll_server:
if not _poll_server_till_success(
50,
0.2,
debug_server_url,
server_dump_dir,
server,
gpu_memory_fraction=0.1):
raise ValueError(
"Failed to start test gRPC debug server at port %d" % server_port)
server.clear_data()
return server_port, debug_server_url, server_dump_dir, server_thread, server
def _poll_server_till_success(max_attempts,
sleep_per_poll_sec,
debug_server_url,
dump_dir,
server,
gpu_memory_fraction=1.0):
"""Poll server until success or exceeding max polling count.
Args:
max_attempts: (int) How many times to poll at maximum
sleep_per_poll_sec: (float) How many seconds to sleep for after each
unsuccessful poll.
debug_server_url: (str) gRPC URL to the debug server.
dump_dir: (str) Dump directory to look for files in. If None, will directly
check data from the server object.
server: The server object.
gpu_memory_fraction: (float) Fraction of GPU memory to be
allocated for the Session used in server polling.
Returns:
(bool) Whether the polling succeeded within max_polls attempts.
"""
poll_count = 0
config = config_pb2.ConfigProto(gpu_options=config_pb2.GPUOptions(
per_process_gpu_memory_fraction=gpu_memory_fraction))
with session.Session(config=config) as sess:
for poll_count in range(max_attempts):
server.clear_data()
print("Polling: poll_count = %d" % poll_count)
x_init_name = "x_init_%d" % poll_count
x_init = constant_op.constant([42.0], shape=[1], name=x_init_name)
x = variables.Variable(x_init, name=x_init_name)
run_options = config_pb2.RunOptions()
debug_utils.add_debug_tensor_watch(
run_options, x_init_name, 0, debug_urls=[debug_server_url])
try:
sess.run(x.initializer, options=run_options)
except errors.FailedPreconditionError:
pass
if dump_dir:
if os.path.isdir(
dump_dir) and debug_data.DebugDumpDir(dump_dir).size > 0:
file_io.delete_recursively(dump_dir)
print("Poll succeeded.")
return True
else:
print("Poll failed. Sleeping for %f s" % sleep_per_poll_sec)
time.sleep(sleep_per_poll_sec)
else:
if server.debug_tensor_values:
print("Poll succeeded.")
return True
else:
print("Poll failed. Sleeping for %f s" % sleep_per_poll_sec)
time.sleep(sleep_per_poll_sec)
return False
tensorflow/python/debug/lib/grpc_tensorflow_server.py
+157
View File
@@ -0,0 +1,157 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Python-based TensorFlow GRPC server.
Takes input arguments cluster_spec, job_name and task_id, and start a blocking
TensorFlow GRPC server.
Usage:
grpc_tensorflow_server.py --cluster_spec=SPEC --job_name=NAME --task_id=ID
Where:
SPEC is <JOB>(,<JOB>)*
JOB is <NAME>|<HOST:PORT>(;<HOST:PORT>)*
NAME is a valid job name ([a-z][0-9a-z]*)
HOST is a hostname or IP address
PORT is a port number
"""
import argparse
import sys
from absl import app
from tensorflow.core.protobuf import config_pb2
from tensorflow.core.protobuf import tensorflow_server_pb2
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.training import server_lib
def parse_cluster_spec(cluster_spec, cluster, verbose=False):
"""Parse content of cluster_spec string and inject info into cluster protobuf.
Args:
cluster_spec: cluster specification string, e.g.,
"local|localhost:2222;localhost:2223"
cluster: cluster protobuf.
verbose: If verbose logging is requested.
Raises:
ValueError: if the cluster_spec string is invalid.
"""
job_strings = cluster_spec.split(",")
if not cluster_spec:
raise ValueError("Empty cluster_spec string")
for job_string in job_strings:
job_def = cluster.job.add()
if job_string.count("|") != 1:
raise ValueError("Not exactly one instance of '|' in cluster_spec")
job_name = job_string.split("|")[0]
if not job_name:
raise ValueError("Empty job_name in cluster_spec")
job_def.name = job_name
if verbose:
logging.info("Added job named \"%s\"", job_name)
job_tasks = job_string.split("|")[1].split(";")
for i in range(len(job_tasks)):
if not job_tasks[i]:
raise ValueError("Empty task string at position %d" % i)
job_def.tasks[i] = job_tasks[i]
if verbose:
logging.info(" Added task \"%s\" to job \"%s\"",
job_tasks[i], job_name)
def main(unused_args):
# Create Protobuf ServerDef
server_def = tensorflow_server_pb2.ServerDef(protocol="grpc")
# Cluster info
parse_cluster_spec(FLAGS.cluster_spec, server_def.cluster, FLAGS.verbose)
# Job name
if not FLAGS.job_name:
raise ValueError("Empty job_name")
server_def.job_name = FLAGS.job_name
# Task index
if FLAGS.task_id < 0:
raise ValueError("Invalid task_id: %d" % FLAGS.task_id)
server_def.task_index = FLAGS.task_id
config = config_pb2.ConfigProto(gpu_options=config_pb2.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction))
# Create GRPC Server instance
server = server_lib.Server(server_def, config=config)
# join() is blocking, unlike start()
server.join()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.register("type", "bool", lambda v: v.lower() == "true")
parser.add_argument(
"--cluster_spec",
type=str,
default="",
help="""\
Cluster spec: SPEC. SPEC is <JOB>(,<JOB>)*," JOB is
<NAME>|<HOST:PORT>(;<HOST:PORT>)*," NAME is a valid job name
([a-z][0-9a-z]*)," HOST is a hostname or IP address," PORT is a
port number." E.g., local|localhost:2222;localhost:2223,
ps|ps0:2222;ps1:2222\
"""
)
parser.add_argument(
"--job_name",
type=str,
default="",
help="Job name: e.g., local"
)
parser.add_argument(
"--task_id",
type=int,
default=0,
help="Task index, e.g., 0"
)
parser.add_argument(
"--gpu_memory_fraction",
type=float,
default=1.0,
help="Fraction of GPU memory allocated",)
parser.add_argument(
"--verbose",
type="bool",
nargs="?",
const=True,
default=False,
help="Verbose mode"
)
FLAGS, unparsed = parser.parse_known_args()
app.run(main=main, argv=[sys.argv[0]] + unparsed)
tensorflow/python/debug/lib/op_callbacks_common.py
+46
View File
@@ -0,0 +1,46 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Common utilities and settings used by tfdbg v2's op callbacks."""
# The ops that are skipped by tfdbg v2's op callbacks.
# They belong to TensorFlow's control flow ops (e.g., "Enter", "StatelessIf")
# and ops that wrap nested tf.function calls.
OP_CALLBACK_SKIP_OPS = (
# TODO(b/139668453): The following skipped ops are related to a limitation
# in the op callback.
b"Enter",
b"Exit",
b"Identity",
b"If",
b"LoopCond",
b"Merge",
b"NextIteration",
b"StatelessIf",
b"StatefulPartitionedCall",
b"Switch",
b"While",
# NOTE(b/154097452): On TPUs, debugger ops are colocated with RemoteCall
# ops. This exclusion prevents an error due to no OpKernel for those
# debugger ops.
b"RemoteCall",
# TPU-specific ops begin.
b"TPUReplicatedInput",
b"TPUReplicateMetadata",
b"TPUCompilationResult",
b"TPUReplicatedOutput",
b"ConfigureDistributedTPU",
# Other special ops used by TensorFlow internally.
b"DestroyResourceOp",
)
tensorflow/python/debug/lib/profiling.py
+104
View File
@@ -0,0 +1,104 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Data structures and algorithms for profiling information."""
import os
class ProfileDatum(object):
"""Profile data point."""
def __init__(self,
device_name,
node_exec_stats,
file_path,
line_number,
func_name,
op_type):
"""Constructor.
Args:
device_name: (string) name of the device.
node_exec_stats: `NodeExecStats` proto.
file_path: path to the source file involved in creating the op.
line_number: line number in the file involved in creating the op.
func_name: name of the function that the line belongs to.
op_type: (string) Operation type.
"""
self.device_name = device_name
self.node_exec_stats = node_exec_stats
self.file_path = file_path
self.line_number = line_number
self.func_name = func_name
if self.file_path:
self.file_line_func = "%s:%d(%s)" % (
os.path.basename(self.file_path), self.line_number, self.func_name)
else:
self.file_line_func = ""
self.op_type = op_type
self.start_time = self.node_exec_stats.all_start_micros
self.op_time = (self.node_exec_stats.op_end_rel_micros -
self.node_exec_stats.op_start_rel_micros)
@property
def exec_time(self):
"""Op execution time plus pre- and post-processing."""
return self.node_exec_stats.all_end_rel_micros
class AggregateProfile(object):
"""Profile summary data for aggregating a number of ProfileDatum."""
def __init__(self, profile_datum):
"""Constructor.
Args:
profile_datum: (`ProfileDatum`) an instance of `ProfileDatum` to
initialize this object with.
"""
self.total_op_time = profile_datum.op_time
self.total_exec_time = profile_datum.exec_time
device_and_node = "%s:%s" % (profile_datum.device_name,
profile_datum.node_exec_stats.node_name)
self._node_to_exec_count = {device_and_node: 1}
def add(self, profile_datum):
"""Accumulate a new instance of ProfileDatum.
Args:
profile_datum: (`ProfileDatum`) an instance of `ProfileDatum` to
accumulate to this object.
"""
self.total_op_time += profile_datum.op_time
self.total_exec_time += profile_datum.exec_time
device_and_node = "%s:%s" % (profile_datum.device_name,
profile_datum.node_exec_stats.node_name)
device_and_node = "%s:%s" % (profile_datum.device_name,
profile_datum.node_exec_stats.node_name)
if device_and_node in self._node_to_exec_count:
self._node_to_exec_count[device_and_node] += 1
else:
self._node_to_exec_count[device_and_node] = 1
@property
def node_count(self):
return len(self._node_to_exec_count)
@property
def node_exec_count(self):
return sum(self._node_to_exec_count.values())
tensorflow/python/debug/lib/profiling_test.py
+96
View File
@@ -0,0 +1,96 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Unit tests for the basic data structures and algorithms for profiling."""
from tensorflow.core.framework import step_stats_pb2
from tensorflow.python.debug.lib import profiling
from tensorflow.python.framework import test_util
from tensorflow.python.platform import googletest
class AggregateProfile(test_util.TensorFlowTestCase):
def setUp(self):
node_1 = step_stats_pb2.NodeExecStats(
node_name="Add/123",
op_start_rel_micros=3,
op_end_rel_micros=5,
all_end_rel_micros=4)
self.profile_datum_1 = profiling.ProfileDatum(
"cpu:0", node_1, "/foo/bar.py", 10, "func1", "Add")
node_2 = step_stats_pb2.NodeExecStats(
node_name="Mul/456",
op_start_rel_micros=13,
op_end_rel_micros=16,
all_end_rel_micros=17)
self.profile_datum_2 = profiling.ProfileDatum(
"cpu:0", node_2, "/foo/bar.py", 11, "func1", "Mul")
node_3 = step_stats_pb2.NodeExecStats(
node_name="Add/123",
op_start_rel_micros=103,
op_end_rel_micros=105,
all_end_rel_micros=4)
self.profile_datum_3 = profiling.ProfileDatum(
"cpu:0", node_3, "/foo/bar.py", 12, "func1", "Add")
node_4 = step_stats_pb2.NodeExecStats(
node_name="Add/123",
op_start_rel_micros=203,
op_end_rel_micros=205,
all_end_rel_micros=4)
self.profile_datum_4 = profiling.ProfileDatum(
"gpu:0", node_4, "/foo/bar.py", 13, "func1", "Add")
def testAggregateProfileConstructorWorks(self):
aggregate_data = profiling.AggregateProfile(self.profile_datum_1)
self.assertEqual(2, aggregate_data.total_op_time)
self.assertEqual(4, aggregate_data.total_exec_time)
self.assertEqual(1, aggregate_data.node_count)
self.assertEqual(1, aggregate_data.node_exec_count)
def testAddToAggregateProfileWithDifferentNodeWorks(self):
aggregate_data = profiling.AggregateProfile(self.profile_datum_1)
aggregate_data.add(self.profile_datum_2)
self.assertEqual(5, aggregate_data.total_op_time)
self.assertEqual(21, aggregate_data.total_exec_time)
self.assertEqual(2, aggregate_data.node_count)
self.assertEqual(2, aggregate_data.node_exec_count)
def testAddToAggregateProfileWithSameNodeWorks(self):
aggregate_data = profiling.AggregateProfile(self.profile_datum_1)
aggregate_data.add(self.profile_datum_2)
aggregate_data.add(self.profile_datum_3)
self.assertEqual(7, aggregate_data.total_op_time)
self.assertEqual(25, aggregate_data.total_exec_time)
self.assertEqual(2, aggregate_data.node_count)
self.assertEqual(3, aggregate_data.node_exec_count)
def testAddToAggregateProfileWithDifferentDeviceSameNodeWorks(self):
aggregate_data = profiling.AggregateProfile(self.profile_datum_1)
aggregate_data.add(self.profile_datum_4)
self.assertEqual(4, aggregate_data.total_op_time)
self.assertEqual(8, aggregate_data.total_exec_time)
self.assertEqual(2, aggregate_data.node_count)
self.assertEqual(2, aggregate_data.node_exec_count)
if __name__ == "__main__":
googletest.main()
tensorflow/python/debug/lib/session_debug_file_test.py
+134
View File
@@ -0,0 +1,134 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for debugger functionalities in tf.Session with file:// URLs."""
import os
import tempfile
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.client import session
from tensorflow.python.debug.lib import debug_data
from tensorflow.python.debug.lib import debug_utils
from tensorflow.python.debug.lib import session_debug_testlib
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variable_v1
from tensorflow.python.platform import googletest
@test_util.run_v1_only("b/120545219")
class SessionDebugFileTest(session_debug_testlib.SessionDebugTestBase):
def _debug_urls(self, run_number=None):
return ["file://%s" % self._debug_dump_dir(run_number=run_number)]
def _debug_dump_dir(self, run_number=None):
if run_number is None:
return self._dump_root
else:
return os.path.join(self._dump_root, "run_%d" % run_number)
def testAllowsDifferentWatchesOnDifferentRuns(self):
"""Test watching different tensors on different runs of the same graph."""
with session.Session(
config=session_debug_testlib.no_rewrite_session_config()) as sess:
u_init_val = [[5.0, 3.0], [-1.0, 0.0]]
v_init_val = [[2.0], [-1.0]]
# Use node names with overlapping namespace (i.e., parent directory) to
# test concurrent, non-racing directory creation.
u_name = "diff_Watch/u"
v_name = "diff_Watch/v"
u_init = constant_op.constant(u_init_val, shape=[2, 2])
u = variable_v1.VariableV1(u_init, name=u_name)
v_init = constant_op.constant(v_init_val, shape=[2, 1])
v = variable_v1.VariableV1(v_init, name=v_name)
w = math_ops.matmul(u, v, name="diff_Watch/matmul")
u.initializer.run()
v.initializer.run()
for i in range(2):
run_options = config_pb2.RunOptions(output_partition_graphs=True)
run_dump_root = self._debug_dump_dir(run_number=i)
debug_urls = self._debug_urls(run_number=i)
if i == 0:
# First debug run: Add debug tensor watch for u.
debug_utils.add_debug_tensor_watch(
run_options, "%s/read" % u_name, 0, debug_urls=debug_urls)
else:
# Second debug run: Add debug tensor watch for v.
debug_utils.add_debug_tensor_watch(
run_options, "%s/read" % v_name, 0, debug_urls=debug_urls)
run_metadata = config_pb2.RunMetadata()
# Invoke Session.run().
sess.run(w, options=run_options, run_metadata=run_metadata)
self.assertEqual(self._expected_partition_graph_count,
len(run_metadata.partition_graphs))
dump = debug_data.DebugDumpDir(
run_dump_root, partition_graphs=run_metadata.partition_graphs)
self.assertTrue(dump.loaded_partition_graphs())
# Each run should have generated only one dumped tensor, not two.
self.assertEqual(1, dump.size)
if i == 0:
self.assertAllClose([u_init_val],
dump.get_tensors("%s/read" % u_name, 0,
"DebugIdentity"))
self.assertGreaterEqual(
dump.get_rel_timestamps("%s/read" % u_name, 0,
"DebugIdentity")[0], 0)
else:
self.assertAllClose([v_init_val],
dump.get_tensors("%s/read" % v_name, 0,
"DebugIdentity"))
self.assertGreaterEqual(
dump.get_rel_timestamps("%s/read" % v_name, 0,
"DebugIdentity")[0], 0)
class SessionDebugConcurrentTest(
session_debug_testlib.DebugConcurrentRunCallsTest):
def setUp(self):
self._num_concurrent_runs = 3
self._dump_roots = []
for _ in range(self._num_concurrent_runs):
self._dump_roots.append(tempfile.mkdtemp())
def tearDown(self):
ops.reset_default_graph()
for dump_root in self._dump_roots:
if os.path.isdir(dump_root):
file_io.delete_recursively(dump_root)
def _get_concurrent_debug_urls(self):
return [("file://%s" % dump_root) for dump_root in self._dump_roots]
if __name__ == "__main__":
googletest.main()
tensorflow/python/debug/lib/session_debug_multi_gpu_test.py
+89
View File
@@ -0,0 +1,89 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for debugger functionalities under multiple (i.e., >1) GPUs."""
import os
import tempfile
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.client import device_lib
from tensorflow.python.client import session
from tensorflow.python.debug.lib import debug_data
from tensorflow.python.debug.lib import debug_utils
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import googletest
class SessionDebugMultiGPUTest(test_util.TensorFlowTestCase):
def setUp(self):
self._dump_root = tempfile.mkdtemp()
def tearDown(self):
ops.reset_default_graph()
# Tear down temporary dump directory.
if os.path.isdir(self._dump_root):
file_io.delete_recursively(self._dump_root)
def testMultiGPUSessionRun(self):
local_devices = device_lib.list_local_devices()
gpu_device_names = []
for device in local_devices:
if device.device_type == "GPU":
gpu_device_names.append(device.name)
gpu_device_names = sorted(gpu_device_names)
if len(gpu_device_names) < 2:
self.skipTest(
"This test requires at least 2 GPUs, but only %d is available." %
len(gpu_device_names))
with session.Session() as sess:
v = variables.Variable([10.0, 15.0], dtype=dtypes.float32, name="v")
with ops.device(gpu_device_names[0]):
u0 = math_ops.add(v, v, name="u0")
with ops.device(gpu_device_names[1]):
u1 = math_ops.multiply(v, v, name="u1")
w = math_ops.subtract(u1, u0, name="w")
self.evaluate(v.initializer)
run_options = config_pb2.RunOptions(output_partition_graphs=True)
debug_utils.watch_graph(run_options, sess.graph,
debug_urls="file://" + self._dump_root)
run_metadata = config_pb2.RunMetadata()
self.assertAllClose(
[80.0, 195.0],
sess.run(w, options=run_options, run_metadata=run_metadata))
debug_dump_dir = debug_data.DebugDumpDir(
self._dump_root, partition_graphs=run_metadata.partition_graphs)
self.assertEqual(3, len(debug_dump_dir.devices()))
self.assertAllClose(
[10.0, 15.0], debug_dump_dir.get_tensors("v", 0, "DebugIdentity")[0])
self.assertAllClose(
[20.0, 30.0], debug_dump_dir.get_tensors("u0", 0, "DebugIdentity")[0])
self.assertAllClose(
[100.0, 225.0],
debug_dump_dir.get_tensors("u1", 0, "DebugIdentity")[0])
if __name__ == "__main__":
googletest.main()
tensorflow/python/debug/lib/session_debug_testlib.py
+1578
View File
File diff suppressed because it is too large Load Diff
tensorflow/python/debug/lib/source_remote.py
+210
View File
@@ -0,0 +1,210 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Communicating tracebacks and source code with debug server."""
import socket
import grpc
from tensorflow.core.debug import debug_service_pb2
from tensorflow.core.protobuf import debug_pb2
from tensorflow.python.debug.lib import common
from tensorflow.python.debug.lib import debug_service_pb2_grpc
from tensorflow.python.debug.lib import source_utils
from tensorflow.python.platform import gfile
from tensorflow.python.profiler import tfprof_logger
def _load_debugged_source_file(file_path, source_file_proto):
file_stat = gfile.Stat(file_path)
source_file_proto.host = socket.gethostname()
source_file_proto.file_path = file_path
source_file_proto.last_modified = file_stat.mtime_nsec
source_file_proto.bytes = file_stat.length
try:
with gfile.Open(file_path, "r") as f:
source_file_proto.lines.extend(f.read().splitlines())
except IOError:
pass
def _string_to_id(string, string_to_id):
if string not in string_to_id:
string_to_id[string] = len(string_to_id)
return string_to_id[string]
def _format_origin_stack(origin_stack, call_traceback_proto):
"""Format a traceback stack for a `CallTraceback` proto.
Args:
origin_stack: The stack list as returned by `traceback.extract_stack()`.
call_traceback_proto: A `CallTraceback` proto whose fields are to be
populated.
"""
string_to_id = {}
string_to_id[None] = 0
for frame in origin_stack:
file_path, lineno, func_name, line_text = frame
call_traceback_proto.origin_stack.traces.add(
file_id=_string_to_id(file_path, string_to_id),
lineno=lineno,
function_id=_string_to_id(func_name, string_to_id),
line_id=_string_to_id(line_text, string_to_id))
id_to_string = call_traceback_proto.origin_id_to_string
for key, value in string_to_id.items():
id_to_string[value] = key if key is not None else ""
def _source_file_paths_outside_tensorflow_py_library(code_defs, id_to_string):
"""Extract source file paths outside TensorFlow Python library.
Args:
code_defs: An iterable of `CodeDef` protos, i.e., an iterable of stack
traces.
id_to_string: A proto map from integer ids to strings.
Returns:
An iterable of source file paths outside the TensorFlow Python library.
"""
file_ids = set()
for code_def in code_defs:
for trace in code_def.traces:
file_ids.add(trace.file_id)
non_tf_files = (id_to_string[file_id] for file_id in file_ids)
non_tf_files = (
f for f in non_tf_files
if not source_utils.guess_is_tensorflow_py_library(f) and gfile.Exists(f))
return non_tf_files
def _send_call_tracebacks(destinations,
origin_stack,
is_eager_execution=False,
call_key=None,
graph=None,
send_source=True):
"""Send the tracebacks of a TensorFlow execution call.
To gRPC debug server(s). This applies to graph execution (`tf.Session.run()`)
calls and eager execution calls.
If `send_source`, also sends the underlying source files outside the
TensorFlow library.
Args:
destinations: gRPC destination addresses, a `str` or a `list` of `str`s,
e.g., "localhost:4242". If a `list`, gRPC requests containing the same
`CallTraceback` proto payload will be sent to all the destinations.
origin_stack: The traceback stack for the origin of the execution call. For
graph execution, this is the traceback of the `tf.Session.run()`
invocation. For eager execution, this is the traceback of the Python
line that executes the eager operation.
is_eager_execution: (`bool`) whether an eager execution call (i.e., not a
`tf.Session.run` or derived methods) is being sent.
call_key: The key of the execution call, as a string. For graph execution,
this is a string describing the feeds, fetches (and targets) names of the
`tf.Session.run` call. For eager execution, this is ignored.
graph: A Python `tf.Graph` object (i.e., *not* a `tf.compat.v1.GraphDef`),
which contains op tracebacks, if applicable.
send_source: Whether the source files involved in the op tracebacks but
outside the TensorFlow library are to be sent.
"""
if not isinstance(destinations, list):
destinations = [destinations]
# Strip grpc:// prefix, if any is present.
destinations = [
dest[len(common.GRPC_URL_PREFIX):]
if dest.startswith(common.GRPC_URL_PREFIX) else dest
for dest in destinations]
call_type = (debug_service_pb2.CallTraceback.EAGER_EXECUTION
if is_eager_execution
else debug_service_pb2.CallTraceback.GRAPH_EXECUTION)
graph_traceback = tfprof_logger.merge_default_with_oplog(
graph, add_trainable_var=False) if graph else None
call_traceback = debug_service_pb2.CallTraceback(
call_type=call_type, call_key=call_key, graph_traceback=graph_traceback,
graph_version=graph.version if graph else None)
_format_origin_stack(origin_stack, call_traceback)
if send_source:
source_file_paths = set()
source_file_paths.update(_source_file_paths_outside_tensorflow_py_library(
(log_entry.code_def for log_entry
in call_traceback.graph_traceback.log_entries),
call_traceback.graph_traceback.id_to_string))
source_file_paths.update(_source_file_paths_outside_tensorflow_py_library(
[call_traceback.origin_stack], call_traceback.origin_id_to_string))
debugged_source_files = []
for file_path in source_file_paths:
source_files = debug_pb2.DebuggedSourceFiles()
_load_debugged_source_file(
file_path, source_files.source_files.add())
debugged_source_files.append(source_files)
for destination in destinations:
no_max_message_sizes = [("grpc.max_receive_message_length", -1),
("grpc.max_send_message_length", -1)]
channel = grpc.insecure_channel(destination, options=no_max_message_sizes)
stub = debug_service_pb2_grpc.EventListenerStub(channel)
stub.SendTracebacks(call_traceback)
if send_source:
for source_files in debugged_source_files:
stub.SendSourceFiles(source_files)
def send_graph_tracebacks(destinations,
run_key,
origin_stack,
graph,
send_source=True):
"""Send the tracebacks of a graph execution call to debug server(s).
Args:
destinations: gRPC destination addresses, a `str` or a `list` of `str`s,
e.g., "localhost:4242". If a `list`, gRPC requests containing the same
`CallTraceback` proto payload will be sent to all the destinations.
run_key: A string describing the feeds, fetches (and targets) names of the
`tf.Session.run` call.
origin_stack: The traceback of the `tf.Session.run()` invocation.
graph: A Python `tf.Graph` object (i.e., *not* a `tf.compat.v1.GraphDef`),
which contains op tracebacks.
send_source: Whether the source files involved in the op tracebacks but
outside the TensorFlow library are to be sent.
"""
_send_call_tracebacks(
destinations, origin_stack, is_eager_execution=False, call_key=run_key,
graph=graph, send_source=send_source)
def send_eager_tracebacks(destinations,
origin_stack,
send_source=True):
"""Send the tracebacks of an eager execution call to debug server(s).
Args:
destinations: gRPC destination addresses, a `str` or a `list` of `str`s,
e.g., "localhost:4242". If a `list`, gRPC requests containing the same
origin_stack: The traceback of the eager operation invocation.
send_source: Whether the source files involved in the op tracebacks but
outside the TensorFlow library are to be sent.
"""
_send_call_tracebacks(
destinations, origin_stack, is_eager_execution=True,
send_source=send_source)
tensorflow/python/debug/lib/source_remote_test.py
+195
View File
@@ -0,0 +1,195 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Unit tests for source_remote."""
import os
import traceback
import grpc
from tensorflow.core.debug import debug_service_pb2
from tensorflow.python.client import session
from tensorflow.python.debug.lib import grpc_debug_test_server
from tensorflow.python.debug.lib import source_remote
from tensorflow.python.debug.lib import source_utils
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import math_ops
# Import resource_variable_ops for the variables-to-tensor implicit conversion.
from tensorflow.python.ops import resource_variable_ops # pylint: disable=unused-import
from tensorflow.python.ops import variables
from tensorflow.python.platform import googletest
from tensorflow.python.platform import test
from tensorflow.python.util import tf_inspect
def line_number_above():
return tf_inspect.stack()[1][2] - 1
class SendTracebacksTest(test_util.TensorFlowTestCase):
@classmethod
def setUpClass(cls):
test_util.TensorFlowTestCase.setUpClass()
(cls._server_port, cls._debug_server_url, cls._server_dump_dir,
cls._server_thread,
cls._server) = grpc_debug_test_server.start_server_on_separate_thread(
poll_server=True)
cls._server_address = "localhost:%d" % cls._server_port
(cls._server_port_2, cls._debug_server_url_2, cls._server_dump_dir_2,
cls._server_thread_2,
cls._server_2) = grpc_debug_test_server.start_server_on_separate_thread()
cls._server_address_2 = "localhost:%d" % cls._server_port_2
cls._curr_file_path = os.path.normpath(os.path.abspath(__file__))
@classmethod
def tearDownClass(cls):
# Stop the test server and join the thread.
cls._server.stop_server().wait()
cls._server_thread.join()
cls._server_2.stop_server().wait()
cls._server_thread_2.join()
test_util.TensorFlowTestCase.tearDownClass()
def tearDown(self):
ops.reset_default_graph()
self._server.clear_data()
self._server_2.clear_data()
super(SendTracebacksTest, self).tearDown()
def _findFirstTraceInsideTensorFlowPyLibrary(self, op):
"""Find the first trace of an op that belongs to the TF Python library."""
for trace in op.traceback:
if source_utils.guess_is_tensorflow_py_library(trace.filename):
return trace
def testSendGraphTracebacksToSingleDebugServer(self):
this_func_name = "testSendGraphTracebacksToSingleDebugServer"
with session.Session() as sess:
a = variables.Variable(21.0, name="a")
a_lineno = line_number_above()
b = variables.Variable(2.0, name="b")
b_lineno = line_number_above()
math_ops.add(a, b, name="x")
x_lineno = line_number_above()
send_stack = traceback.extract_stack()
send_lineno = line_number_above()
source_remote.send_graph_tracebacks(
self._server_address, "dummy_run_key", send_stack, sess.graph)
tb = self._server.query_op_traceback("a")
self.assertIn((self._curr_file_path, a_lineno, this_func_name), tb)
tb = self._server.query_op_traceback("b")
self.assertIn((self._curr_file_path, b_lineno, this_func_name), tb)
tb = self._server.query_op_traceback("x")
self.assertIn((self._curr_file_path, x_lineno, this_func_name), tb)
self.assertIn(
(self._curr_file_path, send_lineno, this_func_name),
self._server.query_origin_stack()[-1])
self.assertEqual(
" a = variables.Variable(21.0, name=\"a\")",
self._server.query_source_file_line(__file__, a_lineno))
# Files in the TensorFlow code base shouldn not have been sent.
tf_trace = self._findFirstTraceInsideTensorFlowPyLibrary(a.op)
tf_trace_file_path = tf_trace.filename
with self.assertRaises(ValueError):
self._server.query_source_file_line(tf_trace_file_path, 0)
self.assertEqual([debug_service_pb2.CallTraceback.GRAPH_EXECUTION],
self._server.query_call_types())
self.assertEqual(["dummy_run_key"], self._server.query_call_keys())
self.assertEqual(
[sess.graph.version], self._server.query_graph_versions())
def testSendGraphTracebacksToTwoDebugServers(self):
this_func_name = "testSendGraphTracebacksToTwoDebugServers"
with session.Session() as sess:
a = variables.Variable(21.0, name="two/a")
a_lineno = line_number_above()
b = variables.Variable(2.0, name="two/b")
b_lineno = line_number_above()
x = math_ops.add(a, b, name="two/x")
x_lineno = line_number_above()
send_traceback = traceback.extract_stack()
send_lineno = line_number_above()
with test.mock.patch.object(
grpc, "insecure_channel",
wraps=grpc.insecure_channel) as mock_grpc_channel:
source_remote.send_graph_tracebacks(
[self._server_address, self._server_address_2],
"dummy_run_key", send_traceback, sess.graph)
mock_grpc_channel.assert_called_with(
test.mock.ANY,
options=[("grpc.max_receive_message_length", -1),
("grpc.max_send_message_length", -1)])
servers = [self._server, self._server_2]
for server in servers:
tb = server.query_op_traceback("two/a")
self.assertIn((self._curr_file_path, a_lineno, this_func_name), tb)
tb = server.query_op_traceback("two/b")
self.assertIn((self._curr_file_path, b_lineno, this_func_name), tb)
tb = server.query_op_traceback("two/x")
self.assertIn((self._curr_file_path, x_lineno, this_func_name), tb)
self.assertIn(
(self._curr_file_path, send_lineno, this_func_name),
server.query_origin_stack()[-1])
self.assertEqual(
" x = math_ops.add(a, b, name=\"two/x\")",
server.query_source_file_line(__file__, x_lineno))
tf_trace = self._findFirstTraceInsideTensorFlowPyLibrary(a.op)
tf_trace_file_path = tf_trace.filename
with self.assertRaises(ValueError):
server.query_source_file_line(tf_trace_file_path, 0)
self.assertEqual([debug_service_pb2.CallTraceback.GRAPH_EXECUTION],
server.query_call_types())
self.assertEqual(["dummy_run_key"], server.query_call_keys())
self.assertEqual([sess.graph.version], server.query_graph_versions())
def testSendEagerTracebacksToSingleDebugServer(self):
this_func_name = "testSendEagerTracebacksToSingleDebugServer"
send_traceback = traceback.extract_stack()
send_lineno = line_number_above()
source_remote.send_eager_tracebacks(self._server_address, send_traceback)
self.assertEqual([debug_service_pb2.CallTraceback.EAGER_EXECUTION],
self._server.query_call_types())
self.assertIn((self._curr_file_path, send_lineno, this_func_name),
self._server.query_origin_stack()[-1])
def testGRPCServerMessageSizeLimit(self):
"""Assert gRPC debug server is started with unlimited message size."""
with test.mock.patch.object(
grpc, "server", wraps=grpc.server) as mock_grpc_server:
(_, _, _, server_thread,
server) = grpc_debug_test_server.start_server_on_separate_thread(
poll_server=True)
mock_grpc_server.assert_called_with(
test.mock.ANY,
options=[("grpc.max_receive_message_length", -1),
("grpc.max_send_message_length", -1)])
server.stop_server().wait()
server_thread.join()
if __name__ == "__main__":
googletest.main()
tensorflow/python/debug/lib/source_utils.py
+375
View File
@@ -0,0 +1,375 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Classes and functions that help to inspect Python source w.r.t. TF graphs."""
import collections
import os
import re
import zipfile
from absl import app
import numpy as np
from tensorflow.python.debug.lib import profiling
_TENSORFLOW_BASEDIR = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.dirname(
os.path.normpath(os.path.abspath(__file__))))))
_ABSL_BASEDIR = os.path.dirname(app.__file__)
UNCOMPILED_SOURCE_SUFFIXES = (".py")
COMPILED_SOURCE_SUFFIXES = (".pyc", ".pyo")
def _norm_abs_path(file_path):
return os.path.normpath(os.path.abspath(file_path))
def is_extension_uncompiled_python_source(file_path):
_, extension = os.path.splitext(file_path)
return extension.lower() in UNCOMPILED_SOURCE_SUFFIXES
def is_extension_compiled_python_source(file_path):
_, extension = os.path.splitext(file_path)
return extension.lower() in COMPILED_SOURCE_SUFFIXES
def _convert_watch_key_to_tensor_name(watch_key):
return watch_key[:watch_key.rfind(":")]
def guess_is_tensorflow_py_library(py_file_path):
"""Guess whether a Python source file is a part of the tensorflow library.
Special cases:
1) Returns False for unit-test files in the library (*_test.py),
2) Returns False for files under python/debug/examples.
Args:
py_file_path: full path of the Python source file in question.
Returns:
(`bool`) Whether the file is inferred to be a part of the tensorflow
library.
"""
if (not is_extension_uncompiled_python_source(py_file_path) and
not is_extension_compiled_python_source(py_file_path)):
return False
py_file_path = _norm_abs_path(py_file_path)
return ((py_file_path.startswith(_TENSORFLOW_BASEDIR) or
py_file_path.startswith(_ABSL_BASEDIR)) and
not py_file_path.endswith("_test.py") and
(os.path.normpath("tensorflow/python/debug/examples") not in
os.path.normpath(py_file_path)))
def load_source(source_file_path):
"""Load the content of a Python source code file.
This function covers the following case:
1. source_file_path points to an existing Python (.py) file on the
file system.
2. source_file_path is a path within a .par file (i.e., a zip-compressed,
self-contained Python executable).
Args:
source_file_path: Path to the Python source file to read.
Returns:
A length-2 tuple:
- Lines of the source file, as a `list` of `str`s.
- The width of the string needed to show the line number in the file.
This is calculated based on the number of lines in the source file.
Raises:
IOError: if loading is unsuccessful.
"""
if os.path.isfile(source_file_path):
with open(source_file_path, "rb") as f:
source_text = f.read().decode("utf-8")
source_lines = source_text.split("\n")
else:
# One possible reason why the file doesn't exist is that it's a path
# inside a .par file. Try that possibility.
source_lines = _try_load_par_source(source_file_path)
if source_lines is None:
raise IOError(
"Source path neither exists nor can be loaded as a .par file: %s" %
source_file_path)
line_num_width = int(np.ceil(np.log10(len(source_lines)))) + 3
return source_lines, line_num_width
def _try_load_par_source(source_file_path):
"""Try loading the source code inside a .par file.
A .par file is a zip-compressed, self-contained Python executable.
It contains the content of individual Python source files that can
be read only through extracting from the zip file.
Args:
source_file_path: The full path to the file inside the .par file. This
path should include the path to the .par file itself, followed by the
intra-par path, e.g.,
"/tmp/my_executable.par/org-tensorflow/tensorflow/python/foo/bar.py".
Returns:
If successful, lines of the source file as a `list` of `str`s.
Else, `None`.
"""
prefix_path = source_file_path
while True:
prefix_path, basename = os.path.split(prefix_path)
if not basename:
break
suffix_path = os.path.normpath(
os.path.relpath(source_file_path, start=prefix_path))
if prefix_path.endswith(".par") and os.path.isfile(prefix_path):
with zipfile.ZipFile(prefix_path) as z:
norm_names = [os.path.normpath(name) for name in z.namelist()]
if suffix_path in norm_names:
with z.open(z.namelist()[norm_names.index(suffix_path)]) as zf:
source_text = zf.read().decode("utf-8")
return source_text.split("\n")
def annotate_source(dump,
source_file_path,
do_dumped_tensors=False,
file_stack_top=False,
min_line=None,
max_line=None):
"""Annotate a Python source file with a list of ops created at each line.
(The annotation doesn't change the source file itself.)
Args:
dump: (`DebugDumpDir`) A `DebugDumpDir` object of which the Python graph
has been loaded.
source_file_path: (`str`) Path to the source file being annotated.
do_dumped_tensors: (`str`) Whether dumped Tensors, instead of ops are to be
used to annotate the source file.
file_stack_top: (`bool`) Whether only the top stack trace in the
specified source file is to be annotated.
min_line: (`None` or `int`) The 1-based line to start annotate the source
file from (inclusive).
max_line: (`None` or `int`) The 1-based line number to end the annotation
at (exclusive).
Returns:
A `dict` mapping 1-based line number to a list of op name(s) created at
that line, or tensor names if `do_dumped_tensors` is True.
Raises:
ValueError: If the dump object does not have a Python graph set.
"""
py_graph = dump.python_graph
if not py_graph:
raise ValueError("Cannot perform source annotation due to a lack of set "
"Python graph in the dump object")
source_file_path = _norm_abs_path(source_file_path)
line_to_op_names = {}
for op in py_graph.get_operations():
for file_path, line_number, _, _ in reversed(dump.node_traceback(op.name)):
if (min_line is not None and line_number < min_line or
max_line is not None and line_number >= max_line):
continue
if _norm_abs_path(file_path) != source_file_path:
continue
if do_dumped_tensors:
watch_keys = dump.debug_watch_keys(op.name)
# Convert watch keys to unique Tensor names.
items_to_append = list(
set(map(_convert_watch_key_to_tensor_name, watch_keys)))
else:
items_to_append = [op.name]
if line_number in line_to_op_names:
line_to_op_names[line_number].extend(items_to_append)
else:
line_to_op_names[line_number] = items_to_append
if file_stack_top:
break
return line_to_op_names
def list_source_files_against_dump(dump,
path_regex_allowlist=None,
node_name_regex_allowlist=None):
"""Generate a list of source files with information regarding ops and tensors.
Args:
dump: (`DebugDumpDir`) A `DebugDumpDir` object of which the Python graph
has been loaded.
path_regex_allowlist: A regular-expression filter for source file path.
node_name_regex_allowlist: A regular-expression filter for node names.
Returns:
A list of tuples regarding the Python source files involved in constructing
the ops and tensors contained in `dump`. Each tuple is:
(source_file_path, is_tf_library, num_nodes, num_tensors, num_dumps,
first_line)
is_tf_library: (`bool`) A guess of whether the file belongs to the
TensorFlow Python library.
num_nodes: How many nodes were created by lines of this source file.
These include nodes with dumps and those without.
num_tensors: How many Tensors were created by lines of this source file.
These include Tensors with dumps and those without.
num_dumps: How many debug Tensor dumps were from nodes (and Tensors)
that were created by this source file.
first_line: The first line number (1-based) that created any nodes or
Tensors in this source file.
The list is sorted by ascending order of source_file_path.
Raises:
ValueError: If the dump object does not have a Python graph set.
"""
py_graph = dump.python_graph
if not py_graph:
raise ValueError("Cannot generate source list due to a lack of set "
"Python graph in the dump object")
path_to_node_names = collections.defaultdict(set)
path_to_tensor_names = collections.defaultdict(set)
path_to_first_line = {}
tensor_name_to_num_dumps = {}
path_regex = (
re.compile(path_regex_allowlist) if path_regex_allowlist else None)
node_name_regex = (
re.compile(node_name_regex_allowlist)
if node_name_regex_allowlist else None)
to_skip_file_paths = set()
for op in py_graph.get_operations():
if node_name_regex and not node_name_regex.match(op.name):
continue
for file_path, line_number, _, _ in dump.node_traceback(op.name):
file_path = _norm_abs_path(file_path)
if (file_path in to_skip_file_paths or
path_regex and not path_regex.match(file_path) or
not os.path.isfile(file_path)):
to_skip_file_paths.add(file_path)
continue
path_to_node_names[file_path].add(op.name)
if file_path in path_to_first_line:
if path_to_first_line[file_path] > line_number:
path_to_first_line[file_path] = line_number
else:
path_to_first_line[file_path] = line_number
for output_tensor in op.outputs:
tensor_name = output_tensor.name
path_to_tensor_names[file_path].add(tensor_name)
watch_keys = dump.debug_watch_keys(op.name)
for watch_key in watch_keys:
node_name, output_slot, debug_op = watch_key.split(":")
tensor_name = "%s:%s" % (node_name, output_slot)
if tensor_name not in tensor_name_to_num_dumps:
tensor_name_to_num_dumps[tensor_name] = len(
dump.get_tensors(node_name, int(output_slot), debug_op))
path_to_num_dumps = {}
for path in path_to_tensor_names:
path_to_num_dumps[path] = sum(
tensor_name_to_num_dumps.get(tensor_name, 0)
for tensor_name in path_to_tensor_names[path])
output = []
for file_path in path_to_node_names:
output.append((
file_path,
guess_is_tensorflow_py_library(file_path),
len(path_to_node_names.get(file_path, {})),
len(path_to_tensor_names.get(file_path, {})),
path_to_num_dumps.get(file_path, 0),
path_to_first_line[file_path]))
return sorted(output, key=lambda x: x[0])
def annotate_source_against_profile(profile_data,
source_file_path,
node_name_filter=None,
op_type_filter=None,
min_line=None,
max_line=None):
"""Annotate a Python source file with profiling information at each line.
(The annotation doesn't change the source file itself.)
Args:
profile_data: (`list` of `ProfileDatum`) A list of `ProfileDatum`.
source_file_path: (`str`) Path to the source file being annotated.
node_name_filter: Regular expression to filter by node name.
op_type_filter: Regular expression to filter by op type.
min_line: (`None` or `int`) The 1-based line to start annotate the source
file from (inclusive).
max_line: (`None` or `int`) The 1-based line number to end the annotation
at (exclusive).
Returns:
A `dict` mapping 1-based line number to a the namedtuple
`profiling.LineOrFuncProfileSummary`.
"""
source_file_path = _norm_abs_path(source_file_path)
node_name_regex = re.compile(node_name_filter) if node_name_filter else None
op_type_regex = re.compile(op_type_filter) if op_type_filter else None
line_to_profile_summary = {}
for profile_datum in profile_data:
if not profile_datum.file_path:
continue
if _norm_abs_path(profile_datum.file_path) != source_file_path:
continue
if (min_line is not None and profile_datum.line_number < min_line or
max_line is not None and profile_datum.line_number >= max_line):
continue
if (node_name_regex and
not node_name_regex.match(profile_datum.node_exec_stats.node_name)):
continue
if op_type_regex and not op_type_regex.match(profile_datum.op_type):
continue
if profile_datum.line_number not in line_to_profile_summary:
line_to_profile_summary[profile_datum.line_number] = (
profiling.AggregateProfile(profile_datum))
else:
line_to_profile_summary[profile_datum.line_number].add(profile_datum)
return line_to_profile_summary
tensorflow/python/debug/lib/source_utils_test.py
+447
View File
@@ -0,0 +1,447 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Unit tests for source_utils."""
import ast
import os
import sys
import tempfile
import zipfile
import numpy as np
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.client import session
from tensorflow.python.debug.lib import debug_data
from tensorflow.python.debug.lib import debug_utils
from tensorflow.python.debug.lib import source_utils
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.ops import math_ops
# Import resource_variable_ops for the variables-to-tensor implicit conversion.
from tensorflow.python.ops import resource_variable_ops # pylint: disable=unused-import
from tensorflow.python.ops import variables
from tensorflow.python.ops import while_loop
from tensorflow.python.platform import googletest
from tensorflow.python.util import tf_inspect
def line_number_above():
"""Get lineno of the AST node immediately above this function's call site.
It is assumed that there is no empty line(s) between the call site and the
preceding AST node.
Returns:
The lineno of the preceding AST node, at the same level of the AST.
If the preceding AST spans multiple lines:
- In Python 3.8+, the lineno of the first line is returned.
- In older Python versions, the lineno of the last line is returned.
"""
# https://bugs.python.org/issue12458: In Python 3.8, traceback started
# to return the lineno of the first line of a multi-line continuation block,
# instead of that of the last line. Therefore, in Python 3.8+, we use `ast` to
# get the lineno of the first line.
call_site_lineno = tf_inspect.stack()[1][2]
if sys.version_info < (3, 8):
return call_site_lineno - 1
else:
with open(__file__, "rb") as f:
source_text = f.read().decode("utf-8")
source_tree = ast.parse(source_text)
prev_node = _find_preceding_ast_node(source_tree, call_site_lineno)
return prev_node.lineno
def _find_preceding_ast_node(node, lineno):
"""Find the ast node immediately before and not including lineno."""
for i, child_node in enumerate(node.body):
if child_node.lineno == lineno:
return node.body[i - 1]
if hasattr(child_node, "body"):
found_node = _find_preceding_ast_node(child_node, lineno)
if found_node:
return found_node
class GuessIsTensorFlowLibraryTest(test_util.TensorFlowTestCase):
def setUp(self):
self.curr_file_path = os.path.normpath(os.path.abspath(__file__))
def tearDown(self):
ops.reset_default_graph()
def testGuessedBaseDirIsProbablyCorrect(self):
# In the non-pip world, code resides in "tensorflow/"
# In the pip world, after virtual pip, code resides in "tensorflow_core/"
# So, we have to check both of them
self.assertIn(
os.path.basename(source_utils._TENSORFLOW_BASEDIR),
["tensorflow", "tensorflow_core"])
def testUnitTestFileReturnsFalse(self):
self.assertFalse(
source_utils.guess_is_tensorflow_py_library(self.curr_file_path))
def testSourceUtilModuleReturnsTrue(self):
self.assertTrue(
source_utils.guess_is_tensorflow_py_library(source_utils.__file__))
@test_util.run_v1_only("Tensor.op is not available in TF 2.x")
def testFileInPythonKernelsPathReturnsTrue(self):
x = constant_op.constant(42.0, name="x")
self.assertTrue(
source_utils.guess_is_tensorflow_py_library(x.op.traceback[-1][0]))
def testDebuggerExampleFilePathReturnsFalse(self):
self.assertFalse(
source_utils.guess_is_tensorflow_py_library(os.path.normpath(
"site-packages/tensorflow/python/debug/examples/debug_mnist.py")))
self.assertFalse(
source_utils.guess_is_tensorflow_py_library(os.path.normpath(
"site-packages/tensorflow/python/debug/examples/v1/example_v1.py")))
self.assertFalse(
source_utils.guess_is_tensorflow_py_library(os.path.normpath(
"site-packages/tensorflow/python/debug/examples/v2/example_v2.py")))
self.assertFalse(
source_utils.guess_is_tensorflow_py_library(os.path.normpath(
"site-packages/tensorflow/python/debug/examples/v3/example_v3.py")))
def testReturnsFalseForNonPythonFile(self):
self.assertFalse(
source_utils.guess_is_tensorflow_py_library(
os.path.join(os.path.dirname(self.curr_file_path), "foo.cc")))
def testReturnsFalseForStdin(self):
self.assertFalse(source_utils.guess_is_tensorflow_py_library("<stdin>"))
def testReturnsFalseForEmptyFileName(self):
self.assertFalse(source_utils.guess_is_tensorflow_py_library(""))
class SourceHelperTest(test_util.TensorFlowTestCase):
def createAndRunGraphHelper(self):
"""Create and run a TensorFlow Graph to generate debug dumps.
This is intentionally done in separate method, to make it easier to test
the stack-top mode of source annotation.
"""
self.dump_root = self.get_temp_dir()
self.curr_file_path = os.path.abspath(
tf_inspect.getfile(tf_inspect.currentframe()))
# Run a simple TF graph to generate some debug dumps that can be used in
# source annotation.
with session.Session() as sess:
self.u_init = constant_op.constant(
np.array([[5.0, 3.0], [-1.0, 0.0]]), shape=[2, 2], name="u_init")
self.u_init_line_number = line_number_above()
self.u = variables.Variable(self.u_init, name="u")
self.u_line_number = line_number_above()
self.v_init = constant_op.constant(
np.array([[2.0], [-1.0]]), shape=[2, 1], name="v_init")
self.v_init_line_number = line_number_above()
self.v = variables.Variable(self.v_init, name="v")
self.v_line_number = line_number_above()
self.w = math_ops.matmul(self.u, self.v, name="w")
self.w_line_number = line_number_above()
self.evaluate(self.u.initializer)
self.evaluate(self.v.initializer)
run_options = config_pb2.RunOptions(output_partition_graphs=True)
debug_utils.watch_graph(
run_options, sess.graph, debug_urls=["file://%s" % self.dump_root])
run_metadata = config_pb2.RunMetadata()
sess.run(self.w, options=run_options, run_metadata=run_metadata)
self.dump = debug_data.DebugDumpDir(
self.dump_root, partition_graphs=run_metadata.partition_graphs)
self.dump.set_python_graph(sess.graph)
def setUp(self):
self.createAndRunGraphHelper()
self.helper_line_number = line_number_above()
def tearDown(self):
if os.path.isdir(self.dump_root):
file_io.delete_recursively(self.dump_root)
ops.reset_default_graph()
def testAnnotateWholeValidSourceFileGivesCorrectResult(self):
source_annotation = source_utils.annotate_source(self.dump,
self.curr_file_path)
self.assertIn(self.u_init.op.name,
source_annotation[self.u_init_line_number])
self.assertIn(self.u.op.name, source_annotation[self.u_line_number])
self.assertIn(self.v_init.op.name,
source_annotation[self.v_init_line_number])
self.assertIn(self.v.op.name, source_annotation[self.v_line_number])
self.assertIn(self.w.op.name, source_annotation[self.w_line_number])
# In the non-stack-top (default) mode, the helper line should be annotated
# with all the ops as well.
self.assertIn(self.u_init.op.name,
source_annotation[self.helper_line_number])
self.assertIn(self.u.op.name, source_annotation[self.helper_line_number])
self.assertIn(self.v_init.op.name,
source_annotation[self.helper_line_number])
self.assertIn(self.v.op.name, source_annotation[self.helper_line_number])
self.assertIn(self.w.op.name, source_annotation[self.helper_line_number])
def testAnnotateWithStackTopGivesCorrectResult(self):
source_annotation = source_utils.annotate_source(
self.dump, self.curr_file_path, file_stack_top=True)
self.assertIn(self.u_init.op.name,
source_annotation[self.u_init_line_number])
self.assertIn(self.u.op.name, source_annotation[self.u_line_number])
self.assertIn(self.v_init.op.name,
source_annotation[self.v_init_line_number])
self.assertIn(self.v.op.name, source_annotation[self.v_line_number])
self.assertIn(self.w.op.name, source_annotation[self.w_line_number])
# In the stack-top mode, the helper line should not have been annotated.
self.assertNotIn(self.helper_line_number, source_annotation)
def testAnnotateSubsetOfLinesGivesCorrectResult(self):
source_annotation = source_utils.annotate_source(
self.dump,
self.curr_file_path,
min_line=self.u_line_number,
max_line=self.u_line_number + 1)
self.assertIn(self.u.op.name, source_annotation[self.u_line_number])
self.assertNotIn(self.v_line_number, source_annotation)
def testAnnotateDumpedTensorsGivesCorrectResult(self):
source_annotation = source_utils.annotate_source(
self.dump, self.curr_file_path, do_dumped_tensors=True)
# Note: Constant Tensors u_init and v_init may not get dumped due to
# constant-folding.
self.assertIn(self.u.name, source_annotation[self.u_line_number])
self.assertIn(self.v.name, source_annotation[self.v_line_number])
self.assertIn(self.w.name, source_annotation[self.w_line_number])
self.assertNotIn(self.u.op.name, source_annotation[self.u_line_number])
self.assertNotIn(self.v.op.name, source_annotation[self.v_line_number])
self.assertNotIn(self.w.op.name, source_annotation[self.w_line_number])
self.assertIn(self.u.name, source_annotation[self.helper_line_number])
self.assertIn(self.v.name, source_annotation[self.helper_line_number])
self.assertIn(self.w.name, source_annotation[self.helper_line_number])
def testCallingAnnotateSourceWithoutPythonGraphRaisesException(self):
self.dump.set_python_graph(None)
with self.assertRaises(ValueError):
source_utils.annotate_source(self.dump, self.curr_file_path)
def testCallingAnnotateSourceOnUnrelatedSourceFileDoesNotError(self):
# Create an unrelated source file.
fd, unrelated_source_path = tempfile.mkstemp()
with open(fd, "wt") as source_file:
source_file.write("print('hello, world')\n")
self.assertEqual({},
source_utils.annotate_source(self.dump,
unrelated_source_path))
# Clean up unrelated source file.
os.remove(unrelated_source_path)
def testLoadingPythonSourceFileWithNonAsciiChars(self):
fd, source_path = tempfile.mkstemp()
with open(fd, "wb") as source_file:
source_file.write(u"print('\U0001f642')\n".encode("utf-8"))
source_lines, _ = source_utils.load_source(source_path)
self.assertEqual(source_lines, [u"print('\U0001f642')", u""])
# Clean up unrelated source file.
os.remove(source_path)
def testLoadNonexistentNonParPathFailsWithIOError(self):
bad_path = os.path.join(self.get_temp_dir(), "nonexistent.py")
with self.assertRaisesRegex(IOError,
"neither exists nor can be loaded.*par.*"):
source_utils.load_source(bad_path)
def testLoadingPythonSourceFileInParFileSucceeds(self):
# Create the .par file first.
temp_file_path = os.path.join(self.get_temp_dir(), "model.py")
with open(temp_file_path, "wb") as f:
f.write(b"import tensorflow as tf\nx = tf.constant(42.0)\n")
par_path = os.path.join(self.get_temp_dir(), "train_model.par")
with zipfile.ZipFile(par_path, "w") as zf:
zf.write(temp_file_path, os.path.join("tensorflow_models", "model.py"))
source_path = os.path.join(par_path, "tensorflow_models", "model.py")
source_lines, _ = source_utils.load_source(source_path)
self.assertEqual(
source_lines, ["import tensorflow as tf", "x = tf.constant(42.0)", ""])
def testLoadingPythonSourceFileInParFileFailsRaisingIOError(self):
# Create the .par file first.
temp_file_path = os.path.join(self.get_temp_dir(), "model.py")
with open(temp_file_path, "wb") as f:
f.write(b"import tensorflow as tf\nx = tf.constant(42.0)\n")
par_path = os.path.join(self.get_temp_dir(), "train_model.par")
with zipfile.ZipFile(par_path, "w") as zf:
zf.write(temp_file_path, os.path.join("tensorflow_models", "model.py"))
source_path = os.path.join(par_path, "tensorflow_models", "nonexistent.py")
with self.assertRaisesRegex(IOError,
"neither exists nor can be loaded.*par.*"):
source_utils.load_source(source_path)
@test_util.run_v1_only("Sessions are not available in TF 2.x")
class ListSourceAgainstDumpTest(test_util.TensorFlowTestCase):
def createAndRunGraphWithWhileLoop(self):
"""Create and run a TensorFlow Graph with a while loop to generate dumps."""
self.dump_root = self.get_temp_dir()
self.curr_file_path = os.path.abspath(
tf_inspect.getfile(tf_inspect.currentframe()))
# Run a simple TF graph to generate some debug dumps that can be used in
# source annotation.
with session.Session() as sess:
loop_body = lambda i: math_ops.add(i, 2)
self.traceback_first_line = line_number_above()
loop_cond = lambda i: math_ops.less(i, 16)
i = constant_op.constant(10, name="i")
loop = while_loop.while_loop(loop_cond, loop_body, [i])
run_options = config_pb2.RunOptions(output_partition_graphs=True)
debug_utils.watch_graph(
run_options, sess.graph, debug_urls=["file://%s" % self.dump_root])
run_metadata = config_pb2.RunMetadata()
sess.run(loop, options=run_options, run_metadata=run_metadata)
self.dump = debug_data.DebugDumpDir(
self.dump_root, partition_graphs=run_metadata.partition_graphs)
self.dump.set_python_graph(sess.graph)
def setUp(self):
self.createAndRunGraphWithWhileLoop()
def tearDown(self):
if os.path.isdir(self.dump_root):
file_io.delete_recursively(self.dump_root)
ops.reset_default_graph()
def testGenerateSourceList(self):
source_list = source_utils.list_source_files_against_dump(self.dump)
# Assert that the file paths are sorted and unique.
file_paths = [item[0] for item in source_list]
self.assertEqual(sorted(file_paths), file_paths)
self.assertEqual(len(set(file_paths)), len(file_paths))
# Assert that each item of source_list has length 6.
for item in source_list:
self.assertTrue(isinstance(item, tuple))
self.assertEqual(6, len(item))
# The while loop body should have executed 3 times. The following table
# lists the tensors and how many times each of them is dumped.
# Tensor name # of times dumped:
# i:0 1
# while/Enter:0 1
# while/Merge:0 4
# while/Merge:1 4
# while/Less/y:0 4
# while/Less:0 4
# while/LoopCond:0 4
# while/Switch:0 1
# while/Switch:1 3
# while/Identity:0 3
# while/Add/y:0 3
# while/Add:0 3
# while/NextIteration:0 3
# while/Exit:0 1
# ----------------------------
# (Total) 39
#
# The total number of nodes is 12.
# The total number of tensors is 14 (2 of the nodes have 2 outputs:
# while/Merge, while/Switch).
_, is_tf_py_library, num_nodes, num_tensors, num_dumps, first_line = (
source_list[file_paths.index(self.curr_file_path)])
self.assertFalse(is_tf_py_library)
self.assertEqual(12, num_nodes)
self.assertEqual(14, num_tensors)
self.assertEqual(39, num_dumps)
self.assertEqual(self.traceback_first_line, first_line)
def testGenerateSourceListWithNodeNameFilter(self):
source_list = source_utils.list_source_files_against_dump(
self.dump, node_name_regex_allowlist=r"while/Add.*")
# Assert that the file paths are sorted.
file_paths = [item[0] for item in source_list]
self.assertEqual(sorted(file_paths), file_paths)
self.assertEqual(len(set(file_paths)), len(file_paths))
# Assert that each item of source_list has length 4.
for item in source_list:
self.assertTrue(isinstance(item, tuple))
self.assertEqual(6, len(item))
# Due to the node-name filtering the result should only contain 2 nodes
# and 2 tensors. The total number of dumped tensors should be 6:
# while/Add/y:0 3
# while/Add:0 3
_, is_tf_py_library, num_nodes, num_tensors, num_dumps, _ = (
source_list[file_paths.index(self.curr_file_path)])
self.assertFalse(is_tf_py_library)
self.assertEqual(2, num_nodes)
self.assertEqual(2, num_tensors)
self.assertEqual(6, num_dumps)
def testGenerateSourceListWithPathRegexFilter(self):
curr_file_basename = os.path.basename(self.curr_file_path)
source_list = source_utils.list_source_files_against_dump(
self.dump,
path_regex_allowlist=(".*" + curr_file_basename.replace(".", "\\.") +
"$"))
self.assertEqual(1, len(source_list))
(file_path, is_tf_py_library, num_nodes, num_tensors, num_dumps,
first_line) = source_list[0]
self.assertEqual(self.curr_file_path, file_path)
self.assertFalse(is_tf_py_library)
self.assertEqual(12, num_nodes)
self.assertEqual(14, num_tensors)
self.assertEqual(39, num_dumps)
self.assertEqual(self.traceback_first_line, first_line)
if __name__ == "__main__":
googletest.main()