chore: import upstream snapshot with attribution

tensorflow/python/debug/wrappers/BUILD

@@ -0,0 +1,196 @@
+load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
+load("//tensorflow:tensorflow.bzl", "py_test")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
+    default_visibility = ["//tensorflow:internal"],
+    licenses = ["notice"],
+)
+
+py_library(
+    name = "framework",
+    srcs = ["framework.py"],
+    strict_deps = True,
+    deps = [
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/client:session",
+        "//tensorflow/python/debug/lib:debug_utils",
+        "//tensorflow/python/framework:errors",
+        "//tensorflow/python/framework:stack",
+        "//tensorflow/python/platform:tf_logging",
+        "//tensorflow/python/training:monitored_session",
+        "//tensorflow/python/util:compat",
+        "//tensorflow/python/util:nest",
+    ],
+)
+
+py_library(
+    name = "dumping_wrapper",
+    srcs = ["dumping_wrapper.py"],
+    strict_deps = True,
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":framework",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/debug/lib:debug_data",
+        "//tensorflow/python/platform:gfile",
+    ],
+)
+
+py_library(
+    name = "grpc_wrapper",
+    srcs = ["grpc_wrapper.py"],
+    strict_deps = True,
+    deps = [
+        ":framework",
+        "//tensorflow/python/debug/lib:common",
+        "//tensorflow/python/debug/lib:source_remote",
+    ],
+)
+
+py_library(
+    name = "local_cli_wrapper",
+    srcs = ["local_cli_wrapper.py"],
+    strict_deps = True,
+    deps = [
+        ":framework",
+        "//tensorflow/python/debug/cli:analyzer_cli",
+        "//tensorflow/python/debug/cli:cli_config",
+        "//tensorflow/python/debug/cli:cli_shared",
+        "//tensorflow/python/debug/cli:command_parser",
+        "//tensorflow/python/debug/cli:debugger_cli_common",
+        "//tensorflow/python/debug/cli:profile_analyzer_cli",
+        "//tensorflow/python/debug/cli:ui_factory",
+        "//tensorflow/python/debug/lib:common",
+        "//tensorflow/python/debug/lib:debug_data",
+        "//tensorflow/python/lib/io:file_io",
+    ],
+)
+
+py_library(
+    name = "hooks",
+    srcs = ["hooks.py"],
+    strict_deps = True,
+    visibility = [
+        "//tensorflow:internal",
+        "//third_party/py/tf_slim:__subpackages__",
+    ],
+    deps = [
+        ":dumping_wrapper",
+        ":framework",
+        ":grpc_wrapper",
+        ":local_cli_wrapper",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/debug/lib:debug_utils",
+        "//tensorflow/python/training:session_run_hook",
+    ],
+)
+
+py_test(
+    name = "framework_test",
+    size = "medium",
+    srcs = ["framework_test.py"],
+    strict_deps = True,
+    tags = [
+        "no_rocm",
+    ],
+    deps = [
+        ":framework",
+        # copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
+        "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/client:session",
+        "//tensorflow/python/debug/lib:debug_data",
+        "//tensorflow/python/framework:constant_op",
+        "//tensorflow/python/framework:errors",
+        "//tensorflow/python/framework:for_generated_wrappers",
+        "//tensorflow/python/framework:test_lib",
+        "//tensorflow/python/lib/io:file_io",
+        "//tensorflow/python/ops:array_ops",
+        "//tensorflow/python/ops:math_ops",
+        "//tensorflow/python/ops:resource_variable_ops",
+        "//tensorflow/python/ops:variables",
+        "//tensorflow/python/platform:test",
+        "//tensorflow/python/training:monitored_session",
+        "//tensorflow/python/util:tf_inspect",
+    ],
+)
+
+py_test(
+    name = "dumping_wrapper_test",
+    size = "small",
+    srcs = ["dumping_wrapper_test.py"],
+    strict_deps = True,
+    deps = [
+        ":dumping_wrapper",
+        ":framework",
+        ":hooks",
+        # copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
+        "//tensorflow/python/client:session",
+        "//tensorflow/python/debug/lib:debug_data",
+        "//tensorflow/python/framework:constant_op",
+        "//tensorflow/python/framework:for_generated_wrappers",
+        "//tensorflow/python/framework:test_lib",
+        "//tensorflow/python/lib/io:file_io",
+        "//tensorflow/python/ops:array_ops",
+        "//tensorflow/python/ops:state_ops",
+        "//tensorflow/python/ops:variable_v1",
+        "//tensorflow/python/platform:gfile",
+        "//tensorflow/python/platform:test",
+        "//tensorflow/python/training:monitored_session",
+    ],
+)
+
+py_test(
+    name = "local_cli_wrapper_test",
+    size = "small",
+    srcs = ["local_cli_wrapper_test.py"],
+    strict_deps = True,
+    deps = [
+        ":local_cli_wrapper",
+        # copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
+        "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/client:session",
+        "//tensorflow/python/debug/cli:cli_config",
+        "//tensorflow/python/debug/cli:cli_shared",
+        "//tensorflow/python/debug/cli:debugger_cli_common",
+        "//tensorflow/python/debug/cli:ui_factory",
+        "//tensorflow/python/framework:constant_op",
+        "//tensorflow/python/framework:errors",
+        "//tensorflow/python/framework:for_generated_wrappers",
+        "//tensorflow/python/framework:test_lib",
+        "//tensorflow/python/lib/io:file_io",
+        "//tensorflow/python/ops:array_ops",
+        "//tensorflow/python/ops:control_flow_ops",
+        "//tensorflow/python/ops:math_ops",
+        "//tensorflow/python/ops:resource_variable_ops",
+        "//tensorflow/python/ops:sparse_ops",
+        "//tensorflow/python/ops:state_ops",
+        "//tensorflow/python/ops:variable_v1",
+        "//tensorflow/python/ops:variables",
+        "//tensorflow/python/platform:test",
+        "//tensorflow/python/training:monitored_session",
+        "//tensorflow/python/training:session_run_hook",
+    ],
+)
+
+py_test(
+    name = "disk_usage_test",
+    size = "small",
+    srcs = ["disk_usage_test.py"],
+    strict_deps = True,
+    deps = [
+        ":dumping_wrapper",
+        ":hooks",
+        # copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
+        "//tensorflow/python/client:session",
+        "//tensorflow/python/framework:constant_op",
+        "//tensorflow/python/framework:dtypes",
+        "//tensorflow/python/framework:test_lib",
+        "//tensorflow/python/ops:state_ops",
+        "//tensorflow/python/ops:variables",
+        "//tensorflow/python/platform:test",
+        "//tensorflow/python/training:monitored_session",
+    ],
+)

tensorflow/python/debug/wrappers/disk_usage_test.py

@@ -0,0 +1,104 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Debugger Wrapper Session Consisting of a Local Curses-based CLI."""
+import os
+import tempfile
+
+from tensorflow.python.client import session
+from tensorflow.python.debug.wrappers import dumping_wrapper
+from tensorflow.python.debug.wrappers import hooks
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+from tensorflow.python.training import monitored_session
+
+
+@test_util.run_v1_only("Sessions are not available in TF 2.x")
+class DumpingDebugWrapperDiskUsageLimitTest(test_util.TensorFlowTestCase):
+
+  @classmethod
+  def setUpClass(cls):
+    # For efficient testing, set the disk usage bytes limit to a small
+    # number (10).
+    os.environ["TFDBG_DISK_BYTES_LIMIT"] = "10"
+
+  def setUp(self):
+    self.session_root = tempfile.mkdtemp()
+
+    self.v = variables.Variable(10.0, dtype=dtypes.float32, name="v")
+    self.delta = constant_op.constant(1.0, dtype=dtypes.float32, name="delta")
+    self.eta = constant_op.constant(-1.4, dtype=dtypes.float32, name="eta")
+    self.inc_v = state_ops.assign_add(self.v, self.delta, name="inc_v")
+    self.dec_v = state_ops.assign_add(self.v, self.eta, name="dec_v")
+
+    self.sess = session.Session()
+    self.sess.run(self.v.initializer)
+
+  def testWrapperSessionNotExceedingLimit(self):
+    def _watch_fn(fetches, feeds):
+      del fetches, feeds
+      return "DebugIdentity", r"(.*delta.*|.*inc_v.*)", r".*"
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess, session_root=self.session_root, watch_fn=_watch_fn)
+    sess.run(self.inc_v)
+
+  def testWrapperSessionExceedingLimit(self):
+    def _watch_fn(fetches, feeds):
+      del fetches, feeds
+      return "DebugIdentity", r".*delta.*", r".*"
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess, session_root=self.session_root, watch_fn=_watch_fn)
+    # Due to the watch function, each run should dump only 1 tensor,
+    # which has a size of 4 bytes, which corresponds to the dumped 'delta:0'
+    # tensor of scalar shape and float32 dtype.
+    # 1st run should pass, after which the disk usage is at 4 bytes.
+    sess.run(self.inc_v)
+    # 2nd run should also pass, after which 8 bytes are used.
+    sess.run(self.inc_v)
+    # 3rd run should fail, because the total byte count (12) exceeds the
+    # limit (10)
+    with self.assertRaises(ValueError):
+      sess.run(self.inc_v)
+
+  def testHookNotExceedingLimit(self):
+    def _watch_fn(fetches, feeds):
+      del fetches, feeds
+      return "DebugIdentity", r".*delta.*", r".*"
+    dumping_hook = hooks.DumpingDebugHook(
+        self.session_root, watch_fn=_watch_fn)
+    mon_sess = monitored_session._HookedSession(self.sess, [dumping_hook])
+    mon_sess.run(self.inc_v)
+
+  def testHookExceedingLimit(self):
+    def _watch_fn(fetches, feeds):
+      del fetches, feeds
+      return "DebugIdentity", r".*delta.*", r".*"
+    dumping_hook = hooks.DumpingDebugHook(
+        self.session_root, watch_fn=_watch_fn)
+    mon_sess = monitored_session._HookedSession(self.sess, [dumping_hook])
+    # Like in `testWrapperSessionExceedingLimit`, the first two calls
+    # should be within the byte limit, but the third one should error
+    # out due to exceeding the limit.
+    mon_sess.run(self.inc_v)
+    mon_sess.run(self.inc_v)
+    with self.assertRaises(ValueError):
+      mon_sess.run(self.inc_v)
+
+
+if __name__ == "__main__":
+  googletest.main()

tensorflow/python/debug/wrappers/dumping_wrapper.py

@@ -0,0 +1,124 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Debugger wrapper session that dumps debug data to file:// URLs."""
+import os
+import threading
+import time
+
+from tensorflow.core.util import event_pb2
+from tensorflow.python.debug.lib import debug_data
+from tensorflow.python.debug.wrappers import framework
+from tensorflow.python.platform import gfile
+
+
+class DumpingDebugWrapperSession(framework.NonInteractiveDebugWrapperSession):
+  """Debug Session wrapper that dumps debug data to filesystem."""
+
+  def __init__(self,
+               sess,
+               session_root,
+               watch_fn=None,
+               thread_name_filter=None,
+               pass_through_operrors=None):
+    """Constructor of DumpingDebugWrapperSession.
+
+    Args:
+      sess: The TensorFlow `Session` object being wrapped.
+      session_root: (`str`) Path to the session root directory. Must be a
+        directory that does not exist or an empty directory. If the directory
+        does not exist, it will be created by the debugger core during debug
+        `tf.Session.run`
+        calls.
+        As the `run()` calls occur, subdirectories will be added to
+        `session_root`. The subdirectories' names has the following pattern:
+          run_<epoch_time_stamp>_<zero_based_run_counter>
+        E.g., run_1480734393835964_ad4c953a85444900ae79fc1b652fb324
+      watch_fn: (`Callable`) A Callable that can be used to define per-run
+        debug ops and watched tensors. See the doc of
+        `NonInteractiveDebugWrapperSession.__init__()` for details.
+      thread_name_filter: Regular-expression white list for threads on which the
+        wrapper session will be active. See doc of `BaseDebugWrapperSession` for
+        more details.
+      pass_through_operrors: If true, all captured OpErrors will be
+        propagated. By default this captures all OpErrors.
+
+    Raises:
+       ValueError: If `session_root` is an existing and non-empty directory or
+       if `session_root` is a file.
+    """
+    framework.NonInteractiveDebugWrapperSession.__init__(
+        self, sess, watch_fn=watch_fn, thread_name_filter=thread_name_filter,
+        pass_through_operrors=pass_through_operrors)
+
+    session_root = os.path.expanduser(session_root)
+    if gfile.Exists(session_root):
+      if not gfile.IsDirectory(session_root):
+        raise ValueError(
+            "session_root path points to a file: %s" % session_root)
+      elif gfile.ListDirectory(session_root):
+        raise ValueError(
+            "session_root path points to a non-empty directory: %s" %
+            session_root)
+    else:
+      gfile.MakeDirs(session_root)
+    self._session_root = session_root
+
+    self._run_counter = 0
+    self._run_counter_lock = threading.Lock()
+
+  def prepare_run_debug_urls(self, fetches, feed_dict):
+    """Implementation of abstract method in superclass.
+
+    See doc of `NonInteractiveDebugWrapperSession.prepare_run_debug_urls()`
+    for details. This implementation creates a run-specific subdirectory under
+    self._session_root and stores information regarding run `fetches` and
+    `feed_dict.keys()` in the subdirectory.
+
+    Args:
+      fetches: Same as the `fetches` argument to `Session.run()`
+      feed_dict: Same as the `feed_dict` argument to `Session.run()`
+
+    Returns:
+      debug_urls: (`str` or `list` of `str`) file:// debug URLs to be used in
+        this `Session.run()` call.
+    """
+
+    # Add a UUID to accommodate the possibility of concurrent run() calls.
+    self._run_counter_lock.acquire()
+    run_dir = os.path.join(self._session_root, "run_%d_%d" %
+                           (int(time.time() * 1e6), self._run_counter))
+    self._run_counter += 1
+    self._run_counter_lock.release()
+    gfile.MkDir(run_dir)
+
+    fetches_event = event_pb2.Event()
+    fetches_event.log_message.message = repr(fetches)
+    fetches_path = os.path.join(
+        run_dir,
+        debug_data.METADATA_FILE_PREFIX + debug_data.FETCHES_INFO_FILE_TAG)
+    with gfile.Open(os.path.join(fetches_path), "wb") as f:
+      f.write(fetches_event.SerializeToString())
+
+    feed_keys_event = event_pb2.Event()
+    feed_keys_event.log_message.message = (repr(feed_dict.keys()) if feed_dict
+                                           else repr(feed_dict))
+
+    feed_keys_path = os.path.join(
+        run_dir,
+        debug_data.METADATA_FILE_PREFIX + debug_data.FEED_KEYS_INFO_FILE_TAG)
+    with gfile.Open(os.path.join(feed_keys_path), "wb") as f:
+      f.write(feed_keys_event.SerializeToString())
+
+    return ["file://" + run_dir]

tensorflow/python/debug/wrappers/dumping_wrapper_test.py

@@ -0,0 +1,380 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Unit Tests for classes in dumping_wrapper.py."""
+import glob
+import os
+import tempfile
+import threading
+
+from tensorflow.python.client import session
+from tensorflow.python.debug.lib import debug_data
+from tensorflow.python.debug.wrappers import dumping_wrapper
+from tensorflow.python.debug.wrappers import framework
+from tensorflow.python.debug.wrappers import hooks
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.lib.io import file_io
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_v1
+from tensorflow.python.platform import gfile
+from tensorflow.python.platform import googletest
+from tensorflow.python.training import monitored_session
+
+
+@test_util.run_v1_only("b/120545219")
+class DumpingDebugWrapperSessionTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    self.session_root = tempfile.mkdtemp()
+
+    self.v = variable_v1.VariableV1(10.0, dtype=dtypes.float32, name="v")
+    self.delta = constant_op.constant(1.0, dtype=dtypes.float32, name="delta")
+    self.eta = constant_op.constant(-1.4, dtype=dtypes.float32, name="eta")
+    self.inc_v = state_ops.assign_add(self.v, self.delta, name="inc_v")
+    self.dec_v = state_ops.assign_add(self.v, self.eta, name="dec_v")
+
+    self.ph = array_ops.placeholder(dtypes.float32, shape=(), name="ph")
+    self.inc_w_ph = state_ops.assign_add(self.v, self.ph, name="inc_w_ph")
+
+    self.sess = session.Session()
+    self.sess.run(self.v.initializer)
+
+  def tearDown(self):
+    ops.reset_default_graph()
+    if os.path.isdir(self.session_root):
+      file_io.delete_recursively(self.session_root)
+
+  def _assert_correct_run_subdir_naming(self, run_subdir):
+    self.assertStartsWith(run_subdir, "run_")
+    self.assertEqual(2, run_subdir.count("_"))
+    self.assertGreater(int(run_subdir.split("_")[1]), 0)
+
+  def testConstructWrapperWithExistingNonEmptyRootDirRaisesException(self):
+    dir_path = os.path.join(self.session_root, "foo")
+    os.mkdir(dir_path)
+    self.assertTrue(os.path.isdir(dir_path))
+
+    with self.assertRaisesRegex(
+        ValueError, "session_root path points to a non-empty directory"):
+      dumping_wrapper.DumpingDebugWrapperSession(
+          session.Session(), session_root=self.session_root)
+
+  def testConstructWrapperWithExistingFileDumpRootRaisesException(self):
+    file_path = os.path.join(self.session_root, "foo")
+    open(file_path, "a").close()  # Create the file
+    self.assertTrue(gfile.Exists(file_path))
+    self.assertFalse(gfile.IsDirectory(file_path))
+    with self.assertRaisesRegex(ValueError,
+                                "session_root path points to a file"):
+      dumping_wrapper.DumpingDebugWrapperSession(
+          session.Session(), session_root=file_path)
+
+  def testConstructWrapperWithNonexistentSessionRootCreatesDirectory(self):
+    new_dir_path = os.path.join(tempfile.mkdtemp(), "new_dir")
+    dumping_wrapper.DumpingDebugWrapperSession(
+        session.Session(), session_root=new_dir_path)
+    self.assertTrue(gfile.IsDirectory(new_dir_path))
+    # Cleanup.
+    gfile.DeleteRecursively(new_dir_path)
+
+  def testDumpingOnASingleRunWorks(self):
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess, session_root=self.session_root)
+    sess.run(self.inc_v)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    self.assertEqual(1, len(dump_dirs))
+
+    self._assert_correct_run_subdir_naming(os.path.basename(dump_dirs[0]))
+    dump = debug_data.DebugDumpDir(dump_dirs[0])
+    self.assertAllClose([10.0], dump.get_tensors("v", 0, "DebugIdentity"))
+
+    self.assertEqual(repr(self.inc_v), dump.run_fetches_info)
+    self.assertEqual(repr(None), dump.run_feed_keys_info)
+
+  def testDumpingOnASingleRunWorksWithRelativePathForDebugDumpDir(self):
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess, session_root=self.session_root)
+    sess.run(self.inc_v)
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    cwd = os.getcwd()
+    try:
+      os.chdir(self.session_root)
+      dump = debug_data.DebugDumpDir(
+          os.path.relpath(dump_dirs[0], self.session_root))
+      self.assertAllClose([10.0], dump.get_tensors("v", 0, "DebugIdentity"))
+    finally:
+      os.chdir(cwd)
+
+  def testDumpingOnASingleRunWithFeedDictWorks(self):
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess, session_root=self.session_root)
+    feed_dict = {self.ph: 3.2}
+    sess.run(self.inc_w_ph, feed_dict=feed_dict)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    self.assertEqual(1, len(dump_dirs))
+
+    self._assert_correct_run_subdir_naming(os.path.basename(dump_dirs[0]))
+    dump = debug_data.DebugDumpDir(dump_dirs[0])
+    self.assertAllClose([10.0], dump.get_tensors("v", 0, "DebugIdentity"))
+
+    self.assertEqual(repr(self.inc_w_ph), dump.run_fetches_info)
+    self.assertEqual(repr(feed_dict.keys()), dump.run_feed_keys_info)
+
+  def testDumpingOnMultipleRunsWorks(self):
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess, session_root=self.session_root)
+    for _ in range(3):
+      sess.run(self.inc_v)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    dump_dirs = sorted(
+        dump_dirs, key=lambda x: int(os.path.basename(x).split("_")[1]))
+    self.assertEqual(3, len(dump_dirs))
+    for i, dump_dir in enumerate(dump_dirs):
+      self._assert_correct_run_subdir_naming(os.path.basename(dump_dir))
+      dump = debug_data.DebugDumpDir(dump_dir)
+      self.assertAllClose([10.0 + 1.0 * i],
+                          dump.get_tensors("v", 0, "DebugIdentity"))
+      self.assertEqual(repr(self.inc_v), dump.run_fetches_info)
+      self.assertEqual(repr(None), dump.run_feed_keys_info)
+
+  def testUsingNonCallableAsWatchFnRaisesTypeError(self):
+    bad_watch_fn = "bad_watch_fn"
+    with self.assertRaisesRegex(TypeError, "watch_fn is not callable"):
+      dumping_wrapper.DumpingDebugWrapperSession(
+          self.sess,
+          session_root=self.session_root,
+          watch_fn=bad_watch_fn)
+
+  def testDumpingWithLegacyWatchFnOnFetchesWorks(self):
+    """Use a watch_fn that returns different allowlists for different runs."""
+
+    def watch_fn(fetches, feeds):
+      del feeds
+      # A watch_fn that picks fetch name.
+      if fetches.name == "inc_v:0":
+        # If inc_v, watch everything.
+        return "DebugIdentity", r".*", r".*"
+      else:
+        # If dec_v, watch nothing.
+        return "DebugIdentity", r"$^", r"$^"
+
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess,
+        session_root=self.session_root,
+        watch_fn=watch_fn)
+
+    for _ in range(3):
+      sess.run(self.inc_v)
+      sess.run(self.dec_v)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    dump_dirs = sorted(
+        dump_dirs, key=lambda x: int(os.path.basename(x).split("_")[1]))
+    self.assertEqual(6, len(dump_dirs))
+
+    for i, dump_dir in enumerate(dump_dirs):
+      self._assert_correct_run_subdir_naming(os.path.basename(dump_dir))
+      dump = debug_data.DebugDumpDir(dump_dir)
+      if i % 2 == 0:
+        self.assertGreater(dump.size, 0)
+        self.assertAllClose([10.0 - 0.4 * (i / 2)],
+                            dump.get_tensors("v", 0, "DebugIdentity"))
+        self.assertEqual(repr(self.inc_v), dump.run_fetches_info)
+        self.assertEqual(repr(None), dump.run_feed_keys_info)
+      else:
+        self.assertEqual(0, dump.size)
+        self.assertEqual(repr(self.dec_v), dump.run_fetches_info)
+        self.assertEqual(repr(None), dump.run_feed_keys_info)
+
+  def testDumpingWithLegacyWatchFnWithNonDefaultDebugOpsWorks(self):
+    """Use a watch_fn that specifies non-default debug ops."""
+
+    def watch_fn(fetches, feeds):
+      del fetches, feeds
+      return ["DebugIdentity", "DebugNumericSummary"], r".*", r".*"
+
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess,
+        session_root=self.session_root,
+        watch_fn=watch_fn)
+
+    sess.run(self.inc_v)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    self.assertEqual(1, len(dump_dirs))
+    dump = debug_data.DebugDumpDir(dump_dirs[0])
+
+    self.assertAllClose([10.0], dump.get_tensors("v", 0, "DebugIdentity"))
+    self.assertEqual(14,
+                     len(dump.get_tensors("v", 0, "DebugNumericSummary")[0]))
+
+  def testDumpingWithWatchFnWithNonDefaultDebugOpsWorks(self):
+    """Use a watch_fn that specifies non-default debug ops."""
+
+    def watch_fn(fetches, feeds):
+      del fetches, feeds
+      return framework.WatchOptions(
+          debug_ops=["DebugIdentity", "DebugNumericSummary"],
+          node_name_regex_allowlist=r"^v.*",
+          op_type_regex_allowlist=r".*",
+          tensor_dtype_regex_allowlist=".*_ref")
+
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess,
+        session_root=self.session_root,
+        watch_fn=watch_fn)
+
+    sess.run(self.inc_v)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    self.assertEqual(1, len(dump_dirs))
+    dump = debug_data.DebugDumpDir(dump_dirs[0])
+
+    self.assertAllClose([10.0], dump.get_tensors("v", 0, "DebugIdentity"))
+    self.assertEqual(14,
+                     len(dump.get_tensors("v", 0, "DebugNumericSummary")[0]))
+
+    dumped_nodes = [dump.node_name for dump in dump.dumped_tensor_data]
+    self.assertNotIn("inc_v", dumped_nodes)
+    self.assertNotIn("delta", dumped_nodes)
+
+  def testDumpingDebugHookWithoutWatchFnWorks(self):
+    dumping_hook = hooks.DumpingDebugHook(self.session_root)
+    mon_sess = monitored_session._HookedSession(self.sess, [dumping_hook])
+    mon_sess.run(self.inc_v)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    self.assertEqual(1, len(dump_dirs))
+
+    self._assert_correct_run_subdir_naming(os.path.basename(dump_dirs[0]))
+    dump = debug_data.DebugDumpDir(dump_dirs[0])
+    self.assertAllClose([10.0], dump.get_tensors("v", 0, "DebugIdentity"))
+
+    self.assertEqual(repr(self.inc_v), dump.run_fetches_info)
+    self.assertEqual(repr(None), dump.run_feed_keys_info)
+
+  def testDumpingDebugHookWithStatefulWatchFnWorks(self):
+    watch_fn_state = {"run_counter": 0}
+
+    def counting_watch_fn(fetches, feed_dict):
+      del fetches, feed_dict
+      watch_fn_state["run_counter"] += 1
+      if watch_fn_state["run_counter"] % 2 == 1:
+        # If odd-index run (1-based), watch every ref-type tensor.
+        return framework.WatchOptions(
+            debug_ops="DebugIdentity", tensor_dtype_regex_allowlist=".*_ref")
+      else:
+        # If even-index run, watch nothing.
+        return framework.WatchOptions(
+            debug_ops="DebugIdentity",
+            node_name_regex_allowlist=r"^$",
+            op_type_regex_allowlist=r"^$")
+
+    dumping_hook = hooks.DumpingDebugHook(
+        self.session_root, watch_fn=counting_watch_fn)
+    mon_sess = monitored_session._HookedSession(self.sess, [dumping_hook])
+    for _ in range(4):
+      mon_sess.run(self.inc_v)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    dump_dirs = sorted(
+        dump_dirs, key=lambda x: int(os.path.basename(x).split("_")[1]))
+    self.assertEqual(4, len(dump_dirs))
+
+    for i, dump_dir in enumerate(dump_dirs):
+      self._assert_correct_run_subdir_naming(os.path.basename(dump_dir))
+      dump = debug_data.DebugDumpDir(dump_dir)
+      if i % 2 == 0:
+        self.assertAllClose([10.0 + 1.0 * i],
+                            dump.get_tensors("v", 0, "DebugIdentity"))
+        self.assertNotIn("delta",
+                         [datum.node_name for datum in dump.dumped_tensor_data])
+      else:
+        self.assertEqual(0, dump.size)
+
+      self.assertEqual(repr(self.inc_v), dump.run_fetches_info)
+      self.assertEqual(repr(None), dump.run_feed_keys_info)
+
+  def testDumpingDebugHookWithStatefulLegacyWatchFnWorks(self):
+    watch_fn_state = {"run_counter": 0}
+
+    def counting_watch_fn(fetches, feed_dict):
+      del fetches, feed_dict
+      watch_fn_state["run_counter"] += 1
+      if watch_fn_state["run_counter"] % 2 == 1:
+        # If odd-index run (1-based), watch everything.
+        return "DebugIdentity", r".*", r".*"
+      else:
+        # If even-index run, watch nothing.
+        return "DebugIdentity", r"$^", r"$^"
+
+    dumping_hook = hooks.DumpingDebugHook(
+        self.session_root, watch_fn=counting_watch_fn)
+    mon_sess = monitored_session._HookedSession(self.sess, [dumping_hook])
+    for _ in range(4):
+      mon_sess.run(self.inc_v)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    dump_dirs = sorted(
+        dump_dirs, key=lambda x: int(os.path.basename(x).split("_")[1]))
+    self.assertEqual(4, len(dump_dirs))
+
+    for i, dump_dir in enumerate(dump_dirs):
+      self._assert_correct_run_subdir_naming(os.path.basename(dump_dir))
+      dump = debug_data.DebugDumpDir(dump_dir)
+      if i % 2 == 0:
+        self.assertAllClose([10.0 + 1.0 * i],
+                            dump.get_tensors("v", 0, "DebugIdentity"))
+      else:
+        self.assertEqual(0, dump.size)
+
+      self.assertEqual(repr(self.inc_v), dump.run_fetches_info)
+      self.assertEqual(repr(None), dump.run_feed_keys_info)
+
+  def testDumpingFromMultipleThreadsObeysThreadNameFilter(self):
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess, session_root=self.session_root,
+        thread_name_filter=r"MainThread$")
+
+    self.assertAllClose(1.0, sess.run(self.delta))
+    child_thread_result = []
+    def child_thread_job():
+      child_thread_result.append(sess.run(self.eta))
+
+    thread = threading.Thread(name="ChildThread", target=child_thread_job)
+    thread.start()
+    thread.join()
+    self.assertAllClose([-1.4], child_thread_result)
+
+    dump_dirs = glob.glob(os.path.join(self.session_root, "run_*"))
+    self.assertEqual(1, len(dump_dirs))
+    dump = debug_data.DebugDumpDir(dump_dirs[0])
+    self.assertEqual(1, dump.size)
+    self.assertEqual("delta", dump.dumped_tensor_data[0].node_name)
+
+  def testDumpingWrapperWithEmptyFetchWorks(self):
+    sess = dumping_wrapper.DumpingDebugWrapperSession(
+        self.sess, session_root=self.session_root)
+    sess.run([])
+
+
+if __name__ == "__main__":
+  googletest.main()

tensorflow/python/debug/wrappers/framework.py

@@ -0,0 +1,981 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Framework of debug wrapper sessions.
+
+A debug wrapper session is a wrapper around a TensorFlow Python Session.
+The wrapper preserves the Session interface, most importantly the run() method,
+while providing abilities to:
+a) Intercept a run() call to a wrapped session and insert debug tensor watches
+   according to externally-specified debug URLs.
+
+b) Release control to an external (i.e., non-Session) object before and after
+   the run() call, so that the external object can perform actions such as
+   launching a UI to let users inspect the intermediate tensors and partition
+   graphs from the run() call.
+
+c) (To be implemented in a future CL) Enter an instruction loop to let an
+   external object (e.g., remote client) launch run() and cont() calls
+   remotely.
+
+*** The lifetime of a debug wrapper session: ***
+
+1) The wrapper session is created by calling the constructor with a
+   wrapped (normal) session as the argument:
+     wrapper = FooDebugWrapperSession(sess)
+   wherein FooDebugWrapperSession is a concrete subclass implementing the
+   abstract BaseDebugWrapperSession class below.
+
+2) Near the end of the constructor call, the on_session_init() callback is
+   invoked, with a OnSessionInitRequest object as the argument. The object
+   carries the wrapped (normal) session object.
+
+3) The callback handles the request and returns a OnSessionInitResponse
+   object with an action field, directing the wrapper session what to do next.
+
+If the action field in the OnSessionInitResponse is PROCEED, the constructor
+returns. Control is released back to the caller of the constructor, which can
+invoke run() method of wrapper session with the same syntax as a non-wrapped
+session, e.g.,:
+  wrapper.run(fetches, feed_dict=feeds, options=run_options)
+
+Below, A1 - A2 is the lifetime of a wrapper run() call if the action is
+PROCEED:
+
+A1) Right at the start of each run() call, the on_run_start() callback is
+    invoked, with an OnRunStartRequest object carrying information such as
+    the fetches, the feed dict, the run options and run metadata used in
+    this run call, along with a count of how many run calls has occurred
+    on this wrapper session. The callback then returns an OnRunStartResponse
+    object, of which the action field directs what the wrapper session
+    actually will do of the run() call.
+
+    If the action is DEBUG_RUN, a debugged (tensor-watched) run will ensue,
+    with the debug URLs supplied in the debug_urls field of the response.
+    These can be file:// or grpc:// URLs, for example.
+
+    If the action is NON_DEBUG_RUN, a non-debug (normal) run will ensue.
+
+A2) Right before the run() returns, the on_run_end() callback is invoked,
+    with an OnRunEndRequest object as the argument, which carries information
+    including the actual action performed in the wrapper run() call and the
+    run_metadata from the run() call.
+
+However, if the action field in OnSessionInitResponse is
+REMOTE_INSTR_LOOP, the constructor will automatically invoke an instruction loop
+that gives the control to a remote caller.
+
+In the remote instruction loop, the following steps will happen:
+
+B1) Callback on_instr_start() is invoked. The callback will return an
+    OnInstrStartResponse object with an action field which can order one of
+    the following actions:
+        i) a run() call with fetches, feeds and debug_urls specified.
+       ii) exit the instruction loop.
+
+B2) The wrapper session carries out the action specified above.
+
+B3) If still in the instruction loop, the wrapper session invokes the
+    on_instr_end() callback. After the on_instr_end() callback returns, jump
+    back to B1.
+
+TODO(cais): Implemented the instruction loop in B1 - B3.
+
+"""
+
+import abc
+import re
+import threading
+
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.client import session
+from tensorflow.python.debug.lib import debug_utils
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import stack
+from tensorflow.python.platform import tf_logging
+from tensorflow.python.training import monitored_session
+from tensorflow.python.util import nest
+from tensorflow.python.util.compat import collections_abc
+
+
+# Helper function.
+def _check_type(obj, expected_types):
+  """Check if an object is of the expected type.
+
+  Args:
+    obj: The object being checked.
+    expected_types: (`type` or an iterable of `type`s) The expected `type`(s)
+      of obj.
+
+  Raises:
+      TypeError: If obj is not an instance of expected_type.
+  """
+  if not isinstance(obj, expected_types):
+    raise TypeError("Expected type %s; got type %s" %
+                    (expected_types, type(obj)))
+
+
+class OnSessionInitRequest:
+  """Request to an on-session-init callback.
+
+  This callback is invoked during the __init__ call to a debug-wrapper session.
+  """
+
+  def __init__(self, sess):
+    """Constructor.
+
+    Args:
+      sess: A tensorflow Session object.
+    """
+
+    _check_type(sess, (session.BaseSession, monitored_session.MonitoredSession))
+    self.session = sess
+
+
+class OnSessionInitAction:
+  """Enum-like values for possible action to take on session init."""
+
+  # Proceed, without special actions, in the wrapper session initialization.
+  # What action the wrapper session performs next is determined by the caller
+  # of the wrapper session. E.g., it can call run().
+  PROCEED = "proceed"
+
+  # Instead of letting the caller of the wrapper session determine what actions
+  # the wrapper session will perform next, enter a loop to receive instructions
+  # from a remote client.
+  # For example, TensorBoard visual debugger can use this action so that it can
+  # launch session.run() calls remotely.
+  REMOTE_INSTR_LOOP = "remote_instr_loop"
+
+
+class OnSessionInitResponse:
+  """Response from an on-session-init callback."""
+
+  def __init__(self, action):
+    """Constructor.
+
+    Args:
+      action: (`OnSessionInitAction`) Debugger action to take on session init.
+    """
+    _check_type(action, str)
+    self.action = action
+
+
+class OnRunStartRequest:
+  """Request to an on-run-start callback.
+
+  This callback is invoked during a run() call of the debug-wrapper
+  session, immediately after the run() call counter is incremented.
+  """
+
+  def __init__(self, fetches, feed_dict, run_options, run_metadata,
+               run_call_count, is_callable_runner=False):
+    """Constructor of `OnRunStartRequest`.
+
+    Args:
+      fetches: Fetch targets of the run() call.
+      feed_dict: The feed dictionary to the run() call.
+      run_options: RunOptions input to the run() call.
+      run_metadata: RunMetadata input to the run() call.
+        The above four arguments are identical to the input arguments to the
+        run() method of a non-wrapped TensorFlow session.
+      run_call_count: 1-based count of how many run calls (including this one)
+        has been invoked.
+      is_callable_runner: (bool) whether a runner returned by
+        Session.make_callable is being run.
+    """
+    self.fetches = fetches
+    self.feed_dict = feed_dict
+    self.run_options = run_options
+    self.run_metadata = run_metadata
+    self.run_call_count = run_call_count
+    self.is_callable_runner = is_callable_runner
+
+
+class OnRunStartAction:
+  """Enum-like values for possible action to take on start of a run() call."""
+
+  # Run once with debug tensor-watching.
+  DEBUG_RUN = "debug_run"
+
+  # Run once with profiler.
+  PROFILE_RUN = "profile_run"
+
+  # Run without debug tensor-watching.
+  NON_DEBUG_RUN = "non_debug_run"
+
+
+
+class OnRunStartResponse:
+  """Request from an on-run-start callback.
+
+  The caller of the callback can use this response object to specify what
+  action the debug-wrapper session actually takes on the run() call.
+  """
+
+  def __init__(self,
+               action,
+               debug_urls,
+               debug_ops="DebugIdentity",
+               node_name_regex_allowlist=None,
+               op_type_regex_allowlist=None,
+               tensor_dtype_regex_allowlist=None,
+               tolerate_debug_op_creation_failures=False):
+    """Constructor of `OnRunStartResponse`.
+
+    Args:
+      action: (`OnRunStartAction`) the action actually taken by the wrapped
+        session for the run() call.
+      debug_urls: (`list` of `str`) debug_urls used in watching the tensors
+        during the run() call.
+      debug_ops: (`str` or `list` of `str`) Debug op(s) to be used by the
+        debugger.
+      node_name_regex_allowlist: Regular-expression allowlist for node
+        name.
+      op_type_regex_allowlist: Regular-expression allowlist for op type.
+      tensor_dtype_regex_allowlist: Regular-expression allowlist for tensor
+        dtype.
+      tolerate_debug_op_creation_failures: Whether debug op creation failures
+        are to be tolerated.
+    """
+
+    _check_type(action, str)
+    self.action = action
+
+    _check_type(debug_urls, list)
+    self.debug_urls = debug_urls
+
+    self.debug_ops = debug_ops
+
+    self.node_name_regex_allowlist = node_name_regex_allowlist
+    self.op_type_regex_allowlist = op_type_regex_allowlist
+    self.tensor_dtype_regex_allowlist = tensor_dtype_regex_allowlist
+    self.tolerate_debug_op_creation_failures = (
+        tolerate_debug_op_creation_failures)
+
+
+class OnRunEndRequest:
+  """Request to an on-run-end callback.
+
+  The callback is invoked immediately before the wrapped run() call ends.
+  """
+
+  def __init__(self,
+               performed_action,
+               run_metadata=None,
+               client_graph_def=None,
+               tf_error=None):
+    """Constructor for `OnRunEndRequest`.
+
+    Args:
+      performed_action: (`OnRunStartAction`) Actually-performed action by the
+        debug-wrapper session.
+      run_metadata: run_metadata output from the run() call (if any).
+      client_graph_def: (GraphDef) GraphDef from the client side, i.e., from
+        the python front end of TensorFlow. Can be obtained with
+        session.graph.as_graph_def().
+      tf_error: (errors.OpError subtypes) TensorFlow OpError that occurred
+        during the run (if any).
+    """
+
+    _check_type(performed_action, str)
+    self.performed_action = performed_action
+
+    if run_metadata is not None:
+      _check_type(run_metadata, config_pb2.RunMetadata)
+    self.run_metadata = run_metadata
+    self.client_graph_def = client_graph_def
+    self.tf_error = tf_error
+
+
+class OnRunEndResponse:
+  """Response from an on-run-end callback."""
+
+  def __init__(self):
+
+    # Currently only a placeholder.
+    pass
+
+
+class BaseDebugWrapperSession(session.SessionInterface, metaclass=abc.ABCMeta):
+  """Base class of debug-wrapper session classes.
+
+  Concrete classes that inherit from this class need to implement the abstract
+  methods such as on_session_init, on_run_start and on_run_end.
+  """
+
+  def __init__(self, sess, thread_name_filter=None,
+               pass_through_operrors=False):
+    """Constructor of `BaseDebugWrapperSession`.
+
+    Args:
+      sess: An (unwrapped) TensorFlow session instance. It should be a subtype
+        of `BaseSession` or `tf.MonitoredSession`.
+      thread_name_filter: Regular-expression filter (allowlist) for name(s) of
+        thread(s) on which the wrapper session will be active. This regular
+        expression is used in a start-anchored fashion on the thread name, i.e.,
+        by applying the `match` method of the compiled pattern. The default
+        `None` means that the wrapper session will be active on all threads.
+        E.g., r"MainThread$", r"QueueRunnerThread.*".
+      pass_through_operrors: If True, all captured OpErrors will be
+        propagated.  By default this captures all OpErrors.
+
+    Raises:
+      ValueError: On invalid `OnSessionInitAction` value.
+      NotImplementedError: If a non-DirectSession sess object is received.
+    """
+
+    _check_type(sess, (session.BaseSession, monitored_session.MonitoredSession))
+
+    # The session being wrapped.
+    self._sess = sess
+    self._thread_name_filter_pattern = (re.compile(thread_name_filter)
+                                        if thread_name_filter else None)
+    # TODO(cais/kstevens): Unittest this pass through feature.
+    self._pass_through_operrors = pass_through_operrors
+
+    # Keeps track of number of run calls that have been performed on this
+    # debug-wrapper session. The count can be used for purposes such as
+    # displaying the state of the Session in a UI and determining a run
+    # number-dependent debug URL.
+    self._run_call_count = 0
+
+    # Invoke on-session-init callback.
+    response = self.on_session_init(OnSessionInitRequest(self._sess))
+    _check_type(response, OnSessionInitResponse)
+
+    if response.action == OnSessionInitAction.PROCEED:
+      pass
+    elif response.action == OnSessionInitAction.REMOTE_INSTR_LOOP:
+      # TODO(cais): Implement REMOTE_INSTR_LOOP
+      raise NotImplementedError(
+          "OnSessionInitAction REMOTE_INSTR_LOOP has not been "
+          "implemented.")
+    else:
+      raise ValueError(
+          "Invalid OnSessionInitAction value: %s" % response.action)
+
+    self._default_session_context_manager = None
+
+    # A cache for callables created from CallableOptions.
+    self._cached_callables_from_options = {}
+
+  @property
+  def graph(self):
+    return self._sess.graph
+
+  @property
+  def graph_def(self):
+    return self._sess.graph_def
+
+  @property
+  def sess_str(self):
+    return self._sess.sess_str
+
+  @property
+  def session(self):
+    return self._sess
+
+  def run(self,
+          fetches,
+          feed_dict=None,
+          options=None,
+          run_metadata=None,
+          callable_runner=None,
+          callable_runner_args=None,
+          callable_options=None):
+    """Wrapper around Session.run() that inserts tensor watch options.
+
+    Args:
+      fetches: Same as the `fetches` arg to regular `Session.run()`.
+      feed_dict: Same as the `feed_dict` arg to regular `Session.run()`.
+      options: Same as the `options` arg to regular `Session.run()`.
+      run_metadata: Same as the `run_metadata` arg to regular `Session.run()`.
+      callable_runner: A `callable` returned by `Session.make_callable()`.
+        If not `None`, `fetches` and `feed_dict` must both be `None`.
+        Mutually exclusive with `callable_options`.
+      callable_runner_args: An optional list of arguments to `callable_runner`
+        or for `callable_options`.
+      callable_options: An instance of `config_pb2.CallableOptions`, to be
+        used with `Session._make_callable_from_options()`. Mutually exclusive
+        with `callable_runner`.
+
+    Returns:
+      Simply forwards the output of the wrapped `Session.run()` call.
+
+    Raises:
+      ValueError: On invalid `OnRunStartAction` value. Or if `callable_runner`
+        is not `None` and either or both of `fetches` and `feed_dict` is `None`.
+    """
+    if callable_runner and callable_options:
+      raise ValueError(
+          "callable_runner and callable_options are mutually exclusive, but "
+          "are both specified in this call to BaseDebugWrapperSession.run().")
+
+    if callable_runner and (fetches or feed_dict):
+      raise ValueError(
+          "callable_runner and fetches/feed_dict are mutually exclusive, "
+          "but are used simultaneously.")
+    elif callable_options and (fetches or feed_dict):
+      raise ValueError(
+          "callable_options and fetches/feed_dict are mutually exclusive, "
+          "but are used simultaneously.")
+
+    self.increment_run_call_count()
+
+    def is_empty(x):
+      """Check whether a possibly nested structure is empty."""
+      if not nest.is_nested(x):
+        return False
+      if isinstance(x, collections_abc.Mapping):
+        return is_empty(list(x.values()))
+      for item in x:
+        if not is_empty(item):
+          return False
+      return True
+
+    empty_fetches = is_empty(fetches)
+    if empty_fetches:
+      tf_logging.info(
+          "Due to empty fetches, tfdbg Session wrapper is letting a "
+          "Session.run pass through without any debugging actions.")
+    if self._is_disabled_thread() or empty_fetches:
+      if callable_runner:
+        return callable_runner(*callable_runner_args)
+      elif callable_options:
+        # pylint:disable=protected-access
+        return self._sess._make_callable_from_options(
+            callable_options)(*callable_runner_args)
+        # pylint:enable=protected-access
+      else:
+        return self._sess.run(fetches,
+                              feed_dict=feed_dict,
+                              options=options,
+                              run_metadata=run_metadata)
+
+    # Invoke on-run-start callback and obtain response.
+    run_start_resp = self.on_run_start(
+        OnRunStartRequest(fetches, feed_dict, options, run_metadata,
+                          self._run_call_count,
+                          is_callable_runner=bool(callable_runner)))
+    _check_type(run_start_resp, OnRunStartResponse)
+
+    if run_start_resp.action == OnRunStartAction.DEBUG_RUN:
+      retvals, run_end_req = self._run_with_debugging(
+          run_start_resp, fetches, feed_dict, options, run_metadata,
+          callable_runner, callable_runner_args, callable_options)
+    elif run_start_resp.action == OnRunStartAction.PROFILE_RUN:
+      retvals, run_end_req = self._run_with_profiling(
+          run_start_resp, fetches, feed_dict, options, run_metadata,
+          callable_runner, callable_runner_args, callable_options)
+    elif run_start_resp.action == OnRunStartAction.NON_DEBUG_RUN:
+      # Invoke run() method of the wrapped session.
+      if callable_runner:
+        retvals = callable_runner(*callable_runner_args)
+      elif callable_options:
+        # pylint:disable=protected-access
+        callable_object = self._sess._make_callable_from_options(
+            callable_options)
+        # pylint:enable=protected-access
+        retvals = callable_object(*callable_runner_args)
+      else:
+        retvals = self._sess.run(
+            fetches,
+            feed_dict=feed_dict,
+            options=options,
+            run_metadata=run_metadata)
+
+      # Prepare arg for the on-run-end callback.
+      run_end_req = OnRunEndRequest(run_start_resp.action)
+    else:
+      raise ValueError(
+          "Invalid OnRunStartAction value: %s" % run_start_resp.action)
+
+    # Invoke on-run-end callback and obtain response.
+    run_end_resp = self.on_run_end(run_end_req)
+    _check_type(run_end_resp, OnRunEndResponse)
+    # Currently run_end_resp is only a placeholder. No action is taken on it.
+
+    return retvals
+
+  def _run_with_debugging(self,
+                          run_start_resp,
+                          fetches,
+                          feed_dict,
+                          options,
+                          run_metadata,
+                          callable_runner,
+                          callable_runner_args,
+                          callable_options):
+    """Perform a session.run() or callable with debugging."""
+    # Decorate RunOption to fill in debugger tensor watch specifications.
+    decorated_run_options = None
+    if callable_options:
+      callable_options_id = id(callable_options)
+      if callable_options_id not in self._cached_callables_from_options:
+        # Make a copy of callable_options to avoid mutating it.
+        new_callable_options = config_pb2.CallableOptions()
+        new_callable_options.CopyFrom(callable_options)
+        decorated_run_options = new_callable_options.run_options
+    else:
+      decorated_run_options = options or config_pb2.RunOptions()
+
+    run_metadata = run_metadata or config_pb2.RunMetadata()
+
+    if decorated_run_options:
+      self._decorate_run_options_for_debug(
+          decorated_run_options,
+          run_start_resp.debug_urls,
+          debug_ops=run_start_resp.debug_ops,
+          node_name_regex_allowlist=(run_start_resp.node_name_regex_allowlist),
+          op_type_regex_allowlist=run_start_resp.op_type_regex_allowlist,
+          tensor_dtype_regex_allowlist=(
+              run_start_resp.tensor_dtype_regex_allowlist),
+          tolerate_debug_op_creation_failures=(
+              run_start_resp.tolerate_debug_op_creation_failures))
+
+    # Invoke the run() method of the wrapped Session. Catch any TensorFlow
+    # runtime errors.
+    tf_error = None
+    try:
+      if callable_runner:
+        retvals = callable_runner(*callable_runner_args,
+                                  options=decorated_run_options,
+                                  run_metadata=run_metadata)
+      elif callable_options:
+        # pylint:disable=protected-access
+        if callable_options_id in self._cached_callables_from_options:
+          callable_object = self._cached_callables_from_options[
+              callable_options_id]
+        else:
+          callable_object = self._sess._make_callable_from_options(
+              new_callable_options)
+          self._cached_callables_from_options[
+              callable_options_id] = callable_object
+        # pylint:enable=protected-access
+        retvals = callable_object(
+            *callable_runner_args, run_metadata=run_metadata)
+      else:
+        retvals = self._sess.run(fetches,
+                                 feed_dict=feed_dict,
+                                 options=decorated_run_options,
+                                 run_metadata=run_metadata)
+    except errors.OpError as op_error:
+      if self._pass_through_operrors:
+        raise op_error
+      tf_error = op_error
+      retvals = op_error
+
+    return retvals, OnRunEndRequest(
+        run_start_resp.action,
+        run_metadata=run_metadata,
+        client_graph_def=self._sess.graph.as_graph_def(),
+        tf_error=tf_error)
+
+  def _run_with_profiling(self,
+                          run_start_resp,
+                          fetches,
+                          feed_dict,
+                          options,
+                          run_metadata,
+                          callable_runner,
+                          callable_runner_args,
+                          callable_options):
+    """Perform a session.run() or callable with profiling."""
+    # Decorate RunOption to fill in debugger tensor watch specifications.
+    decorated_run_options = None
+    if callable_options:
+      callable_options_id = id(callable_options)
+      if callable_options_id not in self._cached_callables_from_options:
+        # Make a copy of callable_options to avoid mutating it.
+        new_callable_options = config_pb2.CallableOptions()
+        new_callable_options.CopyFrom(callable_options)
+        decorated_run_options = new_callable_options.run_options
+    else:
+      decorated_run_options = options or config_pb2.RunOptions()
+    self._decorate_run_options_for_profile(decorated_run_options)
+
+    run_metadata = run_metadata or config_pb2.RunMetadata()
+    if callable_runner:
+      retvals = callable_runner(*callable_runner_args,
+                                options=decorated_run_options,
+                                run_metadata=run_metadata)
+    elif callable_options:
+      # pylint:disable=protected-access
+      callable_object = self._sess._make_callable_from_options(
+          new_callable_options)
+      # pylint:enable=protected-access
+      retvals = callable_object(
+          *callable_runner_args, run_metadata=run_metadata)
+    else:
+      retvals = self._sess.run(fetches,
+                               feed_dict=feed_dict,
+                               options=decorated_run_options,
+                               run_metadata=run_metadata)
+    return retvals, OnRunEndRequest(
+        run_start_resp.action,
+        run_metadata=run_metadata,
+        client_graph_def=self._sess.graph.as_graph_def())
+
+  def _is_disabled_thread(self):
+    thread_name = threading.current_thread().name or ""
+    return (self._thread_name_filter_pattern and
+            not self._thread_name_filter_pattern.match(thread_name))
+
+  def run_step_fn(self, step_fn):
+    return step_fn(
+        monitored_session.MonitoredSession.StepContext(self._sess, self.run))
+
+  def partial_run_setup(self, fetches, feeds=None):
+    """Sets up the feeds and fetches for partial runs in the session."""
+    raise NotImplementedError(
+        "partial_run_setup is not implemented for debug-wrapper sessions.")
+
+  def partial_run(self, handle, fetches, feed_dict=None):
+    raise NotImplementedError(
+        "partial_run is not implemented for debug-wrapper sessions.")
+
+  def list_devices(self, *args, **kwargs):
+    return self._sess.list_devices(*args, **kwargs)
+
+  def reset(self, *args, **kwargs):
+    return self._sess.reset(*args, **kwargs)
+
+  def make_callable(self,
+                    fetches,
+                    feed_list=None,
+                    accept_options=False):
+    runner = self._sess.make_callable(
+        fetches, feed_list=feed_list, accept_options=True)
+    def wrapped_runner(*runner_args, **kwargs):
+      return self.run(None,
+                      feed_dict=None,
+                      options=kwargs.get("options", None),
+                      run_metadata=kwargs.get("run_metadata", None),
+                      callable_runner=runner,
+                      callable_runner_args=runner_args)
+    return wrapped_runner
+
+  def _make_callable_from_options(self, callable_options):
+    def wrapped_runner(*feed_values, **kwargs):
+      return self.run(None,
+                      run_metadata=kwargs.get("run_metadata", None),
+                      callable_options=callable_options,
+                      callable_runner_args=feed_values)
+    return wrapped_runner
+
+  @property
+  def run_call_count(self):
+    return self._run_call_count
+
+  def increment_run_call_count(self):
+    self._run_call_count += 1
+
+  def _is_disk_usage_reset_each_run(self):
+    """Indicates whether disk usage is reset after each Session.run.
+
+    Subclasses that clean up the disk usage after every run should
+    override this protected method.
+
+    Returns:
+      (`bool`) Whether the disk usage amount is reset to zero after
+        each Session.run.
+    """
+    return False
+
+  def _decorate_run_options_for_debug(
+      self,
+      run_options,
+      debug_urls,
+      debug_ops="DebugIdentity",
+      node_name_regex_allowlist=None,
+      op_type_regex_allowlist=None,
+      tensor_dtype_regex_allowlist=None,
+      tolerate_debug_op_creation_failures=False):
+    """Modify a RunOptions object for debug tensor watching.
+
+    Specifies request for outputting partition graphs. Adds
+    debug_tensor_watch_opts with proper debug URLs.
+
+    Args:
+      run_options: (RunOptions) the modified RunOptions object.
+      debug_urls: (list of str) debug URLs to be entered in run_options.
+        debug_tensor_watch_opts.
+      debug_ops: (str or list of str) debug op(s) to be used by the debugger.
+      node_name_regex_allowlist: Regular-expression allowlist for node
+        name.
+      op_type_regex_allowlist: Regular-expression allowlist for op type.
+      tensor_dtype_regex_allowlist: Regular-expression allowlist for tensor
+        dtype.
+      tolerate_debug_op_creation_failures: Whether debug op creation failures
+        are to be tolerated.
+    """
+
+    run_options.output_partition_graphs = True
+    debug_utils.watch_graph(
+        run_options,
+        self._sess.graph,
+        debug_urls=debug_urls,
+        debug_ops=debug_ops,
+        node_name_regex_allowlist=node_name_regex_allowlist,
+        op_type_regex_allowlist=op_type_regex_allowlist,
+        tensor_dtype_regex_allowlist=tensor_dtype_regex_allowlist,
+        tolerate_debug_op_creation_failures=tolerate_debug_op_creation_failures,
+        reset_disk_byte_usage=(self._run_call_count == 1 or
+                               self._is_disk_usage_reset_each_run()))
+
+  def _decorate_run_options_for_profile(self, run_options):
+    """Modify a RunOptions object for profiling TensorFlow graph execution.
+
+    Args:
+      run_options: (RunOptions) the modified RunOptions object.
+    """
+
+    run_options.trace_level = config_pb2.RunOptions.FULL_TRACE
+
+  @abc.abstractmethod
+  def on_session_init(self, request):
+    """Callback invoked during construction of the debug-wrapper session.
+
+    This is a blocking callback.
+    The invocation happens right before the constructor ends.
+
+    Args:
+      request: (`OnSessionInitRequest`) callback request carrying information
+        such as the session being wrapped.
+
+    Returns:
+      An instance of `OnSessionInitResponse`.
+    """
+
+  @abc.abstractmethod
+  def on_run_start(self, request):
+    """Callback invoked on run() calls to the debug-wrapper session.
+
+    This is a blocking callback.
+    The invocation happens after the wrapper's run() call is entered,
+    after an increment of run call counter.
+
+    Args:
+      request: (`OnRunStartRequest`) callback request object carrying
+        information about the run call such as the fetches, feed dict, run
+        options, run metadata, and how many `run()` calls to this wrapper
+        session have occurred.
+
+    Returns:
+      An instance of `OnRunStartResponse`, carrying information to
+        debug URLs used to watch the tensors.
+    """
+
+  @abc.abstractmethod
+  def on_run_end(self, request):
+    """Callback invoked on run() calls to the debug-wrapper session.
+
+    This is a blocking callback.
+    The invocation happens right before the wrapper exits its run() call.
+
+    Args:
+      request: (`OnRunEndRequest`) callback request object carrying information
+        such as the actual action performed by the session wrapper for the
+        run() call.
+
+    Returns:
+      An instance of `OnRunStartResponse`.
+    """
+
+  def as_default(self):
+    return stack.default_session(self)
+
+  def __enter__(self):
+    if self._default_session_context_manager is None:
+      self._default_session_context_manager = self.as_default()
+    return self._default_session_context_manager.__enter__()
+
+  def __exit__(self, exec_type, exec_value, exec_tb):
+    self._default_session_context_manager.__exit__(
+        exec_type, exec_value, exec_tb)
+
+  def __del__(self):
+    if hasattr(self._sess, "__del__"):
+      self._sess.__del__()
+
+  def close(self):
+    self._sess.close()
+
+  # TODO(cais): Add _node_name_regex_allowlist and
+  #   _node_op_type_regex_allowlist.
+
+  def should_stop(self):
+    if hasattr(self._sess, "should_stop"):
+      return self._sess.should_stop()
+    else:
+      raise ValueError(
+          "The wrapped session %r does not have a method called 'should_stop'. "
+          "Do you intend to wrap a tf.MonitoredSession instead?" % self._sess)
+
+
+class WatchOptions:
+  """Type for return values of watch_fn."""
+
+  def __init__(self,
+               debug_ops=None,
+               node_name_regex_allowlist=None,
+               op_type_regex_allowlist=None,
+               tensor_dtype_regex_allowlist=None,
+               tolerate_debug_op_creation_failures=False):
+    """Constructor of WatchOptions: Debug watch options.
+
+    Used as return values of `watch_fn`s.
+
+    Args:
+      debug_ops: (`str` or `list of str`) Debug ops to be used.
+      node_name_regex_allowlist: Regular-expression allowlist for node_name,
+        e.g., `"(weight_[0-9]+|bias_.*)"`
+      op_type_regex_allowlist: Regular-expression allowlist for the op type of
+        nodes, e.g., `"(Variable|Add)"`.
+        If both `node_name_regex_allowlist` and `op_type_regex_allowlist`
+        are set, the two filtering operations will occur in a logical `AND`
+        relation. In other words, a node will be included if and only if it
+        hits both allowlists.
+      tensor_dtype_regex_allowlist: Regular-expression allowlist for Tensor
+        data type, e.g., `"^int.*"`.
+        This allowlist operates in logical `AND` relations to the two allowlists
+        above.
+      tolerate_debug_op_creation_failures: (`bool`) whether debug op creation
+        failures (e.g., due to dtype incompatibility) are to be tolerated by not
+        throwing exceptions.
+    """
+    if debug_ops:
+      self.debug_ops = debug_ops
+    else:
+      self.debug_ops = ["DebugIdentity"]
+    self.node_name_regex_allowlist = node_name_regex_allowlist
+    self.op_type_regex_allowlist = op_type_regex_allowlist
+    self.tensor_dtype_regex_allowlist = tensor_dtype_regex_allowlist
+    self.tolerate_debug_op_creation_failures = (
+        tolerate_debug_op_creation_failures)
+
+  def __repr__(self):
+    return ("WatchOptions(debug_ops=%r, node_name_regex_allowlist=%r, "
+            "op_type_regex_allowlist=%r, tensor_dtype_regex_allowlist=%r, "
+            "tolerate_debug_op_creation_failures=%r)" %
+            (self.debug_ops, self.node_name_regex_allowlist,
+             self.op_type_regex_allowlist, self.tensor_dtype_regex_allowlist,
+             self.tolerate_debug_op_creation_failures))
+
+
+class NonInteractiveDebugWrapperSession(BaseDebugWrapperSession):
+  """Base class for non-interactive (i.e., non-CLI) debug wrapper sessions."""
+
+  def __init__(self, sess, watch_fn=None, thread_name_filter=None,
+               pass_through_operrors=False):
+    """Constructor of NonInteractiveDebugWrapperSession.
+
+    Args:
+      sess: The TensorFlow `Session` object being wrapped.
+      watch_fn: (`Callable`) A Callable that maps the fetches and feeds of a
+        debugged `Session.run()` call to `WatchOptions.`
+        * Args:
+          * `fetches`: the fetches to the `Session.run()` call.
+          * `feeds`: the feeds to the `Session.run()` call.
+
+        * Returns:
+         (`tf_debug.WatchOptions`) An object containing debug options including
+           the debug ops to use, the node names, op types and/or tensor data
+           types to watch, etc. See the documentation of `tf_debug.WatchOptions`
+           for more details.
+      thread_name_filter: Regular-expression white list for threads on which the
+        wrapper session will be active. See doc of `BaseDebugWrapperSession` for
+        more details.
+      pass_through_operrors: If true, all captured OpErrors will be
+        propagated.  By default this captures all OpErrors.
+    Raises:
+       TypeError: If a non-None `watch_fn` is specified and it is not callable.
+    """
+
+    BaseDebugWrapperSession.__init__(
+        self, sess, thread_name_filter=thread_name_filter,
+        pass_through_operrors=pass_through_operrors)
+
+    self._watch_fn = None
+    if watch_fn is not None:
+      if not callable(watch_fn):
+        raise TypeError("watch_fn is not callable")
+      self._watch_fn = watch_fn
+
+  def on_session_init(self, request):
+    """See doc of BaseDebugWrapperSession.on_run_start."""
+
+    return OnSessionInitResponse(OnSessionInitAction.PROCEED)
+
+  @abc.abstractmethod
+  def prepare_run_debug_urls(self, fetches, feed_dict):
+    """Abstract method to be implemented by concrete subclasses.
+
+    This method prepares the run-specific debug URL(s).
+
+    Args:
+      fetches: Same as the `fetches` argument to `Session.run()`
+      feed_dict: Same as the `feed_dict` argument to `Session.run()`
+
+    Returns:
+      debug_urls: (`str` or `list` of `str`) Debug URLs to be used in
+        this `Session.run()` call.
+    """
+
+  def on_run_start(self, request):
+    """See doc of BaseDebugWrapperSession.on_run_start."""
+
+    debug_urls, watch_opts = self._prepare_run_watch_config(
+        request.fetches, request.feed_dict)
+
+    return OnRunStartResponse(
+        OnRunStartAction.DEBUG_RUN,
+        debug_urls,
+        debug_ops=watch_opts.debug_ops,
+        node_name_regex_allowlist=watch_opts.node_name_regex_allowlist,
+        op_type_regex_allowlist=watch_opts.op_type_regex_allowlist,
+        tensor_dtype_regex_allowlist=watch_opts.tensor_dtype_regex_allowlist,
+        tolerate_debug_op_creation_failures=(
+            watch_opts.tolerate_debug_op_creation_failures))
+
+  def _prepare_run_watch_config(self, fetches, feed_dict):
+    """Get the debug_urls, and node/op allowlists for the current run() call.
+
+    Args:
+      fetches: Same as the `fetches` argument to `Session.run()`.
+      feed_dict: Same as the `feed_dict argument` to `Session.run()`.
+
+    Returns:
+      debug_urls: (str or list of str) Debug URLs for the current run() call.
+        Currently, the list consists of only one URL that is a file:// URL.
+      watch_options: (WatchOptions) The return value of a watch_fn, containing
+        options including debug_ops, and allowlists.
+    """
+
+    debug_urls = self.prepare_run_debug_urls(fetches, feed_dict)
+    if self._watch_fn is None:
+      watch_options = WatchOptions()
+    else:
+      watch_options = self._watch_fn(fetches, feed_dict)
+      if isinstance(watch_options, tuple):
+        # For legacy return type (tuples).
+        watch_options = WatchOptions(*watch_options)
+
+    return debug_urls, watch_options
+
+  def on_run_end(self, request):
+    """See doc of BaseDebugWrapperSession.on_run_end."""
+
+    return OnRunEndResponse()

tensorflow/python/debug/wrappers/framework_test.py

@@ -0,0 +1,447 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Framework of debug-wrapped sessions."""
+import os
+import tempfile
+import threading
+
+import numpy as np
+
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.python.client import session
+from tensorflow.python.debug.lib import debug_data
+from tensorflow.python.debug.wrappers import framework
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.lib.io import file_io
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+# Import resource_variable_ops for the variables-to-tensor implicit conversion.
+from tensorflow.python.ops import resource_variable_ops  # pylint: disable=unused-import
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+from tensorflow.python.training import monitored_session
+from tensorflow.python.util import tf_inspect
+
+
+class TestDebugWrapperSession(framework.BaseDebugWrapperSession):
+  """A concrete implementation of BaseDebugWrapperSession for test."""
+
+  def __init__(self, sess, dump_root, observer, thread_name_filter=None):
+    # Supply dump root.
+    self._dump_root = dump_root
+
+    # Supply observer.
+    self._obs = observer
+
+    # Invoke superclass constructor.
+    framework.BaseDebugWrapperSession.__init__(
+        self, sess, thread_name_filter=thread_name_filter)
+
+  def on_session_init(self, request):
+    """Override abstract on-session-init callback method."""
+
+    self._obs["sess_init_count"] += 1
+    self._obs["request_sess"] = request.session
+
+    return framework.OnSessionInitResponse(
+        framework.OnSessionInitAction.PROCEED)
+
+  def on_run_start(self, request):
+    """Override abstract on-run-start callback method."""
+
+    self._obs["on_run_start_count"] += 1
+    self._obs["run_fetches"] = request.fetches
+    self._obs["run_feed_dict"] = request.feed_dict
+
+    return framework.OnRunStartResponse(
+        framework.OnRunStartAction.DEBUG_RUN,
+        ["file://" + self._dump_root])
+
+  def on_run_end(self, request):
+    """Override abstract on-run-end callback method."""
+
+    self._obs["on_run_end_count"] += 1
+    self._obs["performed_action"] = request.performed_action
+    self._obs["tf_error"] = request.tf_error
+
+    return framework.OnRunEndResponse()
+
+
+class TestDebugWrapperSessionBadAction(framework.BaseDebugWrapperSession):
+  """A concrete implementation of BaseDebugWrapperSession for test.
+
+  This class intentionally puts a bad action value in OnSessionInitResponse
+  and/or in OnRunStartAction to test the handling of such invalid cases.
+  """
+
+  def __init__(
+      self,
+      sess,
+      bad_init_action=None,
+      bad_run_start_action=None,
+      bad_debug_urls=None):
+    """Constructor.
+
+    Args:
+      sess: The TensorFlow Session object to be wrapped.
+      bad_init_action: (str) bad action value to be returned during the
+        on-session-init callback.
+      bad_run_start_action: (str) bad action value to be returned during the
+        the on-run-start callback.
+      bad_debug_urls: Bad URL values to be returned during the on-run-start
+        callback.
+    """
+
+    self._bad_init_action = bad_init_action
+    self._bad_run_start_action = bad_run_start_action
+    self._bad_debug_urls = bad_debug_urls
+
+    # Invoke superclass constructor.
+    framework.BaseDebugWrapperSession.__init__(self, sess)
+
+  def on_session_init(self, request):
+    if self._bad_init_action:
+      return framework.OnSessionInitResponse(self._bad_init_action)
+    else:
+      return framework.OnSessionInitResponse(
+          framework.OnSessionInitAction.PROCEED)
+
+  def on_run_start(self, request):
+    debug_urls = self._bad_debug_urls or []
+
+    if self._bad_run_start_action:
+      return framework.OnRunStartResponse(
+          self._bad_run_start_action, debug_urls)
+    else:
+      return framework.OnRunStartResponse(
+          framework.OnRunStartAction.DEBUG_RUN, debug_urls)
+
+  def on_run_end(self, request):
+    return framework.OnRunEndResponse()
+
+
+@test_util.run_v1_only("Sessions are not available in TF 2.x")
+class DebugWrapperSessionTest(test_util.TensorFlowTestCase):
+
+  def _no_rewrite_session_config(self):
+    rewriter_config = rewriter_config_pb2.RewriterConfig(
+        disable_model_pruning=True)
+    graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_config)
+    return config_pb2.ConfigProto(graph_options=graph_options)
+
+  def setUp(self):
+    self._observer = {
+        "sess_init_count": 0,
+        "request_sess": None,
+        "on_run_start_count": 0,
+        "run_fetches": None,
+        "run_feed_dict": None,
+        "on_run_end_count": 0,
+        "performed_action": None,
+        "tf_error": None,
+    }
+
+    self._dump_root = tempfile.mkdtemp()
+
+    self._sess = session.Session(config=self._no_rewrite_session_config())
+
+    self._a_init_val = np.array([[5.0, 3.0], [-1.0, 0.0]])
+    self._b_init_val = np.array([[2.0], [-1.0]])
+    self._c_val = np.array([[-4.0], [6.0]])
+
+    self._a_init = constant_op.constant(
+        self._a_init_val, shape=[2, 2], name="a_init")
+    self._b_init = constant_op.constant(
+        self._b_init_val, shape=[2, 1], name="b_init")
+
+    self._ph = array_ops.placeholder(dtype=dtypes.float64, name="ph")
+
+    self._a = variables.Variable(self._a_init, name="a1")
+    self._b = variables.Variable(self._b_init, name="b")
+    self._c = constant_op.constant(self._c_val, shape=[2, 1], name="c")
+
+    # Matrix product of a and b.
+    self._p = math_ops.matmul(self._a, self._b, name="p1")
+
+    # Matrix product of a and ph.
+    self._q = math_ops.matmul(self._a, self._ph, name="q")
+
+    # Sum of two vectors.
+    self._s = math_ops.add(self._p, self._c, name="s")
+
+    # Initialize the variables.
+    self._sess.run(self._a.initializer)
+    self._sess.run(self._b.initializer)
+
+  def tearDown(self):
+    # Tear down temporary dump directory.
+    if os.path.isdir(self._dump_root):
+      file_io.delete_recursively(self._dump_root)
+
+    ops.reset_default_graph()
+
+  def testSessionInit(self):
+    self.assertEqual(0, self._observer["sess_init_count"])
+
+    wrapper_sess = TestDebugWrapperSession(self._sess, self._dump_root,
+                                           self._observer)
+
+    # Assert that on-session-init callback is invoked.
+    self.assertEqual(1, self._observer["sess_init_count"])
+
+    # Assert that the request to the on-session-init callback carries the
+    # correct session object.
+    self.assertEqual(self._sess, self._observer["request_sess"])
+
+    # Verify that the wrapper session implements the session.SessionInterface.
+    self.assertTrue(isinstance(wrapper_sess, session.SessionInterface))
+    self.assertEqual(self._sess.sess_str, wrapper_sess.sess_str)
+    self.assertEqual(self._sess.graph, wrapper_sess.graph)
+    self.assertEqual(self._sess.graph_def, wrapper_sess.graph_def)
+
+    # Check that the partial_run_setup and partial_run are not implemented for
+    # the debug wrapper session.
+    with self.assertRaises(NotImplementedError):
+      wrapper_sess.partial_run_setup(self._p)
+
+  def testInteractiveSessionInit(self):
+    """The wrapper should work also on other subclasses of session.Session."""
+
+    TestDebugWrapperSession(
+        session.InteractiveSession(), self._dump_root, self._observer)
+
+  def testSessionRun(self):
+    wrapper = TestDebugWrapperSession(
+        self._sess, self._dump_root, self._observer)
+
+    # Check initial state of the observer.
+    self.assertEqual(0, self._observer["on_run_start_count"])
+    self.assertEqual(0, self._observer["on_run_end_count"])
+
+    s = wrapper.run(self._s)
+
+    # Assert the run return value is correct.
+    self.assertAllClose(np.array([[3.0], [4.0]]), s)
+
+    # Assert the on-run-start method is invoked.
+    self.assertEqual(1, self._observer["on_run_start_count"])
+
+    # Assert the on-run-start request reflects the correct fetch.
+    self.assertEqual(self._s, self._observer["run_fetches"])
+
+    # Assert the on-run-start request reflects the correct feed_dict.
+    self.assertIsNone(self._observer["run_feed_dict"])
+
+    # Assert the file debug URL has led to dump on the filesystem.
+    dump = debug_data.DebugDumpDir(self._dump_root)
+    self.assertEqual(7, len(dump.dumped_tensor_data))
+
+    # Assert the on-run-end method is invoked.
+    self.assertEqual(1, self._observer["on_run_end_count"])
+
+    # Assert the performed action field in the on-run-end callback request is
+    # correct.
+    self.assertEqual(
+        framework.OnRunStartAction.DEBUG_RUN,
+        self._observer["performed_action"])
+
+    # No TensorFlow runtime error should have happened.
+    self.assertIsNone(self._observer["tf_error"])
+
+  def testSessionInitInvalidSessionType(self):
+    """Attempt to wrap a non-Session-type object should cause an exception."""
+
+    wrapper = TestDebugWrapperSessionBadAction(self._sess)
+    with self.assertRaisesRegex(TypeError, "Expected type .*; got type .*"):
+      TestDebugWrapperSessionBadAction(wrapper)
+
+  def testSessionInitBadActionValue(self):
+    with self.assertRaisesRegex(
+        ValueError, "Invalid OnSessionInitAction value: nonsense_action"):
+      TestDebugWrapperSessionBadAction(
+          self._sess, bad_init_action="nonsense_action")
+
+  def testRunStartBadActionValue(self):
+    wrapper = TestDebugWrapperSessionBadAction(
+        self._sess, bad_run_start_action="nonsense_action")
+
+    with self.assertRaisesRegex(
+        ValueError, "Invalid OnRunStartAction value: nonsense_action"):
+      wrapper.run(self._s)
+
+  def testRunStartBadURLs(self):
+    # debug_urls ought to be a list of str, not a str. So an exception should
+    # be raised during a run() call.
+    wrapper = TestDebugWrapperSessionBadAction(
+        self._sess, bad_debug_urls="file://foo")
+
+    with self.assertRaisesRegex(TypeError, "Expected type .*; got type .*"):
+      wrapper.run(self._s)
+
+  def testErrorDuringRun(self):
+
+    wrapper = TestDebugWrapperSession(self._sess, self._dump_root,
+                                      self._observer)
+
+    # No matrix size mismatch.
+    self.assertAllClose(
+        np.array([[11.0], [-1.0]]),
+        wrapper.run(self._q, feed_dict={self._ph: np.array([[1.0], [2.0]])}))
+    self.assertEqual(1, self._observer["on_run_end_count"])
+    self.assertIsNone(self._observer["tf_error"])
+
+    # Now there should be a matrix size mismatch error.
+    wrapper.run(self._q, feed_dict={self._ph: np.array([[1.0], [2.0], [3.0]])})
+    self.assertEqual(2, self._observer["on_run_end_count"])
+    self.assertTrue(
+        isinstance(self._observer["tf_error"], errors.InvalidArgumentError))
+
+  def testUsingWrappedSessionShouldWorkAsContextManager(self):
+    wrapper = TestDebugWrapperSession(self._sess, self._dump_root,
+                                      self._observer)
+
+    with wrapper as sess:
+      self.assertAllClose([[3.0], [4.0]], self._s)
+      self.assertEqual(1, self._observer["on_run_start_count"])
+      self.assertEqual([self._s], self._observer["run_fetches"])
+      self.assertEqual(1, self._observer["on_run_end_count"])
+
+      self.assertAllClose(
+          [[11.0], [-1.0]],
+          sess.run(self._q, feed_dict={self._ph: np.array([[1.0], [2.0]])}))
+      self.assertEqual(2, self._observer["on_run_start_count"])
+      self.assertEqual(self._q, self._observer["run_fetches"])
+      self.assertEqual(2, self._observer["on_run_end_count"])
+
+  def testUsingWrappedSessionShouldSupportEvalWithAsDefault(self):
+    wrapper = TestDebugWrapperSession(self._sess, self._dump_root,
+                                      self._observer)
+
+    with wrapper.as_default():
+      foo = constant_op.constant(42, name="foo")
+      self.assertEqual(42, self.evaluate(foo))
+      self.assertEqual([foo], self._observer["run_fetches"])
+
+  def testWrapperShouldSupportSessionClose(self):
+    wrapper = TestDebugWrapperSession(self._sess, self._dump_root,
+                                      self._observer)
+    wrapper.close()
+
+  def testWrapperThreadNameFilterMainThread(self):
+    wrapper = TestDebugWrapperSession(
+        self._sess, self._dump_root, self._observer,
+        thread_name_filter="MainThread")
+
+    child_run_output = []
+    def child_thread_job():
+      child_run_output.append(wrapper.run(self._b_init))
+
+    thread = threading.Thread(name="ChildThread", target=child_thread_job)
+    thread.start()
+    self.assertAllClose(self._a_init_val, wrapper.run(self._a_init))
+    thread.join()
+    self.assertAllClose([self._b_init_val], child_run_output)
+
+    dump = debug_data.DebugDumpDir(self._dump_root)
+    self.assertEqual(1, dump.size)
+    self.assertEqual("a_init", dump.dumped_tensor_data[0].node_name)
+
+  def testWrapperThreadNameFilterChildThread(self):
+    wrapper = TestDebugWrapperSession(
+        self._sess, self._dump_root, self._observer,
+        thread_name_filter=r"Child.*")
+
+    child_run_output = []
+    def child_thread_job():
+      child_run_output.append(wrapper.run(self._b_init))
+
+    thread = threading.Thread(name="ChildThread", target=child_thread_job)
+    thread.start()
+    self.assertAllClose(self._a_init_val, wrapper.run(self._a_init))
+    thread.join()
+    self.assertAllClose([self._b_init_val], child_run_output)
+
+    dump = debug_data.DebugDumpDir(self._dump_root)
+    self.assertEqual(1, dump.size)
+    self.assertEqual("b_init", dump.dumped_tensor_data[0].node_name)
+
+  def testWrapperThreadNameFilterBothThreads(self):
+    wrapper = TestDebugWrapperSession(
+        self._sess, self._dump_root, self._observer,
+        thread_name_filter=None)
+
+    child_run_output = []
+    def child_thread_job():
+      child_run_output.append(wrapper.run(self._b_init))
+
+    thread = threading.Thread(name="ChildThread", target=child_thread_job)
+    thread.start()
+    self.assertAllClose(self._a_init_val, wrapper.run(self._a_init))
+    thread.join()
+    self.assertAllClose([self._b_init_val], child_run_output)
+
+    dump = debug_data.DebugDumpDir(self._dump_root, validate=False)
+    self.assertEqual(2, dump.size)
+    self.assertItemsEqual(
+        ["a_init", "b_init"],
+        [datum.node_name for datum in dump.dumped_tensor_data])
+
+
+def _is_public_method_name(method_name):
+  return (method_name.startswith("__") and method_name.endswith("__")
+          or not method_name.startswith("_"))
+
+
+class SessionWrapperPublicMethodParityTest(test_util.TensorFlowTestCase):
+
+  def testWrapperHasAllPublicMethodsOfSession(self):
+    session_public_methods = [
+        method_tuple[0] for method_tuple in
+        tf_inspect.getmembers(session.Session, predicate=tf_inspect.ismethod)
+        if _is_public_method_name(method_tuple[0])]
+    wrapper_public_methods = [
+        method_tuple[0] for method_tuple in
+        tf_inspect.getmembers(
+            framework.BaseDebugWrapperSession, predicate=tf_inspect.ismethod)
+        if _is_public_method_name(method_tuple[0])]
+    missing_public_methods = [
+        method for method in session_public_methods
+        if method not in wrapper_public_methods]
+    self.assertFalse(missing_public_methods)
+
+  def testWrapperHasAllPublicMethodsOfMonitoredSession(self):
+    session_public_methods = [
+        method_tuple[0] for method_tuple in
+        tf_inspect.getmembers(monitored_session.MonitoredSession,
+                              predicate=tf_inspect.ismethod)
+        if _is_public_method_name(method_tuple[0])]
+    wrapper_public_methods = [
+        method_tuple[0] for method_tuple in
+        tf_inspect.getmembers(
+            framework.BaseDebugWrapperSession, predicate=tf_inspect.ismethod)
+        if _is_public_method_name(method_tuple[0])]
+    missing_public_methods = [
+        method for method in session_public_methods
+        if method not in wrapper_public_methods]
+    self.assertFalse(missing_public_methods)
+
+
+if __name__ == "__main__":
+  googletest.main()

tensorflow/python/debug/wrappers/grpc_wrapper.py

@@ -0,0 +1,213 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Debugger wrapper session that sends debug data to file:// URLs."""
+import signal
+import sys
+import traceback
+
+from tensorflow.python.debug.lib import common
+from tensorflow.python.debug.wrappers import framework
+
+
+def publish_traceback(debug_server_urls,
+                      graph,
+                      feed_dict,
+                      fetches,
+                      old_graph_version):
+  """Publish traceback and source code if graph version is new.
+
+  `graph.version` is compared with `old_graph_version`. If the former is higher
+  (i.e., newer), the graph traceback and the associated source code is sent to
+  the debug server at the specified gRPC URLs.
+
+  Args:
+    debug_server_urls: A single gRPC debug server URL as a `str` or a `list` of
+      debug server URLs.
+    graph: A Python `tf.Graph` object.
+    feed_dict: Feed dictionary given to the `Session.run()` call.
+    fetches: Fetches from the `Session.run()` call.
+    old_graph_version: Old graph version to compare to.
+
+  Returns:
+    If `graph.version > old_graph_version`, the new graph version as an `int`.
+    Else, the `old_graph_version` is returned.
+  """
+  # TODO(cais): Consider moving this back to the top, after grpc becomes a
+  # pip dependency of tensorflow or tf_debug.
+  # pylint:disable=g-import-not-at-top
+  from tensorflow.python.debug.lib import source_remote
+  # pylint:enable=g-import-not-at-top
+  if graph.version > old_graph_version:
+    run_key = common.get_run_key(feed_dict, fetches)
+    source_remote.send_graph_tracebacks(
+        debug_server_urls, run_key, traceback.extract_stack(), graph,
+        send_source=True)
+    return graph.version
+  else:
+    return old_graph_version
+
+
+class GrpcDebugWrapperSession(framework.NonInteractiveDebugWrapperSession):
+  """Debug Session wrapper that send debug data to gRPC stream(s)."""
+
+  def __init__(self,
+               sess,
+               grpc_debug_server_addresses,
+               watch_fn=None,
+               thread_name_filter=None):
+    """Constructor of DumpingDebugWrapperSession.
+
+    Args:
+      sess: The TensorFlow `Session` object being wrapped.
+      grpc_debug_server_addresses: (`str` or `list` of `str`) Single or a list
+        of the gRPC debug server addresses, in the format of
+        <host:port>, with or without the "grpc://" prefix. For example:
+          "localhost:7000",
+          ["localhost:7000", "192.168.0.2:8000"]
+      watch_fn: (`Callable`) A Callable that can be used to define per-run
+        debug ops and watched tensors. See the doc of
+        `NonInteractiveDebugWrapperSession.__init__()` for details.
+      thread_name_filter: Regular-expression white list for threads on which the
+        wrapper session will be active. See doc of `BaseDebugWrapperSession` for
+        more details.
+
+    Raises:
+       TypeError: If `grpc_debug_server_addresses` is not a `str` or a `list`
+         of `str`.
+    """
+    framework.NonInteractiveDebugWrapperSession.__init__(
+        self, sess, watch_fn=watch_fn, thread_name_filter=thread_name_filter)
+
+    if isinstance(grpc_debug_server_addresses, str):
+      self._grpc_debug_server_urls = [
+          self._normalize_grpc_url(grpc_debug_server_addresses)]
+    elif isinstance(grpc_debug_server_addresses, list):
+      self._grpc_debug_server_urls = []
+      for address in grpc_debug_server_addresses:
+        if not isinstance(address, str):
+          raise TypeError(
+              "Expected type str in list grpc_debug_server_addresses, "
+              "received type %s" % type(address))
+        self._grpc_debug_server_urls.append(self._normalize_grpc_url(address))
+    else:
+      raise TypeError(
+          "Expected type str or list in grpc_debug_server_addresses, "
+          "received type %s" % type(grpc_debug_server_addresses))
+
+  def prepare_run_debug_urls(self, fetches, feed_dict):
+    """Implementation of abstract method in superclass.
+
+    See doc of `NonInteractiveDebugWrapperSession.prepare_run_debug_urls()`
+    for details.
+
+    Args:
+      fetches: Same as the `fetches` argument to `Session.run()`
+      feed_dict: Same as the `feed_dict` argument to `Session.run()`
+
+    Returns:
+      debug_urls: (`str` or `list` of `str`) file:// debug URLs to be used in
+        this `Session.run()` call.
+    """
+
+    return self._grpc_debug_server_urls
+
+  def _normalize_grpc_url(self, address):
+    return (common.GRPC_URL_PREFIX + address
+            if not address.startswith(common.GRPC_URL_PREFIX) else address)
+
+
+def _signal_handler(unused_signal, unused_frame):
+  while True:
+    response = input("\nSIGINT received. Quit program? (Y/n): ").strip()
+    if response in ("", "Y", "y"):
+      sys.exit(0)
+    elif response in ("N", "n"):
+      break
+
+
+def register_signal_handler():
+  try:
+    signal.signal(signal.SIGINT, _signal_handler)
+  except ValueError:
+    # This can happen if we are not in the MainThread.
+    pass
+
+
+class TensorBoardDebugWrapperSession(GrpcDebugWrapperSession):
+  """A tfdbg Session wrapper that can be used with TensorBoard Debugger Plugin.
+
+  This wrapper is the same as `GrpcDebugWrapperSession`, except that it uses a
+    predefined `watch_fn` that
+    1) uses `DebugIdentity` debug ops with the `gated_grpc` attribute set to
+        `True` to allow the interactive enabling and disabling of tensor
+       breakpoints.
+    2) watches all tensors in the graph.
+  This saves the need for the user to define a `watch_fn`.
+  """
+
+  def __init__(self,
+               sess,
+               grpc_debug_server_addresses,
+               thread_name_filter=None,
+               send_traceback_and_source_code=True):
+    """Constructor of TensorBoardDebugWrapperSession.
+
+    Args:
+      sess: The `tf.compat.v1.Session` instance to be wrapped.
+      grpc_debug_server_addresses: gRPC address(es) of debug server(s), as a
+        `str` or a `list` of `str`s. E.g., "localhost:2333",
+        "grpc://localhost:2333", ["192.168.0.7:2333", "192.168.0.8:2333"].
+      thread_name_filter: Optional filter for thread names.
+      send_traceback_and_source_code: Whether traceback of graph elements and
+        the source code are to be sent to the debug server(s).
+    """
+    def _gated_grpc_watch_fn(fetches, feeds):
+      del fetches, feeds  # Unused.
+      return framework.WatchOptions(
+          debug_ops=["DebugIdentity(gated_grpc=true)"])
+
+    super().__init__(
+        sess,
+        grpc_debug_server_addresses,
+        watch_fn=_gated_grpc_watch_fn,
+        thread_name_filter=thread_name_filter)
+
+    self._send_traceback_and_source_code = send_traceback_and_source_code
+    # Keeps track of the latest version of Python graph object that has been
+    # sent to the debug servers.
+    self._sent_graph_version = -1
+
+    register_signal_handler()
+
+  def run(self,
+          fetches,
+          feed_dict=None,
+          options=None,
+          run_metadata=None,
+          callable_runner=None,
+          callable_runner_args=None,
+          callable_options=None):
+    if self._send_traceback_and_source_code:
+      self._sent_graph_version = publish_traceback(
+          self._grpc_debug_server_urls, self.graph, feed_dict, fetches,
+          self._sent_graph_version)
+    return super().run(
+        fetches,
+        feed_dict=feed_dict,
+        options=options,
+        run_metadata=run_metadata,
+        callable_runner=callable_runner,
+        callable_runner_args=callable_runner_args,
+        callable_options=callable_options)

tensorflow/python/debug/wrappers/hooks.py

@@ -0,0 +1,336 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""tfdbg CLI as SessionRunHook."""
+
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.debug.lib import debug_utils
+from tensorflow.python.debug.wrappers import dumping_wrapper
+from tensorflow.python.debug.wrappers import framework
+from tensorflow.python.debug.wrappers import grpc_wrapper
+from tensorflow.python.debug.wrappers import local_cli_wrapper
+from tensorflow.python.training import session_run_hook
+
+
+class LocalCLIDebugHook(session_run_hook.SessionRunHook):
+  """Command-line-interface debugger hook.
+
+  Can be used as a hook for `tf.compat.v1.train.MonitoredSession`.
+  """
+
+  def __init__(self,
+               ui_type="readline",
+               dump_root=None,
+               thread_name_filter=None,
+               config_file_path=None):
+    """Create a local debugger command-line interface (CLI) hook.
+
+    Args:
+      ui_type: (`str`) requested user-interface type. Currently supported:
+        (readline).
+      dump_root: (`str`) optional path to the dump root directory. Must be a
+        directory that does not exist or an empty directory. If the directory
+        does not exist, it will be created by the debugger core during debug
+        `run()` calls and removed afterwards.
+      thread_name_filter: Regular-expression white list for threads on which the
+        wrapper session will be active. See doc of `BaseDebugWrapperSession` for
+        more details.
+      config_file_path: Optional override to the default configuration file
+        path, which is at `${HOME}/.tfdbg_config`.
+    """
+
+    self._ui_type = ui_type
+    self._dump_root = dump_root
+    self._thread_name_filter = thread_name_filter
+    self._session_wrapper = None
+    self._pending_tensor_filters = {}
+    self._config_file_path = config_file_path
+
+  def add_tensor_filter(self, filter_name, tensor_filter):
+    """Add a tensor filter.
+
+    See doc of `LocalCLIDebugWrapperSession.add_tensor_filter()` for details.
+    Override default behavior to accommodate the possibility of this method
+    being
+    called prior to the initialization of the underlying
+    `LocalCLIDebugWrapperSession` object.
+
+    Args:
+      filter_name: See doc of `LocalCLIDebugWrapperSession.add_tensor_filter()`
+        for details.
+      tensor_filter: See doc of
+        `LocalCLIDebugWrapperSession.add_tensor_filter()` for details.
+    """
+
+    if self._session_wrapper:
+      self._session_wrapper.add_tensor_filter(filter_name, tensor_filter)
+    else:
+      self._pending_tensor_filters[filter_name] = tensor_filter
+
+  def begin(self):
+    pass
+
+  def before_run(self, run_context):
+    if not self._session_wrapper:
+      self._session_wrapper = local_cli_wrapper.LocalCLIDebugWrapperSession(
+          run_context.session,
+          ui_type=self._ui_type,
+          dump_root=self._dump_root,
+          thread_name_filter=self._thread_name_filter,
+          config_file_path=self._config_file_path)
+
+      # Actually register tensor filters registered prior to the construction
+      # of the underlying LocalCLIDebugWrapperSession object.
+      for filter_name in self._pending_tensor_filters:
+        self._session_wrapper.add_tensor_filter(
+            filter_name, self._pending_tensor_filters[filter_name])
+
+    # Increment run call counter.
+    self._session_wrapper.increment_run_call_count()
+
+    # Adapt run_context to an instance of OnRunStartRequest for invoking
+    # superclass on_run_start().
+    on_run_start_request = framework.OnRunStartRequest(
+        run_context.original_args.fetches, run_context.original_args.feed_dict,
+        None, None, self._session_wrapper.run_call_count)
+
+    on_run_start_response = self._session_wrapper.on_run_start(
+        on_run_start_request)
+    self._performed_action = on_run_start_response.action
+
+    run_args = session_run_hook.SessionRunArgs(
+        None, feed_dict=None, options=config_pb2.RunOptions())
+    if self._performed_action == framework.OnRunStartAction.DEBUG_RUN:
+      # pylint: disable=protected-access
+      self._session_wrapper._decorate_run_options_for_debug(
+          run_args.options,
+          on_run_start_response.debug_urls,
+          debug_ops=on_run_start_response.debug_ops,
+          node_name_regex_allowlist=(
+              on_run_start_response.node_name_regex_allowlist),
+          op_type_regex_allowlist=(
+              on_run_start_response.op_type_regex_allowlist),
+          tensor_dtype_regex_allowlist=(
+              on_run_start_response.tensor_dtype_regex_allowlist),
+          tolerate_debug_op_creation_failures=(
+              on_run_start_response.tolerate_debug_op_creation_failures))
+      # pylint: enable=protected-access
+    elif self._performed_action == framework.OnRunStartAction.PROFILE_RUN:
+      # pylint: disable=protected-access
+      self._session_wrapper._decorate_run_options_for_profile(run_args.options)
+      # pylint: enable=protected-access
+
+    return run_args
+
+  def after_run(self, run_context, run_values):
+    # Adapt run_context and run_values to OnRunEndRequest and invoke superclass
+    # on_run_end()
+    on_run_end_request = framework.OnRunEndRequest(self._performed_action,
+                                                   run_values.run_metadata)
+    self._session_wrapper.on_run_end(on_run_end_request)
+
+
+class DumpingDebugHook(session_run_hook.SessionRunHook):
+  """A debugger hook that dumps debug data to filesystem.
+
+  Can be used as a hook for `tf.compat.v1.train.MonitoredSession`.
+  """
+
+  def __init__(self,
+               session_root,
+               watch_fn=None,
+               thread_name_filter=None):
+    """Create a local debugger command-line interface (CLI) hook.
+
+    Args:
+      session_root: See doc of
+        `dumping_wrapper.DumpingDebugWrapperSession.__init__`.
+      watch_fn: See doc of
+        `dumping_wrapper.DumpingDebugWrapperSession.__init__`.
+      thread_name_filter: Regular-expression white list for threads on which the
+        wrapper session will be active. See doc of `BaseDebugWrapperSession` for
+        more details.
+    """
+
+    self._session_root = session_root
+    self._watch_fn = watch_fn
+    self._thread_name_filter = thread_name_filter
+    self._session_wrapper = None
+
+  def begin(self):
+    pass
+
+  def before_run(self, run_context):
+    reset_disk_byte_usage = False
+    if not self._session_wrapper:
+      self._session_wrapper = dumping_wrapper.DumpingDebugWrapperSession(
+          run_context.session,
+          self._session_root,
+          watch_fn=self._watch_fn,
+          thread_name_filter=self._thread_name_filter)
+      reset_disk_byte_usage = True
+
+    self._session_wrapper.increment_run_call_count()
+
+    # pylint: disable=protected-access
+    debug_urls, watch_options = self._session_wrapper._prepare_run_watch_config(
+        run_context.original_args.fetches, run_context.original_args.feed_dict)
+    # pylint: enable=protected-access
+    run_options = config_pb2.RunOptions()
+    debug_utils.watch_graph(
+        run_options,
+        run_context.session.graph,
+        debug_urls=debug_urls,
+        debug_ops=watch_options.debug_ops,
+        node_name_regex_allowlist=watch_options.node_name_regex_allowlist,
+        op_type_regex_allowlist=watch_options.op_type_regex_allowlist,
+        tensor_dtype_regex_allowlist=watch_options.tensor_dtype_regex_allowlist,
+        tolerate_debug_op_creation_failures=(
+            watch_options.tolerate_debug_op_creation_failures),
+        reset_disk_byte_usage=reset_disk_byte_usage)
+
+    run_args = session_run_hook.SessionRunArgs(
+        None, feed_dict=None, options=run_options)
+    return run_args
+
+  def after_run(self, run_context, run_values):
+    pass
+
+
+class GrpcDebugHook(session_run_hook.SessionRunHook):
+  """A hook that streams debugger-related events to any grpc_debug_server.
+
+  For example, the debugger data server is a grpc_debug_server. The debugger
+  data server writes debugger-related events it receives via GRPC to logdir.
+  This enables debugging features in Tensorboard such as health pills.
+
+  When the arguments of debug_utils.watch_graph changes, strongly consider
+  changing arguments here too so that features are available to tflearn users.
+
+  Can be used as a hook for `tf.compat.v1.train.MonitoredSession`.
+  """
+
+  def __init__(self,
+               grpc_debug_server_addresses,
+               watch_fn=None,
+               thread_name_filter=None):
+    """Constructs a GrpcDebugHook.
+
+    Args:
+      grpc_debug_server_addresses: (`list` of `str`) A list of the gRPC debug
+        server addresses, in the format of <host:port>, with or without the
+        "grpc://" prefix. For example: ["localhost:7000", "192.168.0.2:8000"]
+      watch_fn: A function that allows for customizing which ops to watch at
+        which specific steps. See doc of
+        `dumping_wrapper.DumpingDebugWrapperSession.__init__` for details.
+      thread_name_filter: Regular-expression white list for threads on which the
+        wrapper session will be active. See doc of `BaseDebugWrapperSession` for
+        more details.
+    """
+    self._grpc_debug_wrapper_session = None
+    self._thread_name_filter = thread_name_filter
+    self._grpc_debug_server_addresses = (
+        grpc_debug_server_addresses
+        if isinstance(grpc_debug_server_addresses, list) else
+        [grpc_debug_server_addresses])
+
+    self._watch_fn = watch_fn
+
+  def before_run(self, run_context):
+    """Called right before a session is run.
+
+    Args:
+      run_context: A session_run_hook.SessionRunContext. Encapsulates
+        information on the run.
+
+    Returns:
+      A session_run_hook.SessionRunArgs object.
+    """
+
+    if not self._grpc_debug_wrapper_session:
+      self._grpc_debug_wrapper_session = grpc_wrapper.GrpcDebugWrapperSession(
+          run_context.session,
+          self._grpc_debug_server_addresses,
+          watch_fn=self._watch_fn,
+          thread_name_filter=self._thread_name_filter)
+
+    fetches = run_context.original_args.fetches
+    feed_dict = run_context.original_args.feed_dict
+    watch_options = self._watch_fn(fetches, feed_dict)
+    run_options = config_pb2.RunOptions()
+    debug_utils.watch_graph(
+        run_options,
+        run_context.session.graph,
+        debug_urls=self._grpc_debug_wrapper_session.prepare_run_debug_urls(
+            fetches, feed_dict),
+        debug_ops=watch_options.debug_ops,
+        node_name_regex_allowlist=watch_options.node_name_regex_allowlist,
+        op_type_regex_allowlist=watch_options.op_type_regex_allowlist,
+        tensor_dtype_regex_allowlist=watch_options.tensor_dtype_regex_allowlist,
+        tolerate_debug_op_creation_failures=(
+            watch_options.tolerate_debug_op_creation_failures))
+
+    return session_run_hook.SessionRunArgs(
+        None, feed_dict=None, options=run_options)
+
+
+class TensorBoardDebugHook(GrpcDebugHook):
+  """A tfdbg hook that can be used with TensorBoard Debugger Plugin.
+
+  This hook is the same as `GrpcDebugHook`, except that it uses a predefined
+    `watch_fn` that
+    1) uses `DebugIdentity` debug ops with the `gated_grpc` attribute set to
+        `True`, to allow the interactive enabling and disabling of tensor
+       breakpoints.
+    2) watches all tensors in the graph.
+  This saves the need for the user to define a `watch_fn`.
+  """
+
+  def __init__(self,
+               grpc_debug_server_addresses,
+               thread_name_filter=None,
+               send_traceback_and_source_code=True):
+    """Constructor of TensorBoardDebugHook.
+
+    Args:
+      grpc_debug_server_addresses: gRPC address(es) of debug server(s), as a
+        `str` or a `list` of `str`s. E.g., "localhost:2333",
+        "grpc://localhost:2333", ["192.168.0.7:2333", "192.168.0.8:2333"].
+      thread_name_filter: Optional filter for thread names.
+      send_traceback_and_source_code: Whether traceback of graph elements and
+        the source code are to be sent to the debug server(s).
+    """
+
+    def _gated_grpc_watch_fn(fetches, feeds):
+      del fetches, feeds  # Unused.
+      return framework.WatchOptions(
+          debug_ops=["DebugIdentity(gated_grpc=true)"])
+
+    super(TensorBoardDebugHook, self).__init__(
+        grpc_debug_server_addresses,
+        watch_fn=_gated_grpc_watch_fn,
+        thread_name_filter=thread_name_filter)
+
+    self._grpc_debug_server_addresses = grpc_debug_server_addresses
+    self._send_traceback_and_source_code = send_traceback_and_source_code
+    self._sent_graph_version = -1
+    grpc_wrapper.register_signal_handler()
+
+  def before_run(self, run_context):
+    if self._send_traceback_and_source_code:
+      self._sent_graph_version = grpc_wrapper.publish_traceback(
+          self._grpc_debug_server_addresses, run_context.session.graph,
+          run_context.original_args.feed_dict,
+          run_context.original_args.fetches, self._sent_graph_version)
+    return super(TensorBoardDebugHook, self).before_run(run_context)

tensorflow/python/debug/wrappers/local_cli_wrapper.py

@@ -0,0 +1,636 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Debugger Wrapper Session Consisting of a Local Curses-based CLI."""
+import argparse
+import os
+import sys
+import tempfile
+
+from tensorflow.python.debug.cli import analyzer_cli
+from tensorflow.python.debug.cli import cli_config
+from tensorflow.python.debug.cli import cli_shared
+from tensorflow.python.debug.cli import command_parser
+from tensorflow.python.debug.cli import debugger_cli_common
+from tensorflow.python.debug.cli import profile_analyzer_cli
+from tensorflow.python.debug.cli import ui_factory
+from tensorflow.python.debug.lib import common
+from tensorflow.python.debug.lib import debug_data
+from tensorflow.python.debug.wrappers import framework
+from tensorflow.python.lib.io import file_io
+
+
+_DUMP_ROOT_PREFIX = "tfdbg_"
+
+
+# TODO(donglin) Remove use_random_config_path after b/137652456 is fixed.
+class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
+  """Concrete subclass of BaseDebugWrapperSession implementing a local CLI.
+
+  This class has all the methods that a `session.Session` object has, in order
+  to support debugging with minimal code changes. Invoking its `run()` method
+  will launch the command-line interface (CLI) of tfdbg.
+  """
+
+  def __init__(self,
+               sess,
+               dump_root=None,
+               ui_type="readline",
+               thread_name_filter=None,
+               config_file_path=False):
+    """Constructor of LocalCLIDebugWrapperSession.
+
+    Args:
+      sess: The TensorFlow `Session` object being wrapped.
+      dump_root: (`str`) optional path to the dump root directory. Must be a
+        directory that does not exist or an empty directory. If the directory
+        does not exist, it will be created by the debugger core during debug
+        `run()` calls and removed afterwards. If `None`, the debug dumps will
+        be at tfdbg_<random_string> under the system temp directory.
+      ui_type: (`str`) requested UI type. Currently supported:
+        (readline)
+      thread_name_filter: Regular-expression white list for thread name. See
+        the doc of `BaseDebugWrapperSession` for details.
+      config_file_path: Optional override to the default configuration file
+        path, which is at `${HOME}/.tfdbg_config`.
+
+    Raises:
+      ValueError: If dump_root is an existing and non-empty directory or if
+        dump_root is a file.
+    """
+    framework.BaseDebugWrapperSession.__init__(
+        self, sess, thread_name_filter=thread_name_filter)
+
+    if not dump_root:
+      self._dump_root = tempfile.mkdtemp(prefix=_DUMP_ROOT_PREFIX)
+    else:
+      dump_root = os.path.expanduser(dump_root)
+      if os.path.isfile(dump_root):
+        raise ValueError("dump_root path points to a file: %s" % dump_root)
+      elif os.path.isdir(dump_root) and os.listdir(dump_root):
+        raise ValueError("dump_root path points to a non-empty directory: %s" %
+                         dump_root)
+
+      self._dump_root = dump_root
+
+    self._initialize_argparsers()
+
+    # Registered tensor filters.
+    self._tensor_filters = {}
+    # Register frequently-used filter(s).
+    self.add_tensor_filter("has_inf_or_nan", debug_data.has_inf_or_nan)
+
+    # Below are the state variables of this wrapper object.
+    # _active_tensor_filter: what (if any) tensor filter is in effect. If such
+    #   a filter is in effect, this object will call run() method of the
+    #   underlying TensorFlow Session object until the filter passes. This is
+    #   activated by the "-f" flag of the "run" command.
+    # _run_through_times: keeps track of how many times the wrapper needs to
+    #   run through without stopping at the run-end CLI. It is activated by the
+    #   "-t" option of the "run" command.
+    # _skip_debug: keeps track of whether the current run should be executed
+    #   without debugging. It is activated by the "-n" option of the "run"
+    #   command.
+    #
+    # _run_start_response: keeps track what OnRunStartResponse the wrapper
+    #   should return at the next run-start callback. If this information is
+    #   unavailable (i.e., is None), the run-start CLI will be launched to ask
+    #   the user. This is the case, e.g., right before the first run starts.
+    self._active_tensor_filter = None
+    self._active_filter_exclude_node_names = None
+    self._active_tensor_filter_run_start_response = None
+    self._run_through_times = 1
+    self._skip_debug = False
+    self._run_start_response = None
+    self._is_run_start = True
+    self._ui_type = ui_type
+    self._config = None
+    if config_file_path:
+      self._config = cli_config.CLIConfig(config_file_path=config_file_path)
+
+  def _is_disk_usage_reset_each_run(self):
+    # The dumped tensors are all cleaned up after every Session.run
+    # in a command-line wrapper.
+    return True
+
+  def _initialize_argparsers(self):
+    self._argparsers = {}
+    ap = argparse.ArgumentParser(
+        description="Run through, with or without debug tensor watching.",
+        usage=argparse.SUPPRESS)
+    ap.add_argument(
+        "-t",
+        "--times",
+        dest="times",
+        type=int,
+        default=1,
+        help="How many Session.run() calls to proceed with.")
+    ap.add_argument(
+        "-n",
+        "--no_debug",
+        dest="no_debug",
+        action="store_true",
+        help="Run through without debug tensor watching.")
+    ap.add_argument(
+        "-f",
+        "--till_filter_pass",
+        dest="till_filter_pass",
+        type=str,
+        default="",
+        help="Run until a tensor in the graph passes the specified filter.")
+    ap.add_argument(
+        "-fenn",
+        "--filter_exclude_node_names",
+        dest="filter_exclude_node_names",
+        type=str,
+        default="",
+        help="When applying the tensor filter, exclude node with names "
+        "matching the regular expression. Applicable only if --tensor_filter "
+        "or -f is used.")
+    ap.add_argument(
+        "--node_name_filter",
+        dest="node_name_filter",
+        type=str,
+        default="",
+        help="Regular-expression filter for node names to be watched in the "
+        "run, e.g., loss, reshape.*")
+    ap.add_argument(
+        "--op_type_filter",
+        dest="op_type_filter",
+        type=str,
+        default="",
+        help="Regular-expression filter for op type to be watched in the run, "
+        "e.g., (MatMul|Add), Variable.*")
+    ap.add_argument(
+        "--tensor_dtype_filter",
+        dest="tensor_dtype_filter",
+        type=str,
+        default="",
+        help="Regular-expression filter for tensor dtype to be watched in the "
+        "run, e.g., (float32|float64), int.*")
+    ap.add_argument(
+        "-p",
+        "--profile",
+        dest="profile",
+        action="store_true",
+        help="Run and profile TensorFlow graph execution.")
+    self._argparsers["run"] = ap
+
+    ap = argparse.ArgumentParser(
+        description="Display information about this Session.run() call.",
+        usage=argparse.SUPPRESS)
+    self._argparsers["run_info"] = ap
+
+    self._argparsers["print_feed"] = command_parser.get_print_tensor_argparser(
+        "Print the value of a feed in feed_dict.")
+
+  def add_tensor_filter(self, filter_name, tensor_filter):
+    """Add a tensor filter.
+
+    Args:
+      filter_name: (`str`) name of the filter.
+      tensor_filter: (`callable`) the filter callable. See the doc string of
+        `DebugDumpDir.find()` for more details about its signature.
+    """
+
+    self._tensor_filters[filter_name] = tensor_filter
+
+  def on_session_init(self, request):
+    """Overrides on-session-init callback.
+
+    Args:
+      request: An instance of `OnSessionInitRequest`.
+
+    Returns:
+      An instance of `OnSessionInitResponse`.
+    """
+
+    return framework.OnSessionInitResponse(
+        framework.OnSessionInitAction.PROCEED)
+
+  def on_run_start(self, request):
+    """Overrides on-run-start callback.
+
+    Args:
+      request: An instance of `OnRunStartRequest`.
+
+    Returns:
+      An instance of `OnRunStartResponse`.
+    """
+    self._is_run_start = True
+    self._update_run_calls_state(
+        request.run_call_count, request.fetches, request.feed_dict,
+        is_callable_runner=request.is_callable_runner)
+
+    if self._active_tensor_filter:
+      # If we are running until a filter passes, we just need to keep running
+      # with the previous `OnRunStartResponse`.
+      return self._active_tensor_filter_run_start_response
+
+    self._exit_if_requested_by_user()
+
+    if self._run_call_count > 1 and not self._skip_debug:
+      if self._run_through_times > 0:
+        # Just run through without debugging.
+        return framework.OnRunStartResponse(
+            framework.OnRunStartAction.NON_DEBUG_RUN, [])
+      elif self._run_through_times == 0:
+        # It is the run at which the run-end CLI will be launched: activate
+        # debugging.
+        return (self._run_start_response or
+                framework.OnRunStartResponse(
+                    framework.OnRunStartAction.DEBUG_RUN,
+                    self._get_run_debug_urls()))
+
+    if self._run_start_response is None:
+      self._prep_cli_for_run_start()
+
+      self._run_start_response = self._launch_cli()
+      if self._active_tensor_filter:
+        self._active_tensor_filter_run_start_response = self._run_start_response
+      if self._run_through_times > 1:
+        self._run_through_times -= 1
+
+    self._exit_if_requested_by_user()
+    return self._run_start_response
+
+  def _exit_if_requested_by_user(self):
+    if self._run_start_response == debugger_cli_common.EXPLICIT_USER_EXIT:
+      # Explicit user "exit" command leads to sys.exit(1).
+      print(
+          "Note: user exited from debugger CLI: Calling sys.exit(1).",
+          file=sys.stderr)
+      sys.exit(1)
+
+  def _prep_cli_for_run_start(self):
+    """Prepare (but not launch) the CLI for run-start."""
+    self._run_cli = ui_factory.get_ui(self._ui_type, config=self._config)
+
+    help_intro = debugger_cli_common.RichTextLines([])
+    if self._run_call_count == 1:
+      # Show logo at the onset of the first run.
+      help_intro.extend(cli_shared.get_tfdbg_logo())
+      help_intro.extend(debugger_cli_common.get_tensorflow_version_lines())
+    help_intro.extend(debugger_cli_common.RichTextLines("Upcoming run:"))
+    help_intro.extend(self._run_info)
+
+    self._run_cli.set_help_intro(help_intro)
+
+    # Create initial screen output detailing the run.
+    self._title = "run-start: " + self._run_description
+    self._init_command = "run_info"
+    self._title_color = "blue_on_white"
+
+  def on_run_end(self, request):
+    """Overrides on-run-end callback.
+
+    Actions taken:
+      1) Load the debug dump.
+      2) Bring up the Analyzer CLI.
+
+    Args:
+      request: An instance of OnSessionInitRequest.
+
+    Returns:
+      An instance of OnSessionInitResponse.
+    """
+
+    self._is_run_start = False
+    if request.performed_action == framework.OnRunStartAction.DEBUG_RUN:
+      partition_graphs = None
+      if request.run_metadata and request.run_metadata.partition_graphs:
+        partition_graphs = request.run_metadata.partition_graphs
+      elif request.client_graph_def:
+        partition_graphs = [request.client_graph_def]
+
+      if request.tf_error and not os.path.isdir(self._dump_root):
+        # It is possible that the dump root may not exist due to errors that
+        # have occurred prior to graph execution (e.g., invalid device
+        # assignments), in which case we will just raise the exception as the
+        # unwrapped Session does.
+        raise request.tf_error
+
+      debug_dump = debug_data.DebugDumpDir(
+          self._dump_root, partition_graphs=partition_graphs)
+      debug_dump.set_python_graph(self._sess.graph)
+
+      passed_filter = None
+      passed_filter_exclude_node_names = None
+      if self._active_tensor_filter:
+        if not debug_dump.find(
+            self._tensor_filters[self._active_tensor_filter], first_n=1,
+            exclude_node_names=self._active_filter_exclude_node_names):
+          # No dumped tensor passes the filter in this run. Clean up the dump
+          # directory and move on.
+          self._remove_dump_root()
+          return framework.OnRunEndResponse()
+        else:
+          # Some dumped tensor(s) from this run passed the filter.
+          passed_filter = self._active_tensor_filter
+          passed_filter_exclude_node_names = (
+              self._active_filter_exclude_node_names)
+          self._active_tensor_filter = None
+          self._active_filter_exclude_node_names = None
+
+      self._prep_debug_cli_for_run_end(
+          debug_dump, request.tf_error, passed_filter,
+          passed_filter_exclude_node_names)
+
+      self._run_start_response = self._launch_cli()
+
+      # Clean up the dump generated by this run.
+      self._remove_dump_root()
+    elif request.performed_action == framework.OnRunStartAction.PROFILE_RUN:
+      self._prep_profile_cli_for_run_end(self._sess.graph, request.run_metadata)
+      self._run_start_response = self._launch_cli()
+    else:
+      # No debug information to show following a non-debug run() call.
+      self._run_start_response = None
+
+    # Return placeholder response that currently holds no additional
+    # information.
+    return framework.OnRunEndResponse()
+
+  def _remove_dump_root(self):
+    if os.path.isdir(self._dump_root):
+      file_io.delete_recursively(self._dump_root)
+
+  def _prep_debug_cli_for_run_end(self,
+                                  debug_dump,
+                                  tf_error,
+                                  passed_filter,
+                                  passed_filter_exclude_node_names):
+    """Prepare (but not launch) CLI for run-end, with debug dump from the run.
+
+    Args:
+      debug_dump: (debug_data.DebugDumpDir) The debug dump directory from this
+        run.
+      tf_error: (None or OpError) OpError that happened during the run() call
+        (if any).
+      passed_filter: (None or str) Name of the tensor filter that just passed
+        and caused the preparation of this run-end CLI (if any).
+      passed_filter_exclude_node_names: (None or str) Regular expression used
+        with the tensor filter to exclude ops with names matching the regular
+        expression.
+    """
+
+    if tf_error:
+      help_intro = cli_shared.get_error_intro(tf_error)
+
+      self._init_command = "help"
+      self._title_color = "red_on_white"
+    else:
+      help_intro = None
+      self._init_command = "lt"
+
+      self._title_color = "black_on_white"
+      if passed_filter is not None:
+        # Some dumped tensor(s) from this run passed the filter.
+        self._init_command = "lt -f %s" % passed_filter
+        if passed_filter_exclude_node_names:
+          self._init_command += (" --filter_exclude_node_names %s" %
+                                 passed_filter_exclude_node_names)
+        self._title_color = "red_on_white"
+
+    self._run_cli = analyzer_cli.create_analyzer_ui(
+        debug_dump,
+        self._tensor_filters,
+        ui_type=self._ui_type,
+        on_ui_exit=self._remove_dump_root,
+        config=self._config)
+
+    # Get names of all dumped tensors.
+    dumped_tensor_names = []
+    for datum in debug_dump.dumped_tensor_data:
+      dumped_tensor_names.append("%s:%d" %
+                                 (datum.node_name, datum.output_slot))
+
+    # Tab completions for command "print_tensors".
+    self._run_cli.register_tab_comp_context(["print_tensor", "pt"],
+                                            dumped_tensor_names)
+
+    # Tab completion for commands "node_info", "list_inputs" and
+    # "list_outputs". The list comprehension is used below because nodes()
+    # output can be unicodes and they need to be converted to strs.
+    self._run_cli.register_tab_comp_context(
+        ["node_info", "ni", "list_inputs", "li", "list_outputs", "lo"],
+        [str(node_name) for node_name in debug_dump.nodes()])
+    # TODO(cais): Reduce API surface area for aliases vis-a-vis tab
+    #    completion contexts and registered command handlers.
+
+    self._title = "run-end: " + self._run_description
+
+    if help_intro:
+      self._run_cli.set_help_intro(help_intro)
+
+  def _prep_profile_cli_for_run_end(self, py_graph, run_metadata):
+    self._init_command = "lp"
+    self._run_cli = profile_analyzer_cli.create_profiler_ui(
+        py_graph, run_metadata, ui_type=self._ui_type,
+        config=self._run_cli.config)
+    self._title = "run-end (profiler mode): " + self._run_description
+
+  def _launch_cli(self):
+    """Launch the interactive command-line interface.
+
+    Returns:
+      The OnRunStartResponse specified by the user using the "run" command.
+    """
+
+    self._register_this_run_info(self._run_cli)
+    response = self._run_cli.run_ui(
+        init_command=self._init_command,
+        title=self._title,
+        title_color=self._title_color)
+
+    return response
+
+  def _run_info_handler(self, args, screen_info=None):
+    output = debugger_cli_common.RichTextLines([])
+
+    if self._run_call_count == 1:
+      output.extend(cli_shared.get_tfdbg_logo())
+      output.extend(debugger_cli_common.get_tensorflow_version_lines())
+    output.extend(self._run_info)
+
+    if (not self._is_run_start and
+        debugger_cli_common.MAIN_MENU_KEY in output.annotations):
+      menu = output.annotations[debugger_cli_common.MAIN_MENU_KEY]
+      if "list_tensors" not in menu.captions():
+        menu.insert(
+            0, debugger_cli_common.MenuItem("list_tensors", "list_tensors"))
+
+    return output
+
+  def _print_feed_handler(self, args, screen_info=None):
+    np_printoptions = cli_shared.numpy_printoptions_from_screen_info(
+        screen_info)
+
+    if not self._feed_dict:
+      return cli_shared.error(
+          "The feed_dict of the current run is None or empty.")
+
+    parsed = self._argparsers["print_feed"].parse_args(args)
+    tensor_name, tensor_slicing = (
+        command_parser.parse_tensor_name_with_slicing(parsed.tensor_name))
+
+    feed_key = None
+    feed_value = None
+    for key in self._feed_dict:
+      key_name = common.get_graph_element_name(key)
+      if key_name == tensor_name:
+        feed_key = key_name
+        feed_value = self._feed_dict[key]
+        break
+
+    if feed_key is None:
+      return cli_shared.error(
+          "The feed_dict of the current run does not contain the key %s" %
+          tensor_name)
+    else:
+      return cli_shared.format_tensor(
+          feed_value,
+          feed_key + " (feed)",
+          np_printoptions,
+          print_all=parsed.print_all,
+          tensor_slicing=tensor_slicing,
+          highlight_options=cli_shared.parse_ranges_highlight(parsed.ranges),
+          include_numeric_summary=parsed.numeric_summary)
+
+  def _run_handler(self, args, screen_info=None):
+    """Command handler for "run" command during on-run-start."""
+
+    del screen_info  # Currently unused.
+
+    parsed = self._argparsers["run"].parse_args(args)
+    parsed.node_name_filter = parsed.node_name_filter or None
+    parsed.op_type_filter = parsed.op_type_filter or None
+    parsed.tensor_dtype_filter = parsed.tensor_dtype_filter or None
+
+    if parsed.filter_exclude_node_names and not parsed.till_filter_pass:
+      raise ValueError(
+          "The --filter_exclude_node_names (or -feon) flag is valid only if "
+          "the --till_filter_pass (or -f) flag is used.")
+
+    if parsed.profile:
+      raise debugger_cli_common.CommandLineExit(
+          exit_token=framework.OnRunStartResponse(
+              framework.OnRunStartAction.PROFILE_RUN, []))
+
+    self._skip_debug = parsed.no_debug
+    self._run_through_times = parsed.times
+
+    if parsed.times > 1 or parsed.no_debug:
+      # If requested -t times > 1, the very next run will be a non-debug run.
+      action = framework.OnRunStartAction.NON_DEBUG_RUN
+      debug_urls = []
+    else:
+      action = framework.OnRunStartAction.DEBUG_RUN
+      debug_urls = self._get_run_debug_urls()
+    run_start_response = framework.OnRunStartResponse(
+        action,
+        debug_urls,
+        node_name_regex_allowlist=parsed.node_name_filter,
+        op_type_regex_allowlist=parsed.op_type_filter,
+        tensor_dtype_regex_allowlist=parsed.tensor_dtype_filter)
+
+    if parsed.till_filter_pass:
+      # For the run-till-filter-pass (run -f) mode, use the DEBUG_RUN
+      # option to access the intermediate tensors, and set the corresponding
+      # state flag of the class itself to True.
+      if parsed.till_filter_pass in self._tensor_filters:
+        action = framework.OnRunStartAction.DEBUG_RUN
+        self._active_tensor_filter = parsed.till_filter_pass
+        self._active_filter_exclude_node_names = (
+            parsed.filter_exclude_node_names)
+        self._active_tensor_filter_run_start_response = run_start_response
+      else:
+        # Handle invalid filter name.
+        return debugger_cli_common.RichTextLines(
+            ["ERROR: tensor filter \"%s\" does not exist." %
+             parsed.till_filter_pass])
+
+    # Raise CommandLineExit exception to cause the CLI to exit.
+    raise debugger_cli_common.CommandLineExit(exit_token=run_start_response)
+
+  def _register_this_run_info(self, curses_cli):
+    curses_cli.register_command_handler(
+        "run",
+        self._run_handler,
+        self._argparsers["run"].format_help(),
+        prefix_aliases=["r"])
+    curses_cli.register_command_handler(
+        "run_info",
+        self._run_info_handler,
+        self._argparsers["run_info"].format_help(),
+        prefix_aliases=["ri"])
+    curses_cli.register_command_handler(
+        "print_feed",
+        self._print_feed_handler,
+        self._argparsers["print_feed"].format_help(),
+        prefix_aliases=["pf"])
+
+    if self._tensor_filters:
+      # Register tab completion for the filter names.
+      curses_cli.register_tab_comp_context(["run", "r"],
+                                           list(self._tensor_filters.keys()))
+    if self._feed_dict and hasattr(self._feed_dict, "keys"):
+      # Register tab completion for feed_dict keys.
+      feed_keys = [common.get_graph_element_name(key)
+                   for key in self._feed_dict.keys()]
+      curses_cli.register_tab_comp_context(["print_feed", "pf"], feed_keys)
+
+  def _get_run_debug_urls(self):
+    """Get the debug_urls value for the current run() call.
+
+    Returns:
+      debug_urls: (list of str) Debug URLs for the current run() call.
+        Currently, the list consists of only one URL that is a file:// URL.
+    """
+
+    return ["file://" + self._dump_root]
+
+  def _update_run_calls_state(self,
+                              run_call_count,
+                              fetches,
+                              feed_dict,
+                              is_callable_runner=False):
+    """Update the internal state with regard to run() call history.
+
+    Args:
+      run_call_count: (int) Number of run() calls that have occurred.
+      fetches: a node/tensor or a list of node/tensor that are the fetches of
+        the run() call. This is the same as the fetches argument to the run()
+        call.
+      feed_dict: None of a dict. This is the feed_dict argument to the run()
+        call.
+      is_callable_runner: (bool) whether a runner returned by
+        Session.make_callable is being run.
+    """
+
+    self._run_call_count = run_call_count
+    self._feed_dict = feed_dict
+    self._run_description = cli_shared.get_run_short_description(
+        run_call_count,
+        fetches,
+        feed_dict,
+        is_callable_runner=is_callable_runner)
+    self._run_through_times -= 1
+
+    self._run_info = cli_shared.get_run_start_intro(
+        run_call_count,
+        fetches,
+        feed_dict,
+        self._tensor_filters,
+        is_callable_runner=is_callable_runner)

tensorflow/python/debug/wrappers/local_cli_wrapper_test.py

@@ -0,0 +1,851 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Unit tests for local command-line-interface debug wrapper session."""
+import os
+import tempfile
+
+import numpy as np
+
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.python.client import session
+from tensorflow.python.debug.cli import cli_config
+from tensorflow.python.debug.cli import cli_shared
+from tensorflow.python.debug.cli import debugger_cli_common
+from tensorflow.python.debug.cli import ui_factory
+from tensorflow.python.debug.wrappers import local_cli_wrapper
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.lib.io import file_io
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+# Import resource_variable_ops for the variables-to-tensor implicit conversion.
+from tensorflow.python.ops import resource_variable_ops  # pylint: disable=unused-import
+from tensorflow.python.ops import sparse_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_v1
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+from tensorflow.python.training import monitored_session
+from tensorflow.python.training import session_run_hook
+
+
+class LocalCLIDebuggerWrapperSessionForTest(
+    local_cli_wrapper.LocalCLIDebugWrapperSession):
+  """Subclasses the wrapper class for testing.
+
+  Overrides its CLI-related methods for headless testing environments.
+  Inserts observer variables for assertions.
+  """
+
+  def __init__(self,
+               command_sequence,
+               sess,
+               dump_root=None):
+    """Constructor of the for-test subclass.
+
+    Args:
+      command_sequence: (list of list of str) A list of command arguments,
+        including the command prefix, each element of the list is such as:
+        ["run", "-n"],
+        ["print_feed", "input:0"].
+      sess: See the doc string of LocalCLIDebugWrapperSession.__init__.
+      dump_root: See the doc string of LocalCLIDebugWrapperSession.__init__.
+    """
+
+    local_cli_wrapper.LocalCLIDebugWrapperSession.__init__(
+        self, sess, dump_root=dump_root)
+
+    self._command_sequence = command_sequence
+    self._command_pointer = 0
+
+    # Observer variables.
+    self.observers = {
+        "debug_dumps": [],
+        "tf_errors": [],
+        "run_start_cli_run_numbers": [],
+        "run_end_cli_run_numbers": [],
+        "print_feed_responses": [],
+        "profiler_py_graphs": [],
+        "profiler_run_metadata": [],
+    }
+
+  def _prep_cli_for_run_start(self):
+    pass
+
+  def _prep_debug_cli_for_run_end(self,
+                                  debug_dump,
+                                  tf_error,
+                                  passed_filter,
+                                  passed_filter_exclude_op_names):
+    self.observers["debug_dumps"].append(debug_dump)
+    self.observers["tf_errors"].append(tf_error)
+
+  def _prep_profile_cli_for_run_end(self, py_graph, run_metadata):
+    self.observers["profiler_py_graphs"].append(py_graph)
+    self.observers["profiler_run_metadata"].append(run_metadata)
+
+  def _launch_cli(self):
+    if self._is_run_start:
+      self.observers["run_start_cli_run_numbers"].append(self._run_call_count)
+    else:
+      self.observers["run_end_cli_run_numbers"].append(self._run_call_count)
+
+    readline_cli = ui_factory.get_ui(
+        "readline",
+        config=cli_config.CLIConfig(
+            config_file_path=os.path.join(tempfile.mkdtemp(), ".tfdbg_config")))
+    self._register_this_run_info(readline_cli)
+
+    while self._command_pointer < len(self._command_sequence):
+      command = self._command_sequence[self._command_pointer]
+      self._command_pointer += 1
+
+      try:
+        if command[0] == "run":
+          self._run_handler(command[1:])
+        elif command[0] == "print_feed":
+          self.observers["print_feed_responses"].append(
+              self._print_feed_handler(command[1:]))
+        else:
+          raise ValueError("Unrecognized command prefix: %s" % command[0])
+      except debugger_cli_common.CommandLineExit as e:
+        return e.exit_token
+
+
+@test_util.run_v1_only("b/120545219")
+class LocalCLIDebugWrapperSessionTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    self._tmp_dir = tempfile.mkdtemp()
+
+    self.v = variable_v1.VariableV1(10.0, name="v")
+    self.w = variable_v1.VariableV1(21.0, name="w")
+    self.delta = constant_op.constant(1.0, name="delta")
+    self.inc_v = state_ops.assign_add(self.v, self.delta, name="inc_v")
+
+    self.w_int = control_flow_ops.with_dependencies(
+        [self.inc_v],
+        math_ops.cast(self.w, dtypes.int32, name="w_int_inner"),
+        name="w_int_outer")
+
+    self.ph = array_ops.placeholder(dtypes.float32, name="ph")
+    self.xph = array_ops.transpose(self.ph, name="xph")
+    self.m = constant_op.constant(
+        [[0.0, 1.0, 2.0], [-4.0, -1.0, 0.0]], dtype=dtypes.float32, name="m")
+    self.y = math_ops.matmul(self.m, self.xph, name="y")
+
+    self.sparse_ph = array_ops.sparse_placeholder(
+        dtypes.float32, shape=([5, 5]), name="sparse_placeholder")
+    self.sparse_add = sparse_ops.sparse_add(self.sparse_ph, self.sparse_ph)
+
+    rewriter_config = rewriter_config_pb2.RewriterConfig(
+        disable_model_pruning=True,
+        arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF,
+        dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF)
+    graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_config)
+    config_proto = config_pb2.ConfigProto(graph_options=graph_options)
+    self.sess = session.Session(config=config_proto)
+
+    # Initialize variable.
+    self.sess.run(variables.global_variables_initializer())
+
+  def tearDown(self):
+    ops.reset_default_graph()
+    if os.path.isdir(self._tmp_dir):
+      file_io.delete_recursively(self._tmp_dir)
+
+  def testConstructWrapper(self):
+    local_cli_wrapper.LocalCLIDebugWrapperSession(session.Session())
+
+  def testConstructWrapperWithExistingNonEmptyDumpRoot(self):
+    dir_path = os.path.join(self._tmp_dir, "foo")
+    os.mkdir(dir_path)
+    self.assertTrue(os.path.isdir(dir_path))
+
+    with self.assertRaisesRegex(
+        ValueError, "dump_root path points to a non-empty directory"):
+      local_cli_wrapper.LocalCLIDebugWrapperSession(
+          session.Session(), dump_root=self._tmp_dir)
+
+  def testConstructWrapperWithExistingFileDumpRoot(self):
+    file_path = os.path.join(self._tmp_dir, "foo")
+    open(file_path, "a").close()  # Create the file
+    self.assertTrue(os.path.isfile(file_path))
+    with self.assertRaisesRegex(ValueError, "dump_root path points to a file"):
+      local_cli_wrapper.LocalCLIDebugWrapperSession(
+          session.Session(), dump_root=file_path)
+
+  def testRunsUnderDebugMode(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"], ["run"]], self.sess, dump_root=self._tmp_dir)
+
+    # run under debug mode twice.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    # Verify that the assign_add op did take effect.
+    self.assertAllClose(12.0, self.sess.run(self.v))
+
+    # Assert correct run call numbers for which the CLI has been launched at
+    # run-start and run-end.
+    self.assertEqual([1], wrapped_sess.observers["run_start_cli_run_numbers"])
+    self.assertEqual([1, 2], wrapped_sess.observers["run_end_cli_run_numbers"])
+
+    # Verify that the dumps have been generated and picked up during run-end.
+    self.assertEqual(2, len(wrapped_sess.observers["debug_dumps"]))
+
+    # Verify that the TensorFlow runtime errors are picked up and in this case,
+    # they should be both None.
+    self.assertEqual([None, None], wrapped_sess.observers["tf_errors"])
+
+  def testRunsWithEmptyStringDumpRootWorks(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess, dump_root="")
+
+    # run under debug mode.
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(11.0, self.sess.run(self.v))
+
+  def testRunInfoOutputAtRunEndIsCorrect(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"], ["run"]], self.sess, dump_root=self._tmp_dir)
+
+    wrapped_sess.run(self.inc_v)
+    run_info_output = wrapped_sess._run_info_handler([])
+
+    tfdbg_logo = cli_shared.get_tfdbg_logo()
+
+    # The run_info output in the first run() call should contain the tfdbg logo.
+    self.assertEqual(tfdbg_logo.lines,
+                     run_info_output.lines[:len(tfdbg_logo.lines)])
+    menu = run_info_output.annotations[debugger_cli_common.MAIN_MENU_KEY]
+    self.assertIn("list_tensors", menu.captions())
+
+    wrapped_sess.run(self.inc_v)
+    run_info_output = wrapped_sess._run_info_handler([])
+
+    # The run_info output in the second run() call should NOT contain the logo.
+    self.assertNotEqual(tfdbg_logo.lines,
+                        run_info_output.lines[:len(tfdbg_logo.lines)])
+    menu = run_info_output.annotations[debugger_cli_common.MAIN_MENU_KEY]
+    self.assertIn("list_tensors", menu.captions())
+
+  def testRunsUnderNonDebugMode(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-n"], ["run", "-n"], ["run", "-n"]],
+        self.sess, dump_root=self._tmp_dir)
+
+    # run three times.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(13.0, self.sess.run(self.v))
+
+    self.assertEqual([1, 2, 3],
+                     wrapped_sess.observers["run_start_cli_run_numbers"])
+    self.assertEqual([], wrapped_sess.observers["run_end_cli_run_numbers"])
+
+  def testRunningWithSparsePlaceholderFeedWorks(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess, dump_root=self._tmp_dir)
+
+    sparse_feed = ([[0, 1], [0, 2]], [10.0, 20.0])
+    sparse_result = wrapped_sess.run(
+        self.sparse_add, feed_dict={self.sparse_ph: sparse_feed})
+    self.assertAllEqual([[0, 1], [0, 2]], sparse_result.indices)
+    self.assertAllClose([20.0, 40.0], sparse_result.values)
+
+  def testRunsUnderNonDebugThenDebugMode(self):
+    # Do two NON_DEBUG_RUNs, followed by DEBUG_RUNs.
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-n"], ["run", "-n"], ["run"], ["run"]],
+        self.sess, dump_root=self._tmp_dir)
+
+    # run three times.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(13.0, self.sess.run(self.v))
+
+    self.assertEqual([1, 2, 3],
+                     wrapped_sess.observers["run_start_cli_run_numbers"])
+
+    # Here, the CLI should have been launched only under the third run,
+    # because the first and second runs are NON_DEBUG.
+    self.assertEqual([3], wrapped_sess.observers["run_end_cli_run_numbers"])
+    self.assertEqual(1, len(wrapped_sess.observers["debug_dumps"]))
+    self.assertEqual([None], wrapped_sess.observers["tf_errors"])
+
+  def testRunMultipleTimesWithinLimit(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-t", "3"], ["run"]],
+        self.sess, dump_root=self._tmp_dir)
+
+    # run three times.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(13.0, self.sess.run(self.v))
+
+    self.assertEqual([1], wrapped_sess.observers["run_start_cli_run_numbers"])
+    self.assertEqual([3], wrapped_sess.observers["run_end_cli_run_numbers"])
+    self.assertEqual(1, len(wrapped_sess.observers["debug_dumps"]))
+    self.assertEqual([None], wrapped_sess.observers["tf_errors"])
+
+  def testRunMultipleTimesOverLimit(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-t", "3"]], self.sess, dump_root=self._tmp_dir)
+
+    # run twice, which is less than the number of times specified by the
+    # command.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(12.0, self.sess.run(self.v))
+
+    self.assertEqual([1], wrapped_sess.observers["run_start_cli_run_numbers"])
+    self.assertEqual([], wrapped_sess.observers["run_end_cli_run_numbers"])
+    self.assertEqual(0, len(wrapped_sess.observers["debug_dumps"]))
+    self.assertEqual([], wrapped_sess.observers["tf_errors"])
+
+  def testRunMixingDebugModeAndMultipleTimes(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-n"], ["run", "-t", "2"], ["run"], ["run"]],
+        self.sess, dump_root=self._tmp_dir)
+
+    # run four times.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(14.0, self.sess.run(self.v))
+
+    self.assertEqual([1, 2],
+                     wrapped_sess.observers["run_start_cli_run_numbers"])
+    self.assertEqual([3, 4], wrapped_sess.observers["run_end_cli_run_numbers"])
+    self.assertEqual(2, len(wrapped_sess.observers["debug_dumps"]))
+    self.assertEqual([None, None], wrapped_sess.observers["tf_errors"])
+
+  def testDebuggingMakeCallableTensorRunnerWorks(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess, dump_root=self._tmp_dir)
+    v = variable_v1.VariableV1(42)
+    tensor_runner = wrapped_sess.make_callable(v)
+    self.sess.run(v.initializer)
+
+    self.assertAllClose(42, tensor_runner())
+    self.assertEqual(1, len(wrapped_sess.observers["debug_dumps"]))
+
+  def testDebuggingMakeCallableTensorRunnerWithCustomRunOptionsWorks(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess, dump_root=self._tmp_dir)
+    a = constant_op.constant(42)
+    tensor_runner = wrapped_sess.make_callable(a)
+
+    run_options = config_pb2.RunOptions(
+        trace_level=config_pb2.RunOptions.FULL_TRACE)
+    run_metadata = config_pb2.RunMetadata()
+    self.assertAllClose(
+        42, tensor_runner(options=run_options, run_metadata=run_metadata))
+    self.assertEqual(1, len(wrapped_sess.observers["debug_dumps"]))
+    self.assertGreater(len(run_metadata.step_stats.dev_stats), 0)
+
+  def testDebuggingMakeCallableOperationRunnerWorks(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess, dump_root=self._tmp_dir)
+    v = variable_v1.VariableV1(10.0)
+    inc_v = state_ops.assign_add(v, 1.0)
+    op_runner = wrapped_sess.make_callable(inc_v.op)
+    self.sess.run(v.initializer)
+
+    op_runner()
+    self.assertEqual(1, len(wrapped_sess.observers["debug_dumps"]))
+    self.assertEqual(11.0, self.sess.run(v))
+
+  def testDebuggingMakeCallableRunnerWithFeedListWorks(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess, dump_root=self._tmp_dir)
+    ph1 = array_ops.placeholder(dtypes.float32)
+    ph2 = array_ops.placeholder(dtypes.float32)
+    a = math_ops.add(ph1, ph2)
+    tensor_runner = wrapped_sess.make_callable(a, feed_list=[ph1, ph2])
+
+    self.assertAllClose(42.0, tensor_runner(41.0, 1.0))
+    self.assertEqual(1, len(wrapped_sess.observers["debug_dumps"]))
+
+  def testDebuggingMakeCallableFromOptionsWithZeroFeedWorks(self):
+    variable_1 = variable_v1.VariableV1(
+        10.5, dtype=dtypes.float32, name="variable_1")
+    a = math_ops.add(variable_1, variable_1, "callable_a")
+    math_ops.add(a, a, "callable_b")
+    self.sess.run(variable_1.initializer)
+
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"]] * 3, self.sess, dump_root=self._tmp_dir)
+    callable_options = config_pb2.CallableOptions()
+    callable_options.fetch.append("callable_b")
+    sess_callable = wrapped_sess._make_callable_from_options(callable_options)
+
+    for _ in range(2):
+      callable_output = sess_callable()
+      self.assertAllClose(np.array(42.0, dtype=np.float32), callable_output[0])
+
+    debug_dumps = wrapped_sess.observers["debug_dumps"]
+    self.assertEqual(2, len(debug_dumps))
+    for debug_dump in debug_dumps:
+      node_names = [datum.node_name for datum in debug_dump.dumped_tensor_data]
+      self.assertItemsEqual(
+          ["callable_a", "callable_b", "variable_1", "variable_1/read"],
+          node_names)
+
+  def testDebuggingMakeCallableFromOptionsWithOneFeedWorks(self):
+    ph1 = array_ops.placeholder(dtypes.float32, name="callable_ph1")
+    a = math_ops.add(ph1, ph1, "callable_a")
+    math_ops.add(a, a, "callable_b")
+
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"]] * 3, self.sess, dump_root=self._tmp_dir)
+    callable_options = config_pb2.CallableOptions()
+    callable_options.feed.append("callable_ph1")
+    callable_options.fetch.append("callable_b")
+    sess_callable = wrapped_sess._make_callable_from_options(callable_options)
+
+    ph1_value = np.array([10.5, -10.5], dtype=np.float32)
+
+    for _ in range(2):
+      callable_output = sess_callable(ph1_value)
+      self.assertAllClose(
+          np.array([42.0, -42.0], dtype=np.float32), callable_output[0])
+
+    debug_dumps = wrapped_sess.observers["debug_dumps"]
+    self.assertEqual(2, len(debug_dumps))
+    for debug_dump in debug_dumps:
+      node_names = [datum.node_name for datum in debug_dump.dumped_tensor_data]
+      self.assertIn("callable_a", node_names)
+      self.assertIn("callable_b", node_names)
+
+  def testDebuggingMakeCallableFromOptionsWithTwoFeedsWorks(self):
+    ph1 = array_ops.placeholder(dtypes.float32, name="callable_ph1")
+    ph2 = array_ops.placeholder(dtypes.float32, name="callable_ph2")
+    a = math_ops.add(ph1, ph2, "callable_a")
+    math_ops.add(a, a, "callable_b")
+
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"]] * 3, self.sess, dump_root=self._tmp_dir)
+    callable_options = config_pb2.CallableOptions()
+    callable_options.feed.append("callable_ph1")
+    callable_options.feed.append("callable_ph2")
+    callable_options.fetch.append("callable_b")
+    sess_callable = wrapped_sess._make_callable_from_options(callable_options)
+
+    ph1_value = np.array(5.0, dtype=np.float32)
+    ph2_value = np.array(16.0, dtype=np.float32)
+
+    for _ in range(2):
+      callable_output = sess_callable(ph1_value, ph2_value)
+      self.assertAllClose(np.array(42.0, dtype=np.float32), callable_output[0])
+
+    debug_dumps = wrapped_sess.observers["debug_dumps"]
+    self.assertEqual(2, len(debug_dumps))
+    for debug_dump in debug_dumps:
+      node_names = [datum.node_name for datum in debug_dump.dumped_tensor_data]
+      self.assertIn("callable_a", node_names)
+      self.assertIn("callable_b", node_names)
+
+  def testDebugMakeCallableFromOptionsWithCustomOptionsAndMetadataWorks(self):
+    variable_1 = variable_v1.VariableV1(
+        10.5, dtype=dtypes.float32, name="variable_1")
+    a = math_ops.add(variable_1, variable_1, "callable_a")
+    math_ops.add(a, a, "callable_b")
+    self.sess.run(variable_1.initializer)
+
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess, dump_root=self._tmp_dir)
+    callable_options = config_pb2.CallableOptions()
+    callable_options.fetch.append("callable_b")
+    callable_options.run_options.trace_level = config_pb2.RunOptions.FULL_TRACE
+
+    sess_callable = wrapped_sess._make_callable_from_options(callable_options)
+
+    run_metadata = config_pb2.RunMetadata()
+    # Call the callable with a custom run_metadata.
+    callable_output = sess_callable(run_metadata=run_metadata)
+    # Verify that step_stats is populated in the custom run_metadata.
+    self.assertTrue(run_metadata.step_stats)
+    self.assertAllClose(np.array(42.0, dtype=np.float32), callable_output[0])
+
+    debug_dumps = wrapped_sess.observers["debug_dumps"]
+    self.assertEqual(1, len(debug_dumps))
+    debug_dump = debug_dumps[0]
+    node_names = [datum.node_name for datum in debug_dump.dumped_tensor_data]
+    self.assertItemsEqual(
+        ["callable_a", "callable_b", "variable_1", "variable_1/read"],
+        node_names)
+
+  def testRuntimeErrorShouldBeCaught(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess, dump_root=self._tmp_dir)
+
+    # Do a run that should lead to an TensorFlow runtime error.
+    wrapped_sess.run(self.y, feed_dict={self.ph: [[0.0], [1.0], [2.0]]})
+
+    self.assertEqual([1], wrapped_sess.observers["run_start_cli_run_numbers"])
+    self.assertEqual([1], wrapped_sess.observers["run_end_cli_run_numbers"])
+    self.assertEqual(1, len(wrapped_sess.observers["debug_dumps"]))
+
+    # Verify that the runtime error is caught by the wrapped session properly.
+    self.assertEqual(1, len(wrapped_sess.observers["tf_errors"]))
+    tf_error = wrapped_sess.observers["tf_errors"][0]
+    self.assertEqual("y", tf_error.op.name)
+
+  def testRunTillFilterPassesShouldLaunchCLIAtCorrectRun(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-f", "v_greater_than_twelve"],
+         ["run", "-f", "v_greater_than_twelve"],
+         ["run"]],
+        self.sess,
+        dump_root=self._tmp_dir)
+
+    def v_greater_than_twelve(datum, tensor):
+      return datum.node_name == "v" and tensor > 12.0
+
+    # Verify that adding the same tensor filter more than once is tolerated
+    # (i.e., as if it were added only once).
+    wrapped_sess.add_tensor_filter("v_greater_than_twelve",
+                                   v_greater_than_twelve)
+    wrapped_sess.add_tensor_filter("v_greater_than_twelve",
+                                   v_greater_than_twelve)
+
+    # run five times.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(15.0, self.sess.run(self.v))
+
+    self.assertEqual([1], wrapped_sess.observers["run_start_cli_run_numbers"])
+
+    # run-end CLI should NOT have been launched for run #2 and #3, because only
+    # starting from run #4 v becomes greater than 12.0.
+    self.assertEqual([4, 5], wrapped_sess.observers["run_end_cli_run_numbers"])
+
+    self.assertEqual(2, len(wrapped_sess.observers["debug_dumps"]))
+    self.assertEqual([None, None], wrapped_sess.observers["tf_errors"])
+
+  def testRunTillFilterPassesWithExcludeOpNames(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-f", "greater_than_twelve",
+          "--filter_exclude_node_names", "inc_v.*"],
+         ["run"], ["run"]],
+        self.sess,
+        dump_root=self._tmp_dir)
+
+    def greater_than_twelve(datum, tensor):
+      del datum  # Unused.
+      return tensor > 12.0
+
+    # Verify that adding the same tensor filter more than once is tolerated
+    # (i.e., as if it were added only once).
+    wrapped_sess.add_tensor_filter("greater_than_twelve", greater_than_twelve)
+
+    # run five times.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(14.0, self.sess.run(self.v))
+
+    self.assertEqual([1], wrapped_sess.observers["run_start_cli_run_numbers"])
+
+    # Due to the --filter_exclude_op_names flag, the run-end CLI should show up
+    # not after run 3, but after run 4.
+    self.assertEqual([4], wrapped_sess.observers["run_end_cli_run_numbers"])
+
+  def testRunTillFilterPassesWorksInConjunctionWithOtherNodeNameFilter(self):
+    """Test that --.*_filter flags work in conjunction with -f.
+
+    In other words, test that you can use a tensor filter on a subset of
+    the tensors.
+    """
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-f", "v_greater_than_twelve", "--node_name_filter", "v$"],
+         ["run", "-f", "v_greater_than_twelve", "--node_name_filter", "v$"],
+         ["run"]],
+        self.sess,
+        dump_root=self._tmp_dir)
+
+    def v_greater_than_twelve(datum, tensor):
+      return datum.node_name == "v" and tensor > 12.0
+    wrapped_sess.add_tensor_filter("v_greater_than_twelve",
+                                   v_greater_than_twelve)
+
+    # run five times.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(15.0, self.sess.run(self.v))
+
+    self.assertEqual([1], wrapped_sess.observers["run_start_cli_run_numbers"])
+
+    # run-end CLI should NOT have been launched for run #2 and #3, because only
+    # starting from run #4 v becomes greater than 12.0.
+    self.assertEqual([4, 5], wrapped_sess.observers["run_end_cli_run_numbers"])
+
+    debug_dumps = wrapped_sess.observers["debug_dumps"]
+    self.assertEqual(2, len(debug_dumps))
+    self.assertEqual(1, len(debug_dumps[0].dumped_tensor_data))
+    self.assertEqual("v:0", debug_dumps[0].dumped_tensor_data[0].tensor_name)
+    self.assertEqual(1, len(debug_dumps[1].dumped_tensor_data))
+    self.assertEqual("v:0", debug_dumps[1].dumped_tensor_data[0].tensor_name)
+
+  def testRunsUnderDebugModeWithWatchFnFilteringNodeNames(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "--node_name_filter", "inc.*"],
+         ["run", "--node_name_filter", "delta"],
+         ["run"]],
+        self.sess, dump_root=self._tmp_dir)
+
+    # run under debug mode twice.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    # Verify that the assign_add op did take effect.
+    self.assertAllClose(12.0, self.sess.run(self.v))
+
+    # Verify that the dumps have been generated and picked up during run-end.
+    self.assertEqual(2, len(wrapped_sess.observers["debug_dumps"]))
+
+    dumps = wrapped_sess.observers["debug_dumps"][0]
+    self.assertEqual(1, dumps.size)
+    self.assertEqual("inc_v", dumps.dumped_tensor_data[0].node_name)
+
+    dumps = wrapped_sess.observers["debug_dumps"][1]
+    self.assertEqual(1, dumps.size)
+    self.assertEqual("delta", dumps.dumped_tensor_data[0].node_name)
+
+  def testRunsUnderDebugModeWithWatchFnFilteringOpTypes(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "--node_name_filter", "delta"],
+         ["run", "--op_type_filter", "AssignAdd"],
+         ["run"]],
+        self.sess, dump_root=self._tmp_dir)
+
+    # run under debug mode twice.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    # Verify that the assign_add op did take effect.
+    self.assertAllClose(12.0, self.sess.run(self.v))
+
+    # Verify that the dumps have been generated and picked up during run-end.
+    self.assertEqual(2, len(wrapped_sess.observers["debug_dumps"]))
+
+    dumps = wrapped_sess.observers["debug_dumps"][0]
+    self.assertEqual(1, dumps.size)
+    self.assertEqual("delta", dumps.dumped_tensor_data[0].node_name)
+
+    dumps = wrapped_sess.observers["debug_dumps"][1]
+    self.assertEqual(1, dumps.size)
+    self.assertEqual("inc_v", dumps.dumped_tensor_data[0].node_name)
+
+  def testRunsUnderDebugModeWithWatchFnFilteringTensorDTypes(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "--op_type_filter", "Variable.*"],
+         ["run", "--tensor_dtype_filter", "int32"],
+         ["run"]],
+        self.sess, dump_root=self._tmp_dir)
+
+    # run under debug mode twice.
+    wrapped_sess.run(self.w_int)
+    wrapped_sess.run(self.w_int)
+
+    # Verify that the dumps have been generated and picked up during run-end.
+    self.assertEqual(2, len(wrapped_sess.observers["debug_dumps"]))
+
+    dumps = wrapped_sess.observers["debug_dumps"][0]
+    self.assertEqual(2, dumps.size)
+    self.assertItemsEqual(
+        ["v", "w"], [dumps.dumped_tensor_data[i].node_name for i in [0, 1]])
+
+    dumps = wrapped_sess.observers["debug_dumps"][1]
+    self.assertEqual(2, dumps.size)
+    self.assertEqual(
+        ["w_int_inner", "w_int_outer"],
+        [dumps.dumped_tensor_data[i].node_name for i in [0, 1]])
+
+  def testRunsUnderDebugModeWithWatchFnFilteringOpTypesAndTensorDTypes(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "--op_type_filter", "Cast", "--tensor_dtype_filter", "int32"],
+         ["run"]],
+        self.sess, dump_root=self._tmp_dir)
+
+    # run under debug mode twice.
+    wrapped_sess.run(self.w_int)
+
+    # Verify that the dumps have been generated and picked up during run-end.
+    self.assertEqual(1, len(wrapped_sess.observers["debug_dumps"]))
+
+    dumps = wrapped_sess.observers["debug_dumps"][0]
+    self.assertEqual(1, dumps.size)
+    self.assertEqual("w_int_inner", dumps.dumped_tensor_data[0].node_name)
+
+  def testPrintFeedPrintsFeedValueForTensorFeedKey(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["print_feed", "ph:0"], ["run"], ["run"]], self.sess)
+
+    self.assertAllClose(
+        [[5.0], [-1.0]],
+        wrapped_sess.run(self.y, feed_dict={self.ph: [[0.0, 1.0, 2.0]]}))
+    print_feed_responses = wrapped_sess.observers["print_feed_responses"]
+    self.assertEqual(1, len(print_feed_responses))
+    self.assertEqual(
+        ["Tensor \"ph:0 (feed)\":", "", "[[0.0, 1.0, 2.0]]"],
+        print_feed_responses[0].lines)
+
+  def testPrintFeedPrintsFeedValueForTensorNameFeedKey(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["print_feed", "ph:0"], ["run"], ["run"]], self.sess)
+
+    self.assertAllClose(
+        [[5.0], [-1.0]],
+        wrapped_sess.run(self.y, feed_dict={"ph:0": [[0.0, 1.0, 2.0]]}))
+    print_feed_responses = wrapped_sess.observers["print_feed_responses"]
+    self.assertEqual(1, len(print_feed_responses))
+    self.assertEqual(
+        ["Tensor \"ph:0 (feed)\":", "", "[[0.0, 1.0, 2.0]]"],
+        print_feed_responses[0].lines)
+
+  def testPrintFeedPrintsErrorForInvalidFeedKey(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["print_feed", "spam"], ["run"], ["run"]], self.sess)
+
+    self.assertAllClose(
+        [[5.0], [-1.0]],
+        wrapped_sess.run(self.y, feed_dict={"ph:0": [[0.0, 1.0, 2.0]]}))
+    print_feed_responses = wrapped_sess.observers["print_feed_responses"]
+    self.assertEqual(1, len(print_feed_responses))
+    self.assertEqual(
+        ["ERROR: The feed_dict of the current run does not contain the key "
+         "spam"], print_feed_responses[0].lines)
+
+  def testPrintFeedPrintsErrorWhenFeedDictIsNone(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["print_feed", "spam"], ["run"], ["run"]], self.sess)
+
+    wrapped_sess.run(self.w_int)
+    print_feed_responses = wrapped_sess.observers["print_feed_responses"]
+    self.assertEqual(1, len(print_feed_responses))
+    self.assertEqual(
+        ["ERROR: The feed_dict of the current run is None or empty."],
+        print_feed_responses[0].lines)
+
+  def testRunUnderProfilerModeWorks(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-p"], ["run"]], self.sess)
+
+    wrapped_sess.run(self.w_int)
+
+    self.assertEqual(1, len(wrapped_sess.observers["profiler_run_metadata"]))
+    self.assertTrue(
+        wrapped_sess.observers["profiler_run_metadata"][0].step_stats)
+    self.assertEqual(1, len(wrapped_sess.observers["profiler_py_graphs"]))
+    self.assertIsInstance(
+        wrapped_sess.observers["profiler_py_graphs"][0], ops.Graph)
+
+  def testCallingHookDelBeforeAnyRun(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess)
+    del wrapped_sess
+
+  def testCallingShouldStopMethodOnNonWrappedNonMonitoredSessionErrors(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], self.sess)
+    with self.assertRaisesRegex(
+        ValueError,
+        r"The wrapped session .* does not have a method .*should_stop.*"):
+      wrapped_sess.should_stop()
+
+  def testLocalCLIDebugWrapperSessionWorksOnMonitoredSession(self):
+    monitored_sess = monitored_session.MonitoredSession()
+    wrapped_monitored_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"], ["run"]], monitored_sess)
+    self.assertFalse(wrapped_monitored_sess.should_stop())
+
+  def testRunsWithEmptyFetchWorks(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"]], self.sess, dump_root="")
+
+    run_output = wrapped_sess.run([])
+    self.assertEqual([], run_output)
+
+  def testRunsWithEmptyNestedFetchWorks(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run"]], self.sess, dump_root="")
+
+    run_output = wrapped_sess.run({"foo": {"baz": []}, "bar": ()})
+    self.assertEqual({"foo": {"baz": []}, "bar": ()}, run_output)
+
+  def testSessionRunHook(self):
+    a = array_ops.placeholder(dtypes.float32, [10])
+    b = a + 1
+    c = b * 2
+
+    class Hook(session_run_hook.SessionRunHook):
+
+      def before_run(self, _):
+        return session_run_hook.SessionRunArgs(fetches=c)
+
+    class Hook2(session_run_hook.SessionRunHook):
+
+      def before_run(self, _):
+        return session_run_hook.SessionRunArgs(fetches=b)
+
+    sess = session.Session()
+    sess = LocalCLIDebuggerWrapperSessionForTest([["run"], ["run"]], sess)
+
+    class SessionCreator(object):
+
+      def create_session(self):
+        return sess
+
+    final_sess = monitored_session.MonitoredSession(
+        session_creator=SessionCreator(), hooks=[Hook(), Hook2()])
+
+    final_sess.run(b, feed_dict={a: np.arange(10)})
+    debug_dumps = sess.observers["debug_dumps"]
+    self.assertEqual(1, len(debug_dumps))
+    debug_dump = debug_dumps[0]
+    node_names = [datum.node_name for datum in debug_dump.dumped_tensor_data]
+    self.assertIn(b.op.name, node_names)
+
+
+if __name__ == "__main__":
+  googletest.main()