chore: import upstream snapshot with attribution

tensorflow/tools/common/BUILD

@@ -0,0 +1,82 @@
+# Description:
+#   Common functionality for TensorFlow tooling
+
+load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
+load("//tensorflow:tensorflow.bzl", "VERSION_MAJOR", "py_test")
+load("//tensorflow/core/platform:build_config_root.bzl", "if_pywrap")
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
+    default_visibility = [
+        "//tensorflow:__subpackages__",
+        "//third_party/py/tf_keras/api:__subpackages__",
+    ],
+    licenses = ["notice"],
+)
+
+py_library(
+    name = "public_api",
+    srcs = ["public_api.py"],
+    strict_deps = True,
+    deps = ["//tensorflow/python/util:tf_inspect"],
+)
+
+py_test(
+    name = "public_api_test",
+    srcs = ["public_api_test.py"],
+    data = if_pywrap(
+        if_false = select({
+            "//tensorflow:macos": ["//tensorflow:libtensorflow_framework.dylib"],
+            "//tensorflow:windows": [],
+            "//conditions:default": ["//tensorflow:libtensorflow_framework.so.%s" % VERSION_MAJOR],
+        }),
+    ),
+    strict_deps = True,
+    deps = [
+        ":public_api",
+        # copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
+        "//tensorflow/python/platform:test",
+    ],
+)
+
+py_library(
+    name = "traverse",
+    srcs = ["traverse.py"],
+    strict_deps = True,
+    deps = ["//tensorflow/python/util:tf_inspect"],
+)
+
+py_test(
+    name = "traverse_test",
+    srcs = ["traverse_test.py"],
+    data = if_pywrap(
+        if_false = select({
+            "//tensorflow:macos": ["//tensorflow:libtensorflow_framework.dylib"],
+            "//tensorflow:windows": [],
+            "//conditions:default": ["//tensorflow:libtensorflow_framework.so.%s" % VERSION_MAJOR],
+        }),
+    ),
+    strict_deps = True,
+    deps = [
+        ":test_module1",
+        ":test_module2",
+        ":traverse",
+        # copybara:uncomment "//third_party/py/google/protobuf:use_fast_cpp_protos",
+        "//tensorflow/python/platform:test",
+    ],
+)
+
+py_library(
+    name = "test_module1",
+    srcs = ["test_module1.py"],
+    strict_deps = True,
+    deps = [
+        ":test_module2",
+    ],
+)
+
+py_library(
+    name = "test_module2",
+    srcs = ["test_module2.py"],
+    strict_deps = True,
+)

tensorflow/tools/common/public_api.py

@@ -0,0 +1,147 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Visitor restricting traversal to only the public tensorflow API."""
+
+import re
+
+from tensorflow.python.util import tf_inspect
+
+
+class PublicAPIVisitor:
+  """Visitor to use with `traverse` to visit exactly the public TF API."""
+
+  def __init__(self, visitor):
+    """Constructor.
+
+    `visitor` should be a callable suitable as a visitor for `traverse`. It will
+    be called only for members of the public TensorFlow API.
+
+    Args:
+      visitor: A visitor to call for the public API.
+    """
+    self._visitor = visitor
+    self._root_name = 'tf'
+
+    # Modules/classes we want to suppress entirely.
+    self._private_map = {
+        'tf': [
+            'compiler',
+            'core',
+            # TODO(scottzhu): See b/227410870 for more details. Currently
+            # dtensor API is exposed under tf.experimental.dtensor, but in the
+            # meantime, we have tensorflow/dtensor directory which will be treat
+            # as a python package. We want to avoid step into the
+            # tensorflow/dtensor directory when visit the API.
+            # When the tf.dtensor becomes the public API, it will actually pick
+            # up from tf.compat.v2.dtensor as priority and hide the
+            # tensorflow/dtensor package.
+            'security',
+            'dtensor',
+            'python',
+            'tsl',  # TODO(tlongeri): Remove after TSL is moved out of TF.
+        ],
+        # Some implementations have this internal module that we shouldn't
+        # expose.
+        'tf.flags': ['cpp_flags'],
+    }
+
+    # Modules/classes we do not want to descend into if we hit them. Usually,
+    # system modules exposed through platforms for compatibility reasons.
+    # Each entry maps a module path to a name to ignore in traversal.
+    self._do_not_descend_map = {
+        'tf': [
+            'examples',
+            'flags',  # Don't add flags
+            # TODO(drpng): This can be removed once sealed off.
+            'platform',
+            # TODO(drpng): This can be removed once sealed.
+            'pywrap_tensorflow',
+            # TODO(drpng): This can be removed once sealed.
+            'user_ops',
+            'tools',
+            'tensorboard',
+        ],
+
+        ## Everything below here is legitimate.
+        # It'll stay, but it's not officially part of the API.
+        'tf.app': ['flags'],
+        # Imported for compatibility between py2/3.
+        'tf.test': ['mock'],
+    }
+
+  @property
+  def private_map(self):
+    """A map from parents to symbols that should not be included at all.
+
+    This map can be edited, but it should not be edited once traversal has
+    begun.
+
+    Returns:
+      The map marking symbols to not include.
+    """
+    return self._private_map
+
+  @property
+  def do_not_descend_map(self):
+    """A map from parents to symbols that should not be descended into.
+
+    This map can be edited, but it should not be edited once traversal has
+    begun.
+
+    Returns:
+      The map marking symbols to not explore.
+    """
+    return self._do_not_descend_map
+
+  def set_root_name(self, root_name):
+    """Override the default root name of 'tf'."""
+    self._root_name = root_name
+
+  def _is_private(self, path, name, obj=None):
+    """Return whether a name is private."""
+    # TODO(wicke): Find out what names to exclude.
+    del obj  # Unused.
+    return ((path in self._private_map and name in self._private_map[path]) or
+            (name.startswith('_') and not re.match('__.*__$', name) or
+             name in ['__base__', '__class__', '__next_in_mro__']))
+
+  def _do_not_descend(self, path, name):
+    """Safely queries if a specific fully qualified name should be excluded."""
+    return (path in self._do_not_descend_map and
+            name in self._do_not_descend_map[path])
+
+  def __call__(self, path, parent, children):
+    """Visitor interface, see `traverse` for details."""
+
+    # Avoid long waits in cases of pretty unambiguous failure.
+    if tf_inspect.ismodule(parent) and len(path.split('.')) > 10:
+      raise RuntimeError('Modules nested too deep:\n%s.%s\n\nThis is likely a '
+                         'problem with an accidental public import.' %
+                         (self._root_name, path))
+
+    # Includes self._root_name
+    full_path = '.'.join([self._root_name, path]) if path else self._root_name
+
+    # Remove things that are not visible.
+    for name, child in list(children):
+      if self._is_private(full_path, name, child):
+        children.remove((name, child))
+
+    self._visitor(path, parent, children)
+
+    # Remove things that are visible, but which should not be descended into.
+    for name, child in list(children):
+      if self._do_not_descend(full_path, name):
+        children.remove((name, child))

tensorflow/tools/common/public_api_test.py

@@ -0,0 +1,64 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow.tools.common.public_api."""
+
+from tensorflow.python.platform import googletest
+from tensorflow.tools.common import public_api
+
+
+class PublicApiTest(googletest.TestCase):
+
+  class TestVisitor(object):
+
+    def __init__(self):
+      self.symbols = set()
+      self.last_parent = None
+      self.last_children = None
+
+    def __call__(self, path, parent, children):
+      self.symbols.add(path)
+      self.last_parent = parent
+      self.last_children = list(children)  # Make a copy to preserve state.
+
+  def test_call_forward(self):
+    visitor = self.TestVisitor()
+    children = [('name1', 'thing1'), ('name2', 'thing2')]
+    public_api.PublicAPIVisitor(visitor)('test', 'dummy', children)
+    self.assertEqual(set(['test']), visitor.symbols)
+    self.assertEqual('dummy', visitor.last_parent)
+    self.assertEqual([('name1', 'thing1'), ('name2', 'thing2')],
+                     visitor.last_children)
+
+  def test_private_child_removal(self):
+    visitor = self.TestVisitor()
+    children = [('name1', 'thing1'), ('_name2', 'thing2')]
+    public_api.PublicAPIVisitor(visitor)('test', 'dummy', children)
+    # Make sure the private symbols are removed before the visitor is called.
+    self.assertEqual([('name1', 'thing1')], visitor.last_children)
+    self.assertEqual([('name1', 'thing1')], children)
+
+  def test_no_descent_child_removal(self):
+    visitor = self.TestVisitor()
+    children = [('name1', 'thing1'), ('mock', 'thing2')]
+    public_api.PublicAPIVisitor(visitor)('test', 'dummy', children)
+    # Make sure not-to-be-descended-into symbols are removed after the visitor
+    # is called.
+    self.assertEqual([('name1', 'thing1'), ('mock', 'thing2')],
+                     visitor.last_children)
+    self.assertEqual([('name1', 'thing1')], children)
+
+
+if __name__ == '__main__':
+  googletest.main()

tensorflow/tools/common/test_module1.py

@@ -0,0 +1,27 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A module target for TraverseTest.test_module."""
+
+from tensorflow.tools.common import test_module2
+
+
+class ModuleClass1(object):
+
+  def __init__(self):
+    self._m2 = test_module2.ModuleClass2()
+
+  def __model_class1_method__(self):
+    pass
+

tensorflow/tools/common/test_module2.py

@@ -0,0 +1,25 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A module target for TraverseTest.test_module."""
+
+
+class ModuleClass2(object):
+
+  def __init__(self):
+    pass
+
+  def __model_class1_method__(self):
+    pass
+

tensorflow/tools/common/traverse.py

@@ -0,0 +1,103 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Traversing Python modules and classes."""
+
+import enum
+import sys
+
+from tensorflow.python.util import tf_inspect
+
+__all__ = ['traverse']
+
+
+def _traverse_internal(root, visit, stack, path):
+  """Internal helper for traverse."""
+
+  # Only traverse modules and classes
+  if not tf_inspect.isclass(root) and not tf_inspect.ismodule(root):
+    return
+
+  try:
+    children = tf_inspect.getmembers(root)
+
+    # Add labels for duplicate values in Enum.
+    if tf_inspect.isclass(root) and issubclass(root, enum.Enum):
+      for enum_member in root.__members__.items():
+        if enum_member not in children:
+          children.append(enum_member)
+      children = sorted(children)
+  except ImportError:
+    # Children could be missing for one of two reasons:
+    # 1. On some Python installations, some modules do not support enumerating
+    #    members, leading to import errors.
+    # 2. Children are lazy-loaded.
+    try:
+      children = []
+      for child_name in root.__all__:
+        children.append((child_name, getattr(root, child_name)))
+    except AttributeError:
+      children = []
+
+  new_stack = stack + [root]
+  visit(path, root, children)
+  for name, child in children:
+    # Do not descend into built-in modules
+    if tf_inspect.ismodule(
+        child) and child.__name__ in sys.builtin_module_names:
+      continue
+
+    # Break cycles
+    if any(child is item for item in new_stack):  # `in`, but using `is`
+      continue
+
+    child_path = path + '.' + name if path else name
+    _traverse_internal(child, visit, new_stack, child_path)
+
+
+def traverse(root, visit):
+  """Recursively enumerate all members of `root`.
+
+  Similar to the Python library function `os.path.walk`.
+
+  Traverses the tree of Python objects starting with `root`, depth first.
+  Parent-child relationships in the tree are defined by membership in modules or
+  classes. The function `visit` is called with arguments
+  `(path, parent, children)` for each module or class `parent` found in the tree
+  of python objects starting with `root`. `path` is a string containing the name
+  with which `parent` is reachable from the current context. For example, if
+  `root` is a local class called `X` which contains a class `Y`, `visit` will be
+  called with `('Y', X.Y, children)`).
+
+  If `root` is not a module or class, `visit` is never called. `traverse`
+  never descends into built-in modules.
+
+  `children`, a list of `(name, object)` pairs are determined by
+  `tf_inspect.getmembers`. To avoid visiting parts of the tree, `children` can
+  be modified in place, using `del` or slice assignment.
+
+  Cycles (determined by reference equality, `is`) stop the traversal. A stack of
+  objects is kept to find cycles. Objects forming cycles may appear in
+  `children`, but `visit` will not be called with any object as `parent` which
+  is already in the stack.
+
+  Traversing system modules can take a long time, it is advisable to pass a
+  `visit` callable which denylists such modules.
+
+  Args:
+    root: A python object with which to start the traversal.
+    visit: A function taking arguments `(path, parent, children)`. Will be
+      called for each object found in the traversal.
+  """
+  _traverse_internal(root, visit, [], '')

tensorflow/tools/common/traverse_test.py

@@ -0,0 +1,75 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Python module traversal."""
+
+from tensorflow.python.platform import googletest
+from tensorflow.tools.common import test_module1
+from tensorflow.tools.common import test_module2
+from tensorflow.tools.common import traverse
+
+
+class TestVisitor(object):
+
+  def __init__(self):
+    self.call_log = []
+
+  def __call__(self, path, parent, children):
+    self.call_log += [(path, parent, children)]
+
+
+class TraverseTest(googletest.TestCase):
+
+  def test_cycle(self):
+
+    class Cyclist(object):
+      pass
+    Cyclist.cycle = Cyclist
+
+    visitor = TestVisitor()
+    traverse.traverse(Cyclist, visitor)
+    # We simply want to make sure we terminate.
+
+  def test_module(self):
+    visitor = TestVisitor()
+    traverse.traverse(test_module1, visitor)
+
+    called = [parent for _, parent, _ in visitor.call_log]
+
+    self.assertIn(test_module1.ModuleClass1, called)
+    self.assertIn(test_module2.ModuleClass2, called)
+
+  def test_class(self):
+    visitor = TestVisitor()
+    traverse.traverse(TestVisitor, visitor)
+    self.assertEqual(TestVisitor,
+                     visitor.call_log[0][1])
+    # There are a bunch of other members, but make sure that the ones we know
+    # about are there.
+    self.assertIn('__init__', [name for name, _ in visitor.call_log[0][2]])
+    self.assertIn('__call__', [name for name, _ in visitor.call_log[0][2]])
+
+    # There are more classes descended into, at least __class__ and
+    # __class__.__base__, neither of which are interesting to us, and which may
+    # change as part of Python version etc., so we don't test for them.
+
+  def test_non_class(self):
+    integer = 5
+    visitor = TestVisitor()
+    traverse.traverse(integer, visitor)
+    self.assertEqual([], visitor.call_log)
+
+
+if __name__ == '__main__':
+  googletest.main()