wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/tools/docs/BUILD
+237
View File
@@ -0,0 +1,237 @@
# Description:
# Doc generator
load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary")
load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
load("//tensorflow:tensorflow.bzl", "py_test")
load("//tensorflow:tensorflow.default.bzl", "get_compatible_with_portable")
load(
"//tensorflow/core/platform:build_config_root.bzl",
"tf_gpu_tests_tags",
)
load("//tensorflow/python/tpu:tpu.bzl", "tpu_py_strict_test")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:__subpackages__"],
licenses = ["notice"],
)
tpu_module = [
"tpu.",
"distribute.tpu_strategy",
"distribute.cluster_resolver.tpu",
"distribute.cluster_resolver.tpu_oss",
]
keras_module = [
"keras.",
]
# tf.distribute docstring often uses GPU, so they're only covered in
# tf_doctest_gpu.
distribute_module = [
"distribute.",
]
py_library(
name = "tf_doctest_lib",
srcs = ["tf_doctest_lib.py"],
strict_deps = True,
visibility = [
"//tensorflow:__subpackages__",
"//tensorflow_text/google:__pkg__",
],
deps = [
"//third_party/py/numpy",
],
)
py_test(
name = "tf_doctest",
srcs = ["tf_doctest.py"],
args = ["--module_prefix_skip=" + ",".join(tpu_module + distribute_module + keras_module)],
shard_count = 4,
strict_deps = True,
tags = [
"no_oss", # b/275546007
"no_pip",
"no_rocm", # No need to rerun this test for ROCm config.
"no_windows", # numpy prints differently on windows.
"noasan",
],
deps = [
":tf_doctest_lib",
"//tensorflow:tensorflow_py",
"//tensorflow/python/eager:context",
"//tensorflow/python/ops:logging_ops",
"//third_party/py/numpy",
"@absl_py//absl/flags",
"@absl_py//absl/testing:absltest",
],
)
tpu_py_strict_test(
name = "tf_doctest_tpu",
srcs = ["tf_doctest.py"],
args = ["--module=" + ",".join(tpu_module)],
disable_experimental = True,
disable_tfrt = True,
disable_v3 = True,
main = "tf_doctest.py",
tags = ["no_oss"],
deps = [
":tf_doctest_lib",
"//tensorflow:tensorflow_py_no_contrib",
"//tensorflow/python/eager:context",
"//tensorflow/python/ops:logging_ops",
"//third_party/py/numpy",
"@absl_py//absl/flags",
"@absl_py//absl/testing:absltest",
],
)
py_test(
name = "tf_doctest_gpu",
srcs = ["tf_doctest.py"],
args = [
"--module=distribute.",
"--module_prefix_skip=" + ",".join(tpu_module),
"--required_gpus=2",
],
main = "tf_doctest.py",
strict_deps = True,
tags = [
"no_pip",
"no_rocm",
"no_windows", # numpy prints differently on windows.
"noasan",
"nomsan",
"notsan",
] + tf_gpu_tests_tags(),
deps = [
":tf_doctest_lib",
"//tensorflow:tensorflow_py",
"//tensorflow/python/eager:context",
"//tensorflow/python/ops:logging_ops",
"//third_party/py/numpy",
"@absl_py//absl/flags",
"@absl_py//absl/testing:absltest",
],
)
py_test(
name = "tf_doctest_test",
srcs = ["tf_doctest_test.py"],
strict_deps = True,
tags = ["no_pip"],
deps = [
":tf_doctest_lib",
"@absl_py//absl/testing:absltest",
"@absl_py//absl/testing:parameterized",
],
)
py_library(
name = "fenced_doctest_lib",
srcs = ["fenced_doctest_lib.py"],
strict_deps = True,
deps = [
":tf_doctest_lib",
"@pypi//astor",
],
)
py_test(
name = "fenced_doctest_test",
srcs = ["fenced_doctest_test.py"],
strict_deps = True,
tags = [
"no_oss",
"no_pip",
],
deps = [
":fenced_doctest_lib",
"@absl_py//absl/testing:absltest",
"@absl_py//absl/testing:parameterized",
],
)
py_library(
name = "doc_controls",
srcs = ["doc_controls.py"],
compatible_with = get_compatible_with_portable(),
strict_deps = True,
visibility = ["//visibility:public"],
)
py_test(
name = "generate2_test",
size = "medium",
srcs = ["generate2_test.py"],
strict_deps = True,
tags = [
"manual",
"optonly",
],
deps = [
":generate2_lib",
"//tensorflow:tensorflow_py",
"//tensorflow/python/platform:test",
"@pypi//packaging",
],
)
py_binary(
name = "generate2",
srcs = ["generate2.py"],
strict_deps = True,
deps = [
":generate2_lib",
],
)
py_library(
# Opensource only
name = "base_dir_oss",
srcs = ["base_dir.py"],
strict_deps = False,
deps = [],
)
py_library(
name = "generate2_lib",
srcs = ["generate2.py"],
strict_deps = True,
deps = [
":base_dir_oss",
"//tensorflow:tensorflow_py_no_contrib",
"//tensorflow/python/framework:ops",
"//tensorflow/python/util:pywrap_xla_ops",
"//tensorflow/python/util:tf_export",
"//tensorflow/python/util:tf_inspect",
"@absl_py//absl:app",
"@absl_py//absl/flags",
"@pypi//packaging",
],
)
py_binary(
name = "build_cc_api_headers",
srcs = ["build_cc_api_headers.py"],
strict_deps = True,
deps = [
"@absl_py//absl:app",
"@absl_py//absl/flags",
],
)
py_binary(
name = "build_java_api_docs",
srcs = ["build_java_api_docs.py"],
strict_deps = True,
deps = [
"@absl_py//absl:app",
"@absl_py//absl/flags",
],
)
tensorflow/tools/docs/base_dir.py
+129
View File
@@ -0,0 +1,129 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Opensource base_dir configuration for tensorflow doc-generator."""
import pathlib
import keras
from packaging import version
import tensorboard
import tensorflow as tf
from tensorflow_docs.api_generator import public_api
try:
import tensorflow_estimator # pylint: disable=[g-import-not-at-top, g-deprecated-tf-checker]
except ImportError:
tensorflow_estimator = None
def get_base_dirs_and_prefixes(code_url_prefix):
"""Returns the base_dirs and code_prefixes for OSS TensorFlow api gen."""
base_dir = pathlib.Path(tf.__file__).parent
if "dev" in tf.__version__:
keras_url_prefix = "https://github.com/keras-team/keras/tree/master/keras/src"
else:
keras_url_prefix = (
f"https://github.com/keras-team/keras/tree/v{keras.__version__}/keras/src"
)
if version.parse(tf.__version__) >= version.parse("2.16"):
# First match takes precedence.
# Objects are dropped if they have no match.
base_dirs = [
# The real keras source files are now in `site-packages/keras/src/...`
pathlib.Path(keras.__file__).parent / "src",
# The generated module files in tensorflow are in keras
# under `site-packages/keras/api/_v2/keras/...`.
pathlib.Path(tf.keras.__file__).parent,
# The generated api-module files are now in `site-packages/keras/...`
pathlib.Path(keras.__file__).parent,
pathlib.Path(tensorboard.__file__).parent,
# The tensorflow base dir goes last because `tf.keras``
base_dir,
]
code_url_prefixes = (
keras_url_prefix,
# None -> don't link to the generated keras api-module files.
None,
None,
f"https://github.com/tensorflow/tensorboard/tree/{tensorboard.__version__}/tensorboard",
code_url_prefix,
)
elif version.parse(tf.__version__) >= version.parse("2.13"):
# First match takes precedence.
# Objects are dropped if they have no match.
base_dirs = [
# The real keras source files are now in `site-packages/keras/src/...`
pathlib.Path(keras.__file__).parent / "src",
# The generated module files in tensorflow are in keras
# under `site-packages/keras/api/_v2/keras/...`.
pathlib.Path(tf.keras.__file__).parent,
# The generated api-module files are now in `site-packages/keras/...`
pathlib.Path(keras.__file__).parent,
pathlib.Path(tensorboard.__file__).parent,
pathlib.Path(tensorflow_estimator.__file__).parent,
# The tensorflow base dir goes last because `tf.keras``
base_dir,
]
code_url_prefixes = (
keras_url_prefix,
# None -> don't link to the generated keras api-module files.
None,
None,
f"https://github.com/tensorflow/tensorboard/tree/{tensorboard.__version__}/tensorboard",
"https://github.com/tensorflow/estimator/tree/master/tensorflow_estimator",
code_url_prefix,
)
elif version.parse(tf.__version__) >= version.parse("2.9"):
base_dirs = [
base_dir,
pathlib.Path(keras.__file__).parent,
pathlib.Path(tensorboard.__file__).parent,
pathlib.Path(tensorflow_estimator.__file__).parent,
]
code_url_prefixes = (
code_url_prefix,
keras_url_prefix,
f"https://github.com/tensorflow/tensorboard/tree/{tensorboard.__version__}/tensorboard",
"https://github.com/tensorflow/estimator/tree/master/tensorflow_estimator",
)
else:
raise ValueError("Unsupported: version < 2.9")
return base_dirs, code_url_prefixes
def explicit_filter_keep_keras(parent_path, parent, children):
"""Like explicit_package_contents_filter, but keeps keras."""
new_children = public_api.explicit_package_contents_filter(
parent_path, parent, children)
if parent_path[-1] not in ["tf", "v1", "v2"]:
return new_children
had_keras = any(name == "keras" for name, child in children)
has_keras = any(name == "keras" for name, child in new_children)
if had_keras and not has_keras:
new_children.append(("keras", parent.keras))
return sorted(new_children, key=lambda x: x[0])
def get_callbacks():
return [explicit_filter_keep_keras]
tensorflow/tools/docs/build_cc_api_headers.py
+60
View File
@@ -0,0 +1,60 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Generate C++ reference docs for TensorFlow.org."""
import os
import pathlib
import subprocess
from absl import app
from absl import flags
FLAGS = flags.FLAGS
# These flags are required by infrastructure, not all of them are used.
flags.DEFINE_string('output_dir', None,
("Use this branch as the root version and don't"
' create in version directory'))
# __file__ is the path to this file
DOCS_TOOLS_DIR = pathlib.Path(__file__).resolve().parent
TENSORFLOW_ROOT = DOCS_TOOLS_DIR.parents[2]
def build_headers(output_dir):
"""Builds the headers files for TF."""
os.makedirs(output_dir, exist_ok=True)
# `$ yes | configure`
yes = subprocess.Popen(['yes', ''], stdout=subprocess.PIPE)
configure = subprocess.Popen([TENSORFLOW_ROOT / 'configure'],
stdin=yes.stdout,
cwd=TENSORFLOW_ROOT)
configure.communicate()
subprocess.check_call(['bazel', 'build', 'tensorflow/cc:cc_ops'],
cwd=TENSORFLOW_ROOT)
subprocess.check_call(
['cp', '--dereference', '-r', 'bazel-bin', output_dir / 'bazel-genfiles'],
cwd=TENSORFLOW_ROOT)
def main(argv):
del argv
build_headers(pathlib.Path(FLAGS.output_dir))
if __name__ == '__main__':
flags.mark_flags_as_required(['output_dir'])
app.run(main)
tensorflow/tools/docs/build_java_api_docs.py
+81
View File
@@ -0,0 +1,81 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Generate Java reference docs for TensorFlow.org."""
import pathlib
import shutil
import subprocess
import tempfile
from absl import app
from absl import flags
from tensorflow_docs.api_generator import gen_java
FLAGS = flags.FLAGS
# These flags are required by infrastructure, not all of them are used.
flags.DEFINE_string('output_dir', None,
("Use this branch as the root version and don't"
' create in version directory'))
flags.DEFINE_string('site_path', 'api_docs/java',
'Path prefix in the _toc.yaml')
flags.DEFINE_string('code_url_prefix', None,
'[UNUSED] The url prefix for links to code.')
flags.DEFINE_bool(
'search_hints', True,
'[UNUSED] Include metadata search hints in the generated files')
# Use this flag to disable bazel generation if you're not setup for it.
flags.DEFINE_bool('gen_ops', True, 'enable/disable bazel-generated ops')
# __file__ is the path to this file
DOCS_TOOLS_DIR = pathlib.Path(__file__).resolve().parent
TENSORFLOW_ROOT = DOCS_TOOLS_DIR.parents[2]
SOURCE_PATH = TENSORFLOW_ROOT / 'tensorflow/java/src/main/java'
OP_SOURCE_PATH = (
TENSORFLOW_ROOT /
'bazel-bin/tensorflow/java/ops/src/main/java/org/tensorflow/op')
def main(unused_argv):
merged_source = pathlib.Path(tempfile.mkdtemp())
shutil.copytree(SOURCE_PATH, merged_source / 'java')
if FLAGS.gen_ops:
# `$ yes | configure`
yes = subprocess.Popen(['yes', ''], stdout=subprocess.PIPE)
configure = subprocess.Popen([TENSORFLOW_ROOT / 'configure'],
stdin=yes.stdout,
cwd=TENSORFLOW_ROOT)
configure.communicate()
subprocess.check_call(
['bazel', 'build', '//tensorflow/java:java_op_gen_sources'],
cwd=TENSORFLOW_ROOT)
shutil.copytree(OP_SOURCE_PATH, merged_source / 'java/org/tensorflow/ops')
gen_java.gen_java_docs(
package='org.tensorflow',
source_path=merged_source / 'java',
output_dir=pathlib.Path(FLAGS.output_dir),
site_path=pathlib.Path(FLAGS.site_path))
if __name__ == '__main__':
flags.mark_flags_as_required(['output_dir'])
app.run(main)
tensorflow/tools/docs/doc_controls.py
+368
View File
@@ -0,0 +1,368 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Documentation control decorators."""
from typing import Optional, TypeVar
T = TypeVar("T")
_DEPRECATED = "_tf_docs_deprecated"
def set_deprecated(obj: T) -> T:
"""Explicitly tag an object as deprecated for the doc generator."""
setattr(obj, _DEPRECATED, None)
return obj
_INHERITABLE_HEADER = "_tf_docs_inheritable_header"
def inheritable_header(text):
def _wrapped(cls):
setattr(cls, _INHERITABLE_HEADER, text)
return cls
return _wrapped
def get_inheritable_header(obj) -> Optional[str]:
return getattr(obj, _INHERITABLE_HEADER, None)
header = inheritable_header
get_header = get_inheritable_header
_DO_NOT_DOC = "_tf_docs_do_not_document"
def do_not_generate_docs(obj: T) -> T:
"""A decorator: Do not generate docs for this object.
For example the following classes:
```
class Parent(object):
def method1(self):
pass
def method2(self):
pass
class Child(Parent):
def method1(self):
pass
def method2(self):
pass
```
Produce the following api_docs:
```
/Parent.md
# method1
# method2
/Child.md
# method1
# method2
```
This decorator allows you to skip classes or methods:
```
@do_not_generate_docs
class Parent(object):
def method1(self):
pass
def method2(self):
pass
class Child(Parent):
@do_not_generate_docs
def method1(self):
pass
def method2(self):
pass
```
This will only produce the following docs:
```
/Child.md
# method2
```
Note: This is implemented by adding a hidden attribute on the object, so it
cannot be used on objects which do not allow new attributes to be added. So
this decorator must go *below* `@property`, `@classmethod`,
or `@staticmethod`:
```
class Example(object):
@property
@do_not_generate_docs
def x(self):
return self._x
```
Args:
obj: The object to hide from the generated docs.
Returns:
obj
"""
setattr(obj, _DO_NOT_DOC, None)
return obj
_DO_NOT_DOC_INHERITABLE = "_tf_docs_do_not_doc_inheritable"
def do_not_doc_inheritable(obj: T) -> T:
"""A decorator: Do not generate docs for this method.
This version of the decorator is "inherited" by subclasses. No docs will be
generated for the decorated method in any subclass. Even if the sub-class
overrides the method.
For example, to ensure that `method1` is **never documented** use this
decorator on the base-class:
```
class Parent(object):
@do_not_doc_inheritable
def method1(self):
pass
def method2(self):
pass
class Child(Parent):
def method1(self):
pass
def method2(self):
pass
```
This will produce the following docs:
```
/Parent.md
# method2
/Child.md
# method2
```
When generating docs for a class's arributes, the `__mro__` is searched and
the attribute will be skipped if this decorator is detected on the attribute
on any class in the `__mro__`.
Note: This is implemented by adding a hidden attribute on the object, so it
cannot be used on objects which do not allow new attributes to be added. So
this decorator must go *below* `@property`, `@classmethod`,
or `@staticmethod`:
```
class Example(object):
@property
@do_not_doc_inheritable
def x(self):
return self._x
```
Args:
obj: The class-attribute to hide from the generated docs.
Returns:
obj
"""
setattr(obj, _DO_NOT_DOC_INHERITABLE, None)
return obj
_FOR_SUBCLASS_IMPLEMENTERS = "_tf_docs_tools_for_subclass_implementers"
def for_subclass_implementers(obj: T) -> T:
"""A decorator: Only generate docs for this method in the defining class.
Also group this method's docs with and `@abstractmethod` in the class's docs.
No docs will generated for this class attribute in sub-classes.
The canonical use case for this is `tf.keras.layers.Layer.call`: It's a
public method, essential for anyone implementing a subclass, but it should
never be called directly.
Works on method, or other class-attributes.
When generating docs for a class's arributes, the `__mro__` is searched and
the attribute will be skipped if this decorator is detected on the attribute
on any **parent** class in the `__mro__`.
For example:
```
class Parent(object):
@for_subclass_implementers
def method1(self):
pass
def method2(self):
pass
class Child1(Parent):
def method1(self):
pass
def method2(self):
pass
class Child2(Parent):
def method1(self):
pass
def method2(self):
pass
```
This will produce the following docs:
```
/Parent.md
# method1
# method2
/Child1.md
# method2
/Child2.md
# method2
```
Note: This is implemented by adding a hidden attribute on the object, so it
cannot be used on objects which do not allow new attributes to be added. So
this decorator must go *below* `@property`, `@classmethod`,
or `@staticmethod`:
```
class Example(object):
@property
@for_subclass_implementers
def x(self):
return self._x
```
Args:
obj: The class-attribute to hide from the generated docs.
Returns:
obj
"""
setattr(obj, _FOR_SUBCLASS_IMPLEMENTERS, None)
return obj
do_not_doc_in_subclasses = for_subclass_implementers
_DOC_PRIVATE = "_tf_docs_doc_private"
def doc_private(obj: T) -> T:
"""A decorator: Generates docs for private methods/functions.
For example:
```
class Try:
@doc_controls.doc_private
def _private(self):
...
```
As a rule of thumb, private(beginning with `_`) methods/functions are
not documented.
This decorator allows to force document a private method/function.
Args:
obj: The class-attribute to hide from the generated docs.
Returns:
obj
"""
setattr(obj, _DOC_PRIVATE, None)
return obj
_DOC_IN_CURRENT_AND_SUBCLASSES = "_tf_docs_doc_in_current_and_subclasses"
def doc_in_current_and_subclasses(obj: T) -> T:
"""Overrides `do_not_doc_in_subclasses` decorator.
If this decorator is set on a child class's method whose parent's method
contains `do_not_doc_in_subclasses`, then that will be overriden and the
child method will get documented. All classes inherting from the child will
also document that method.
For example:
```
class Parent:
@do_not_doc_in_subclasses
def method1(self):
pass
def method2(self):
pass
class Child1(Parent):
@doc_in_current_and_subclasses
def method1(self):
pass
def method2(self):
pass
class Child2(Parent):
def method1(self):
pass
def method2(self):
pass
class Child11(Child1):
pass
```
This will produce the following docs:
```
/Parent.md
# method1
# method2
/Child1.md
# method1
# method2
/Child2.md
# method2
/Child11.md
# method1
# method2
```
Args:
obj: The class-attribute to hide from the generated docs.
Returns:
obj
"""
setattr(obj, _DOC_IN_CURRENT_AND_SUBCLASSES, None)
return obj
tensorflow/tools/docs/fenced_doctest_lib.py
+239
View File
@@ -0,0 +1,239 @@
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Run doctests for tensorflow."""
import ast
import doctest
import os
import re
import textwrap
from typing import Any, Callable, Dict, Iterable, Optional
import astor
from tensorflow.tools.docs import tf_doctest_lib
def load_from_files(
files,
globs: Optional[Dict[str, Any]] = None,
set_up: Optional[Callable[[Any], None]] = None,
tear_down: Optional[Callable[[Any], None]] = None) -> doctest.DocFileSuite:
"""Creates a doctest suite from the files list.
Args:
files: A list of file paths to test.
globs: The global namespace the tests are run in.
set_up: Run before each test, receives the test as argument.
tear_down: Run after each test, receives the test as argument.
Returns:
A DocFileSuite containing the tests.
"""
if globs is None:
globs = {}
# __fspath__ isn't respected everywhere in doctest so convert paths to
# strings.
files = [os.fspath(f) for f in files]
globs['_print_if_not_none'] = _print_if_not_none
# Ref: https://docs.python.org/3/library/doctest.html#doctest.DocFileSuite
return doctest.DocFileSuite(
*files,
module_relative=False,
parser=FencedCellParser(fence_label='python'),
globs=globs,
setUp=set_up,
tearDown=tear_down,
checker=FencedCellOutputChecker(),
optionflags=(doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE
| doctest.IGNORE_EXCEPTION_DETAIL
| doctest.DONT_ACCEPT_BLANKLINE),
)
class FencedCellOutputChecker(tf_doctest_lib.TfDoctestOutputChecker):
"""TfDoctestChecker with a different warning message."""
MESSAGE = textwrap.dedent("""\n
##############################################################
# Check the documentation (go/g3doctest) on how to write
# testable g3docs.
##############################################################
""")
class FencedCellParser(doctest.DocTestParser):
"""Implements test parsing for ``` fenced cells.
https://docs.python.org/3/library/doctest.html#doctestparser-objects
The `get_examples` method receives a string and returns an
iterable of `doctest.Example` objects.
"""
patched = False
def __init__(self, fence_label='python'):
super().__init__()
if not self.patched:
# The default doctest compiles in "single" mode. The fenced block may
# contain multiple statements. The `_patch_compile` function fixes the
# compile mode.
doctest.compile = _patch_compile
print(
textwrap.dedent("""
*********************************************************************
* Caution: `fenced_doctest` patches `doctest.compile` don't use this
* in the same binary as any other doctests.
*********************************************************************
"""))
type(self).patched = True
# Match anything, except if the look-behind sees a closing fence.
no_fence = '(.(?<!```))*?'
self.fence_cell_re = re.compile(
rf"""
^( # After a newline
\s*```\s*({fence_label})\n # Open a labeled ``` fence
(?P<doctest>{no_fence}) # Match anything except a closing fence
\n\s*```\s*(\n|$) # Close the fence.
)
( # Optional!
[\s\n]* # Any number of blank lines.
```\s*\n # Open ```
(?P<output>{no_fence}) # Anything except a closing fence
\n\s*``` # Close the fence.
)?
""",
# Multiline so ^ matches after a newline
re.MULTILINE |
# Dotall so `.` matches newlines.
re.DOTALL |
# Verbose to allow comments/ignore-whitespace.
re.VERBOSE)
def get_examples(self,
string: str,
name: str = '<string>') -> Iterable[doctest.Example]:
# Check for a file-level skip comment.
if re.search('<!--.*?doctest.*?skip.*?all.*?-->', string, re.IGNORECASE):
return
for match in self.fence_cell_re.finditer(string):
if re.search('doctest.*skip', match.group(0), re.IGNORECASE):
continue
groups = match.groupdict()
source = textwrap.dedent(groups['doctest'])
want = groups['output']
if want is not None:
want = textwrap.dedent(want)
yield doctest.Example(
lineno=string[:match.start()].count('\n') + 1,
source=source,
want=want)
def _print_if_not_none(obj):
"""Print like a notebook: Show the repr if the object is not None.
`_patch_compile` Uses this on the final expression in each cell.
This way the outputs feel like notebooks.
Args:
obj: the object to print.
"""
if obj is not None:
print(repr(obj))
def _patch_compile(source,
filename,
mode,
flags=0,
dont_inherit=False,
optimize=-1):
"""Patch `doctest.compile` to make doctest to behave like a notebook.
Default settings for doctest are configured to run like a repl: one statement
at a time. The doctest source uses `compile(..., mode="single")`
So to let doctest act like a notebook:
1. We need `mode="exec"` (easy)
2. We need the last expression to be printed (harder).
To print the last expression, just wrap the last expression in
`_print_if_not_none(expr)`. To detect the last expression use `AST`.
If the last node is an expression modify the ast to call
`_print_if_not_none` on it, convert the ast back to source and compile that.
https://docs.python.org/3/library/functions.html#compile
Args:
source: Can either be a normal string, a byte string, or an AST object.
filename: Argument should give the file from which the code was read; pass
some recognizable value if it wasn’t read from a file ('<string>' is
commonly used).
mode: [Ignored] always use exec.
flags: Compiler options.
dont_inherit: Compiler options.
optimize: Compiler options.
Returns:
The resulting code object.
"""
# doctest passes some dummy string as the file name, AFAICT
# but tf.function freaks-out if this doesn't look like a
# python file name.
del filename
# Doctest always passes "single" here, you need exec for multiple lines.
del mode
source_ast = ast.parse(source)
final = source_ast.body[-1]
if isinstance(final, ast.Expr):
# Wrap the final expression as `_print_if_not_none(expr)`
print_it = ast.Expr(
lineno=-1,
col_offset=-1,
value=ast.Call(
func=ast.Name(
id='_print_if_not_none',
ctx=ast.Load(),
lineno=-1,
col_offset=-1),
lineno=-1,
col_offset=-1,
args=[final], # wrap the final Expression
keywords=[]))
source_ast.body[-1] = print_it
# It's not clear why this step is necessary. `compile` is supposed to handle
# AST directly.
source = astor.to_source(source_ast)
return compile(
source,
filename='dummy.py',
mode='exec',
flags=flags,
dont_inherit=dont_inherit,
optimize=optimize)
tensorflow/tools/docs/fenced_doctest_test.py
+240
View File
@@ -0,0 +1,240 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for fenced_doctest."""
from typing import List, Optional, Tuple
from absl.testing import absltest
from absl.testing import parameterized
from tensorflow.tools.docs import fenced_doctest_lib
EXAMPLES = [
# pyformat: disable
('simple', [('code', None)], """
Hello
``` python
code
```
Goodbye
"""),
('output', [('code', 'result')], """
Hello
``` python
code
```
```
result
```
Goodbye
"""),
('not-output', [('code', None)], """
Hello
``` python
code
```
``` bash
result
```
Goodbye
"""),
('first', [('code', None)], """
``` python
code
```
Goodbye
"""[1:]),
('last', [('code', None)], """
Hello
``` python
code
```"""),
('last_output', [('code', 'result')], """
Hello
``` python
code
```
```
result
```"""),
('skip-unlabeled', [], """
Hello
```
skip
```
Goodbye
"""),
('skip-wrong-label', [], """
Hello
``` sdkfjgsd
skip
```
Goodbye
"""),
('doctest_skip', [], """
Hello
``` python
doctest: +SKIP
```
Goodbye
"""),
('skip_all', [], """
<!-- doctest: skip-all -->
Hello
``` python
a
```
``` python
b
```
Goodbye
"""),
('two', [('a', None), ('b', None)], """
Hello
``` python
a
```
``` python
b
```
Goodbye
"""),
('two-outputs', [('a', 'A'), ('b', 'B')], """
Hello
``` python
a
```
```
A
```
``` python
b
```
```
B
```
Goodbye
"""),
('list', [('a', None), ('b', 'B'), ('c', 'C'), ('d', None)], """
Hello
``` python
a
```
``` python
b
```
```
B
```
List:
* first
``` python
c
```
```
C
```
``` python
d
```
* second
Goodbye
"""),
('multiline', [('a\nb', 'A\nB')], """
Hello
``` python
a
b
```
```
A
B
```
Goodbye
""")
]
ExampleTuples = List[Tuple[str, Optional[str]]]
class G3DoctestTest(parameterized.TestCase):
def _do_test(self, expected_example_tuples, string):
parser = fenced_doctest_lib.FencedCellParser(fence_label='python')
example_tuples = []
for example in parser.get_examples(string, name=self._testMethodName):
source = example.source.rstrip('\n')
want = example.want
if want is not None:
want = want.rstrip('\n')
example_tuples.append((source, want))
self.assertEqual(expected_example_tuples, example_tuples)
@parameterized.named_parameters(*EXAMPLES)
def test_parser(self, expected_example_tuples: ExampleTuples, string: str):
self._do_test(expected_example_tuples, string)
@parameterized.named_parameters(*EXAMPLES)
def test_parser_no_blanks(self, expected_example_tuples: ExampleTuples,
string: str):
string = string.replace('\n\n', '\n')
self._do_test(expected_example_tuples, string)
if __name__ == '__main__':
absltest.main()
tensorflow/tools/docs/generate2.py
+361
View File
@@ -0,0 +1,361 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A tool to generate api_docs for TensorFlow2.
```
python generate2.py --output_dir=/tmp/out
```
Requires a local installation of `tensorflow_docs`:
```
pip install git+https://github.com/tensorflow/docs
```
"""
import contextlib
import pathlib
import textwrap
from typing import NamedTuple
from absl import app
from absl import flags
from packaging import version
import tensorflow as tf
from tensorflow_docs.api_generator import doc_controls
from tensorflow_docs.api_generator import doc_generator_visitor
from tensorflow_docs.api_generator import generate_lib
from tensorflow_docs.api_generator.pretty_docs import base_page
from tensorflow_docs.api_generator.pretty_docs import module_page
import yaml
from tensorflow.python.framework import ops
from tensorflow.python.util import tf_export
from tensorflow.python.util import tf_inspect
if version.parse(tf.__version__) >= version.parse("2.14-dev"):
from tensorflow.python.util.pywrap_xla_ops import get_gpu_kernel_names # pylint: disable=g-import-not-at-top
# Caution: the google and oss versions of this import are different.
import base_dir # pylint: disable=g-import-not-at-top
# pylint: disable=g-import-not-at-top
try:
from tensorflow.python.types import doc_typealias
_EXTRA_DOCS = getattr(doc_typealias, "_EXTRA_DOCS", {})
del doc_typealias
except ImportError:
_EXTRA_DOCS = {}
# pylint: enable=g-import-not-at-top
# `tf` has an `__all__` that doesn't list important things like `keras`.
# The doc generator recognizes `__all__` as the list of public symbols.
# So patch `tf.__all__` to list everything.
tf.__all__ = [item_name for item_name, value in tf_inspect.getmembers(tf)]
# tf_export generated two copies of the module objects.
# This will just list compat.v2 as an alias for tf. Close enough, let's not
# duplicate all the module skeleton files.
tf.compat.v2 = tf
tf.losses = tf.keras.losses
tf.metrics = tf.keras.metrics
tf.optimizers = tf.keras.optimizers
tf.initializers = tf.keras.initializers
MIN_NUM_FILES_EXPECTED = 2000
FLAGS = flags.FLAGS
flags.DEFINE_string(
"code_url_prefix",
"/code/stable/tensorflow",
"A url to prepend to code paths when creating links to defining code")
flags.DEFINE_string("output_dir", "/tmp/out",
"A directory, where the docs will be output to.")
flags.DEFINE_bool("search_hints", True,
"Include meta-data search hints at the top of each file.")
flags.DEFINE_string(
"site_path", "",
"The path prefix (up to `.../api_docs/python`) used in the "
"`_toc.yaml` and `_redirects.yaml` files")
_PRIVATE_MAP = {
"tf": ["python", "core", "compiler", "examples", "tools", "contrib"],
# There's some aliasing between the compats and v1/2s, so it's easier to
# block by name and location than by deleting, or hiding objects.
"tf.compat.v1.compat": ["v1", "v2"],
"tf.compat.v2.compat": ["v1", "v2"]
}
tf.__doc__ = """
## TensorFlow
```
pip install tensorflow
```
"""
try:
tf.estimator.Estimator = doc_controls.inheritable_header(textwrap.dedent("""\
Warning: TensorFlow 2.15 included the final release of the `tf-estimator`
package. Estimators will not be available in TensorFlow 2.16 or after. See the
[migration guide](https://www.tensorflow.org/guide/migrate/migrating_estimator)
for more information about how to convert off of Estimators."
"""))(tf.estimator.Estimator)
except AttributeError:
pass
class RawOpsPageInfo(module_page.ModulePageInfo):
"""Generates a custom page for `tf.raw_ops`."""
DEFAULT_BUILDER_CLASS = base_page.TemplatePageBuilder
def build(self):
# Skip the ModulePage implementation, which doesn't use a template.
content = base_page.PageInfo.build(self)
if version.parse(tf.__version__) >= version.parse("2.14-dev"):
raw_ops_doc = self.generate_raw_ops_doc_ge_214()
else:
raw_ops_doc = self.generate_raw_ops_doc_lt_214()
return "\n".join([content, raw_ops_doc])
def generate_raw_ops_doc_lt_214(self):
"""Generates docs for `tf.raw_ops`."""
del self
warning = textwrap.dedent("""\n
Note: `tf.raw_ops` provides direct/low level access to all TensorFlow ops.
See [the RFC](https://github.com/tensorflow/community/blob/master/rfcs/20181225-tf-raw-ops.md)
for details. Unless you are library writer, you likely do not need to use
these ops directly.""")
table_header = textwrap.dedent("""
| Op Name | Has Gradient |
|---------|:------------:|""")
parts = [warning, table_header]
for op_name in sorted(dir(tf.raw_ops)):
try:
ops._gradient_registry.lookup(op_name) # pylint: disable=protected-access
has_gradient = "\N{HEAVY CHECK MARK}\N{VARIATION SELECTOR-16}"
except LookupError:
has_gradient = "\N{CROSS MARK}"
if not op_name.startswith("_"):
path = pathlib.Path("/") / FLAGS.site_path / "tf/raw_ops" / op_name
path = path.with_suffix(".md")
link = ('<a id={op_name} href="{path}">{op_name}</a>').format(
op_name=op_name, path=str(path))
parts.append("| {link} | {has_gradient} |".format(
link=link, has_gradient=has_gradient))
return "\n".join(parts)
def generate_raw_ops_doc_ge_214(self):
"""Generates docs for `tf.raw_ops`."""
del self
warning = textwrap.dedent("""\n
Note: `tf.raw_ops` provides direct/low level access to all TensorFlow ops.
See [the RFC](https://github.com/tensorflow/community/blob/master/rfcs/20181225-tf-raw-ops.md)
for details. Unless you are library writer, you likely do not need to use
these ops directly.""")
table_header = textwrap.dedent("""
| Op Name | Has Gradient | GPU XLA Support |
|---------|:------------:|:---------------:|""")
parts = [warning, table_header]
xla_compiled_ops = get_gpu_kernel_names()
for op_name in sorted(dir(tf.raw_ops)):
try:
ops._gradient_registry.lookup(op_name) # pylint: disable=protected-access
has_gradient = "\N{HEAVY CHECK MARK}\N{VARIATION SELECTOR-16}"
except LookupError:
has_gradient = "\N{CROSS MARK}"
is_xla_compilable = "\N{CROSS MARK}"
if op_name in xla_compiled_ops:
is_xla_compilable = "\N{HEAVY CHECK MARK}\N{VARIATION SELECTOR-16}"
if not op_name.startswith("_"):
path = pathlib.Path("/") / FLAGS.site_path / "tf/raw_ops" / op_name
path = path.with_suffix(".md")
link = ('<a id={op_name} href="{path}">{op_name}</a>').format(
op_name=op_name, path=str(path)
)
parts.append(
"| {link} | {has_gradient} | {is_xla_compilable} |".format(
link=link,
has_gradient=has_gradient,
is_xla_compilable=is_xla_compilable,
)
)
return "\n".join(parts)
# The doc generator isn't aware of tf_export.
# So prefix the score tuples with -1 when this is the canonical name, +1
# otherwise. The generator chooses the name with the lowest score.
class TfExportAwareVisitor(doc_generator_visitor.DocGeneratorVisitor):
"""A `tf_export`, `keras_export` and `estimator_export` aware doc_visitor."""
class TfNameScore(NamedTuple):
canonical_score: int
name_score: doc_generator_visitor.DocGeneratorVisitor.NameScore
def _score_name(self, path: doc_generator_visitor.ApiPath) -> TfNameScore:
name = ".".join(path)
all_exports = [
tf_export.TENSORFLOW_API_NAME,
tf_export.KERAS_API_NAME,
]
try:
all_exports.append(tf_export.ESTIMATOR_API_NAME)
except AttributeError:
pass
canonical = None
for api_name in all_exports:
try:
canonical = tf_export.get_canonical_name_for_symbol(
self._index[name], api_name=api_name)
except AttributeError:
canonical = None
if canonical is not None:
break
canonical_score = 1
if canonical is not None and name == "tf." + canonical:
canonical_score = -1
return self.TfNameScore(canonical_score, super()._score_name(path))
def build_docs(output_dir, code_url_prefix, search_hints):
"""Build api docs for tensorflow v2.
Args:
output_dir: A string path, where to put the files.
code_url_prefix: prefix for "Defined in" links.
search_hints: Bool. Include meta-data search hints at the top of each file.
"""
output_dir = pathlib.Path(output_dir)
site_path = pathlib.Path("/", FLAGS.site_path)
doc_controls.set_deprecated(tf.compat.v1)
try:
doc_controls.set_deprecated(tf.estimator)
except AttributeError:
pass
doc_controls.set_deprecated(tf.feature_column)
doc_controls.set_deprecated(tf.keras.preprocessing)
# The custom page will be used for raw_ops.md not the one generated above.
doc_controls.set_custom_page_builder_cls(tf.raw_ops, RawOpsPageInfo)
# Hide raw_ops from search.
for name, obj in tf_inspect.getmembers(tf.raw_ops):
if not name.startswith("_"):
doc_controls.hide_from_search(obj)
for cls in [tf.Module, tf.keras.layers.Layer, tf.keras.optimizers.Optimizer]:
doc_controls.decorate_all_class_attributes(
decorator=doc_controls.do_not_doc_in_subclasses,
cls=cls,
skip=["__init__"])
do_not_document = ["tf.__internal__",
"tf.keras.__internal__",
"tf.keras.wrappers",
"tf.__operators__",
"tf.tools",
"tf.compat.v1.pywrap_tensorflow",
"tf.pywrap_tensorflow",
"tf.flags",
"tf.batch_mat_mul_v3",
"tf.sparse_segment_sum_grad"]
for path in do_not_document:
item = tf
for part in path.split(".")[1:]:
item = getattr(item, part, None)
if item is None:
continue
doc_controls.do_not_generate_docs(item)
base_dirs, code_url_prefixes = base_dir.get_base_dirs_and_prefixes(
code_url_prefix)
doc_generator = generate_lib.DocGenerator(
root_title="TensorFlow 2",
py_modules=[("tf", tf)],
base_dir=base_dirs,
search_hints=search_hints,
code_url_prefix=code_url_prefixes,
site_path=site_path,
visitor_cls=TfExportAwareVisitor,
private_map=_PRIVATE_MAP,
extra_docs=_EXTRA_DOCS,
callbacks=base_dir.get_callbacks())
doc_generator.build(output_dir)
@contextlib.contextmanager
def edit_yaml_file(path):
content = yaml.safe_load(path.read_text())
yield content
with path.open("w") as f:
yaml.dump(content, f, default_flow_style=False)
toc_path = output_dir / "tf/_toc.yaml"
with edit_yaml_file(toc_path) as toc:
# Replace the overview path for 'TensorFlow' to
# `/api_docs/python/tf_overview`. This will be redirected to
# `/api_docs/python/tf`.
toc["toc"][0]["section"][0]["path"] = str(site_path / "tf_overview")
redirects_path = output_dir / "tf/_redirects.yaml"
with edit_yaml_file(redirects_path) as redirects:
redirects["redirects"].append({
"from": str(site_path / "tf_overview"),
"to": str(site_path / "tf"),
})
num_files = len(list(output_dir.rglob("*")))
if num_files < MIN_NUM_FILES_EXPECTED:
raise ValueError(
f"The TensorFlow api should be more than {MIN_NUM_FILES_EXPECTED} files"
f"(found {num_files}).")
def main(argv):
del argv
build_docs(
output_dir=FLAGS.output_dir,
code_url_prefix=FLAGS.code_url_prefix,
search_hints=FLAGS.search_hints)
if __name__ == "__main__":
app.run(main)
tensorflow/tools/docs/generate2_test.py
+95
View File
@@ -0,0 +1,95 @@
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tensorflow.tools.docs.generate2."""
import os
import pathlib
import shutil
import types
from unittest import mock
from packaging import version
import tensorflow as tf
import yaml
from tensorflow.python.platform import googletest
from tensorflow.tools.docs import generate2
class AutoModule(types.ModuleType):
def __getattr__(self, name):
if name.startswith('_'):
raise AttributeError()
mod = AutoModule(name)
setattr(self, name, mod)
return mod
# Make a mock tensorflow package that won't take too long to test.
fake_tf = AutoModule('FakeTensorFlow')
fake_tf.Module = tf.Module # pylint: disable=invalid-name
fake_tf.feature_column.nummeric_column = tf.feature_column.numeric_column
fake_tf.keras.Model = tf.keras.Model
fake_tf.keras.preprocessing = tf.keras.preprocessing
fake_tf.keras.layers.Layer = tf.keras.layers.Layer
fake_tf.keras.optimizers.Optimizer = tf.keras.optimizers.Optimizer
fake_tf.nn.sigmoid_cross_entropy_with_logits = (
tf.nn.sigmoid_cross_entropy_with_logits
)
fake_tf.raw_ops.Add = tf.raw_ops.Add
fake_tf.raw_ops.Print = tf.raw_ops.Print # op with no XLA support
fake_tf.summary.audio = tf.summary.audio
fake_tf.summary.audio2 = tf.summary.audio
fake_tf.__version__ = tf.__version__
class Generate2Test(googletest.TestCase):
@mock.patch.object(generate2, 'tf', fake_tf)
def test_end_to_end(self):
generate2.MIN_NUM_FILES_EXPECTED = 1
output_dir = pathlib.Path(googletest.GetTempDir())/'output'
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
generate2.build_docs(
output_dir=output_dir,
code_url_prefix='',
search_hints=True,
)
raw_ops_page = (output_dir/'tf/raw_ops.md').read_text()
self.assertIn('/tf/raw_ops/Add.md', raw_ops_page)
toc = yaml.safe_load((output_dir / 'tf/_toc.yaml').read_text())
self.assertEqual({
'title': 'Overview',
'path': '/tf_overview'
}, toc['toc'][0]['section'][0])
redirects = yaml.safe_load((output_dir / 'tf/_redirects.yaml').read_text())
self.assertIn({'from': '/tf_overview', 'to': '/tf'}, redirects['redirects'])
if version.parse(fake_tf.__version__) >= version.parse('2.14'):
self.assertIn(
'<a id=Add href="/tf/raw_ops/Add.md">Add</a> | ✔️ | ✔️ |', raw_ops_page
)
self.assertIn(
'<a id=Print href="/tf/raw_ops/Print.md">Print</a> | ✔️ | ❌ |',
raw_ops_page,
)
if __name__ == '__main__':
googletest.main()
tensorflow/tools/docs/tf_doctest.py
+217
View File
@@ -0,0 +1,217 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Run doctests for tensorflow."""
import importlib
import os
import pkgutil
import sys
from absl import flags
from absl.testing import absltest
import numpy as np
import tensorflow.compat.v2 as tf
from tensorflow.python.eager import context
from tensorflow.python.ops import logging_ops
from tensorflow.tools.docs import tf_doctest_lib
# We put doctest after absltest so that it picks up the unittest monkeypatch.
# Otherwise doctest tests aren't runnable at all.
import doctest # pylint: disable=g-bad-import-order
tf.compat.v1.enable_v2_behavior()
# `enable_interactive_logging` must come after `enable_v2_behavior`.
logging_ops.enable_interactive_logging()
FLAGS = flags.FLAGS
flags.DEFINE_list('module', [], 'A list of specific module to run doctest on.')
flags.DEFINE_list('module_prefix_skip', [],
'A list of modules to ignore when resolving modules.')
flags.DEFINE_boolean('list', None,
'List all the modules in the core package imported.')
flags.DEFINE_integer('required_gpus', 0,
'The number of GPUs required for the tests.')
# Both --module and --module_prefix_skip are relative to PACKAGE.
PACKAGES = [
'tensorflow.python.',
'tensorflow.lite.python.',
]
def recursive_import(root):
"""Recursively imports all the sub-modules under a root package.
Args:
root: A python package.
"""
for _, name, _ in pkgutil.walk_packages(
root.__path__, prefix=root.__name__ + '.'):
try:
importlib.import_module(name)
except (AttributeError, ImportError):
pass
def find_modules():
"""Finds all the modules in the core package imported.
Returns:
A list containing all the modules in tensorflow.python.
"""
tf_modules = []
for name, module in sys.modules.items():
# The below for loop is a constant time loop.
for package in PACKAGES:
if name.startswith(package):
tf_modules.append(module)
return tf_modules
def filter_on_submodules(all_modules, submodules):
"""Filters all the modules based on the modules flag.
The module flag has to be relative to the core package imported.
For example, if `module=keras.layers` then, this function will return
all the modules in the submodule.
Args:
all_modules: All the modules in the core package.
submodules: Submodules to filter from all the modules.
Returns:
All the modules in the submodule.
"""
filtered_modules = []
for mod in all_modules:
for submodule in submodules:
# The below for loop is a constant time loop.
for package in PACKAGES:
if package + submodule in mod.__name__:
filtered_modules.append(mod)
return filtered_modules
def setup_gpu(required_gpus):
"""Sets up the GPU devices.
If there're more available GPUs than needed, it hides the additional ones. If
there're less, it creates logical devices. This is to make sure the tests see
a fixed number of GPUs regardless of the environment.
Args:
required_gpus: an integer. The number of GPUs required.
Raises:
ValueError: if num_gpus is larger than zero but no GPU is available.
"""
if required_gpus == 0:
return
available_gpus = tf.config.experimental.list_physical_devices('GPU')
if not available_gpus:
raise ValueError('requires at least one physical GPU')
if len(available_gpus) >= required_gpus:
tf.config.set_visible_devices(available_gpus[:required_gpus])
else:
# Create logical GPUs out of one physical GPU for simplicity. Note that the
# other physical GPUs are still available and corresponds to one logical GPU
# each.
num_logical_gpus = required_gpus - len(available_gpus) + 1
logical_gpus = [
tf.config.LogicalDeviceConfiguration(memory_limit=256)
for _ in range(num_logical_gpus)
]
tf.config.set_logical_device_configuration(available_gpus[0], logical_gpus)
class TfTestCase(tf.test.TestCase):
def set_up(self, test):
# Enable soft device placement to run distributed doctests.
tf.config.set_soft_device_placement(True)
self.setUp()
context.async_wait()
def tear_down(self, test):
self.tearDown()
def load_tests(unused_loader, tests, unused_ignore):
"""Loads all the tests in the docstrings and runs them."""
tf_modules = find_modules()
if FLAGS.module:
tf_modules = filter_on_submodules(tf_modules, FLAGS.module)
if FLAGS.list:
print('**************************************************')
for mod in tf_modules:
print(mod.__name__)
print('**************************************************')
return tests
test_shard_index = int(os.environ.get('TEST_SHARD_INDEX', '0'))
total_test_shards = int(os.environ.get('TEST_TOTAL_SHARDS', '1'))
tf_modules = sorted(tf_modules, key=lambda mod: mod.__name__)
for n, module in enumerate(tf_modules):
if (n % total_test_shards) != test_shard_index:
continue
# If I break the loop comprehension, then the test times out in `small`
# size.
if any(
module.__name__.startswith(package + prefix) # pylint: disable=g-complex-comprehension
for prefix in FLAGS.module_prefix_skip for package in PACKAGES):
continue
testcase = TfTestCase()
tests.addTests(
doctest.DocTestSuite(
module,
test_finder=doctest.DocTestFinder(exclude_empty=False),
extraglobs={
'tf': tf,
'np': np,
'os': os
},
setUp=testcase.set_up,
tearDown=testcase.tear_down,
checker=tf_doctest_lib.TfDoctestOutputChecker(),
optionflags=(doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE
| doctest.IGNORE_EXCEPTION_DETAIL
| doctest.DONT_ACCEPT_BLANKLINE),
))
return tests
# We can only create logical devices before initializing Tensorflow. This is
# called by unittest framework before running any test.
# https://docs.python.org/3/library/unittest.html#setupmodule-and-teardownmodule
def setUpModule():
setup_gpu(FLAGS.required_gpus)
if __name__ == '__main__':
absltest.main()
tensorflow/tools/docs/tf_doctest_lib.py
+224
View File
@@ -0,0 +1,224 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Run doctests for tensorflow."""
import doctest
import re
import textwrap
import numpy as np
class _FloatExtractor(object):
"""Class for extracting floats from a string.
For example:
>>> text_parts, floats = _FloatExtractor()("Text 1.0 Text")
>>> text_parts
["Text ", " Text"]
>>> floats
np.array([1.0])
"""
# Note: non-capturing groups "(?" are not returned in matched groups, or by
# re.split.
_FLOAT_RE = re.compile(
r"""
( # Captures the float value.
(?:
[-+]| # Start with a sign is okay anywhere.
(?: # Otherwise:
^| # Start after the start of string
(?<=[^\w.]) # Not after a word char, or a .
)
)
(?: # Digits and exponent - something like:
{digits_dot_maybe_digits}{exponent}?| # "1.0" "1." "1.0e3", "1.e3"
{dot_digits}{exponent}?| # ".1" ".1e3"
{digits}{exponent}| # "1e3"
{digits}(?=j) # "300j"
)
)
j? # Optional j for cplx numbers, not captured.
(?= # Only accept the match if
$| # * At the end of the string, or
[^\w.] # * Next char is not a word char or "."
)
""".format(
# Digits, a "." and optional more digits: "1.1".
digits_dot_maybe_digits=r'(?:[0-9]+\.(?:[0-9]*))',
# A "." with trailing digits ".23"
dot_digits=r'(?:\.[0-9]+)',
# digits: "12"
digits=r'(?:[0-9]+)',
# The exponent: An "e" or "E", optional sign, and at least one digit.
# "e-123", "E+12", "e12"
exponent=r'(?:[eE][-+]?[0-9]+)'),
re.VERBOSE)
def __call__(self, string):
"""Extracts floats from a string.
>>> text_parts, floats = _FloatExtractor()("Text 1.0 Text")
>>> text_parts
["Text ", " Text"]
>>> floats
np.array([1.0])
Args:
string: the string to extract floats from.
Returns:
A (string, array) pair, where `string` has each float replaced by "..."
and `array` is a `float32` `numpy.array` containing the extracted floats.
"""
texts = []
floats = []
for i, part in enumerate(self._FLOAT_RE.split(string)):
if i % 2 == 0:
texts.append(part)
else:
floats.append(float(part))
return texts, np.array(floats)
class TfDoctestOutputChecker(doctest.OutputChecker, object):
"""Customizes how `want` and `got` are compared, see `check_output`."""
def __init__(self, *args, **kwargs):
super(TfDoctestOutputChecker, self).__init__(*args, **kwargs)
self.extract_floats = _FloatExtractor()
self.text_good = None
self.float_size_good = None
_ADDRESS_RE = re.compile(r'\bat 0x[0-9a-f]*?>')
# TODO(yashkatariya): Add other tensor's string substitutions too.
# tf.RaggedTensor doesn't need one.
_NUMPY_OUTPUT_RE = re.compile(r'<tf.Tensor.*?numpy=(.*?)>', re.DOTALL)
def _allclose(self, want, got, rtol=1e-3, atol=1e-3):
return np.allclose(want, got, rtol=rtol, atol=atol)
def _tf_tensor_numpy_output(self, string):
modified_string = self._NUMPY_OUTPUT_RE.sub(r'\1', string)
return modified_string, modified_string != string
MESSAGE = textwrap.dedent("""\n
#############################################################
Check the documentation (https://www.tensorflow.org/community/contribute/docs_ref) on how to
write testable docstrings.
#############################################################""")
def check_output(self, want, got, optionflags):
"""Compares the docstring output to the output gotten by running the code.
Python addresses in the output are replaced with wildcards.
Float values in the output compared as using `np.allclose`:
* Float values are extracted from the text and replaced with wildcards.
* The wildcard text is compared to the actual output.
* The float values are compared using `np.allclose`.
The method returns `True` if both the text comparison and the numeric
comparison are successful.
The numeric comparison will fail if either:
* The wrong number of floats are found.
* The float values are not within tolerence.
Args:
want: The output in the docstring.
got: The output generated after running the snippet.
optionflags: Flags passed to the doctest.
Returns:
A bool, indicating if the check was successful or not.
"""
# If the docstring's output is empty and there is some output generated
# after running the snippet, return True. This is because if the user
# doesn't want to display output, respect that over what the doctest wants.
if got and not want:
return True
if want is None:
want = ''
if want == got:
return True
# Replace python's addresses with ellipsis (`...`) since it can change on
# each execution.
want = self._ADDRESS_RE.sub('at ...>', want)
# Replace tf.Tensor strings with only their numpy field values.
want, want_changed = self._tf_tensor_numpy_output(want)
if want_changed:
got, _ = self._tf_tensor_numpy_output(got)
# Separate out the floats, and replace `want` with the wild-card version
# "result=7.0" => "result=..."
want_text_parts, self.want_floats = self.extract_floats(want)
# numpy sometimes pads floats in arrays with spaces
# got: [1.2345, 2.3456, 3.0 ] want: [1.2345, 2.3456, 3.0001]
# And "normalize whitespace" only works when there's at least one space,
# so strip them and let the wildcard handle it.
want_text_parts = [part.strip(' ') for part in want_text_parts]
want_text_wild = '...'.join(want_text_parts)
if '....' in want_text_wild:
# If a float comes just after a period you'll end up four dots and the
# first three count as the ellipsis. Replace it with three dots.
want_text_wild = re.sub(r'\.\.\.\.+', '...', want_text_wild)
# Find the floats in the string returned by the test
_, self.got_floats = self.extract_floats(got)
self.text_good = super(TfDoctestOutputChecker, self).check_output(
want=want_text_wild, got=got, optionflags=optionflags)
if not self.text_good:
return False
if self.want_floats.size == 0:
# If there are no floats in the "want" string, ignore all the floats in
# the result. "np.array([ ... ])" matches "np.array([ 1.0, 2.0 ])"
return True
self.float_size_good = (self.want_floats.size == self.got_floats.size)
if self.float_size_good:
return self._allclose(self.want_floats, self.got_floats)
else:
return False
def output_difference(self, example, got, optionflags):
got = [got]
# If the some of the float output is hidden with `...`, `float_size_good`
# will be False. This is because the floats extracted from the string is
# converted into a 1-D numpy array. Hence hidding floats is not allowed
# anymore.
if self.text_good:
if not self.float_size_good:
got.append("\n\nCAUTION: tf_doctest doesn't work if *some* of the "
"*float output* is hidden with a \"...\".")
got.append(self.MESSAGE)
got = '\n'.join(got)
return (super(TfDoctestOutputChecker,
self).output_difference(example, got, optionflags))
tensorflow/tools/docs/tf_doctest_test.py
+220
View File
@@ -0,0 +1,220 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tf_doctest."""
import doctest
from absl.testing import absltest
from absl.testing import parameterized
from tensorflow.tools.docs import tf_doctest_lib
class TfDoctestOutputCheckerTest(parameterized.TestCase):
@parameterized.parameters(
# Don't match ints.
['result = 1', []],
# Match floats.
['0.0', [0.]],
['text 1.0 text', [1.]],
['text 1. text', [1.]],
['text .1 text', [.1]],
['text 1e3 text', [1000.]],
['text 1.e3 text', [1000.]],
['text +1. text', [1.]],
['text -1. text', [-1.]],
['text 1e+3 text', [1000.]],
['text 1e-3 text', [0.001]],
['text +1E3 text', [1000.]],
['text -1E3 text', [-1000.]],
['text +1e-3 text', [0.001]],
['text -1e+3 text', [-1000.]],
# Match at the start and end of a string.
['.1', [.1]],
['.1 text', [.1]],
['text .1', [.1]],
['0.1 text', [.1]],
['text 0.1', [.1]],
['0. text', [0.]],
['text 0.', [0.]],
['1e-1 text', [.1]],
['text 1e-1', [.1]],
# Don't match floats mixed into text
['text1.0 text', []],
['text 1.0text', []],
['text1.0text', []],
['0x12e4', []], # not 12000
['TensorBoard: http://128.0.0.1:8888', []],
# With a newline
['1.0 text\n 2.0 3.0 text', [1., 2., 3.]],
# With ints and a float.
['shape (1,2,3) value -1e9', [-1e9]],
# "." after a float.
['No floats at end of sentence: 1.0.', []],
['No floats with ellipsis: 1.0...', []],
# A numpy array
["""array([[1., 2., 3.],
[4., 5., 6.]], dtype=float32)""", [1, 2, 3, 4, 5, 6]
],
# Match both parts of a complex number
# python style
['(0.0002+30000j)', [0.0002, 30000]],
['(2.3e-10-3.34e+9j)', [2.3e-10, -3.34e+9]],
# numpy style
['array([1.27+5.j])', [1.27, 5]],
['(2.3e-10+3.34e+9j)', [2.3e-10, 3.34e+9]],
["""array([1.27e-09+5.e+00j,
2.30e+01-1.e-03j])""", [1.27e-09, 5.e+00, 2.30e+01, -1.e-03]],
# Check examples in tolerence.
['1e-6', [0]],
['0.0', [1e-6]],
['1.000001e9', [1e9]],
['1e9', [1.000001e9]],
)
def test_extract_floats(self, text, expected_floats):
extract_floats = tf_doctest_lib._FloatExtractor()
output_checker = tf_doctest_lib.TfDoctestOutputChecker()
(text_parts, extracted_floats) = extract_floats(text)
text_with_wildcards = '...'.join(text_parts)
# Check that the lengths match before doing anything else.
try:
self.assertLen(extracted_floats, len(expected_floats))
except AssertionError as e:
msg = '\n\n expected: {}\n found: {}'.format(
expected_floats, extracted_floats)
e.args = (e.args[0] + msg,)
raise e
# The floats should match according to allclose
try:
self.assertTrue(
output_checker._allclose(expected_floats, extracted_floats))
except AssertionError as e:
msg = '\n\nexpected: {}\nfound: {}'.format(expected_floats,
extracted_floats)
e.args = (e.args[0] + msg,)
raise e
# The wildcard text should match the input text, according to the
# OutputChecker base class.
try:
self.assertTrue(doctest.OutputChecker().check_output(
want=text_with_wildcards, got=text, optionflags=doctest.ELLIPSIS))
except AssertionError as e:
msg = '\n\n expected: {}\n found: {}'.format(
text_with_wildcards, text)
e.args = (e.args[0] + msg,)
raise e
@parameterized.parameters(
# Check examples out of tolerence.
['1.001e-2', [0]],
['0.0', [1.001e-3]],
)
def test_fail_tolerences(self, text, expected_floats):
extract_floats = tf_doctest_lib._FloatExtractor()
output_checker = tf_doctest_lib.TfDoctestOutputChecker()
(_, extracted_floats) = extract_floats(text)
# These floats should not match according to allclose
try:
self.assertFalse(
output_checker._allclose(expected_floats, extracted_floats))
except AssertionError as e:
msg = ('\n\nThese matched! They should not have.\n'
'\n\n Expected: {}\n found: {}'.format(
expected_floats, extracted_floats))
e.args = (e.args[0] + msg,)
raise e
def test_want_no_floats(self):
want = 'text ... text'
got = 'text 1.0 1.2 1.9 text'
output_checker = tf_doctest_lib.TfDoctestOutputChecker()
self.assertTrue(
output_checker.check_output(
want=want, got=got, optionflags=doctest.ELLIPSIS))
@parameterized.parameters(['text [1.0 ] text', 'text [1.00] text'],
['text [ 1.0] text', 'text [1.0 ] text'],
['text [ 1.0 ] text', 'text [ 1.0] text'],
['text [1.000] text', 'text [ 1.0 ] text'])
def test_extra_spaces(self, want, got):
output_checker = tf_doctest_lib.TfDoctestOutputChecker()
self.assertTrue(
output_checker.check_output(
want=want, got=got, optionflags=doctest.ELLIPSIS))
@parameterized.parameters(['Hello. 2.0', 'Hello. 2.0000001'],
['Hello... 2.0', 'Hello 2.0000001'])
def test_extra_dots(self, want, got):
output_checker = tf_doctest_lib.TfDoctestOutputChecker()
self.assertTrue(
output_checker.check_output(
want=want, got=got, optionflags=doctest.ELLIPSIS
)
)
@parameterized.parameters(['1.0, ..., 1.0', '1.0, 1.0, 1.0'],
['1.0, 1.0..., 1.0', '1.0, 1.002, 1.0'])
def test_wrong_float_counts(self, want, got):
output_checker = tf_doctest_lib.TfDoctestOutputChecker()
output_checker.check_output(
want=want, got=got, optionflags=doctest.ELLIPSIS)
example = doctest.Example('None', want=want)
result = output_checker.output_difference(
example=example, got=got, optionflags=doctest.ELLIPSIS)
self.assertIn("doesn't work if *some* of the", result)
@parameterized.parameters(
['<...>', ('<...>', False)],
['TensorFlow', ('TensorFlow', False)],
[
'tf.Variable([[1, 2], [3, 4]])',
('tf.Variable([[1, 2], [3, 4]])', False)
],
['<tf.Tensor: shape=(), dtype=float32, numpy=inf>', ('inf', True)],
[
'<tf.RaggedTensor:... shape=(2, 2), numpy=1>',
('<tf.RaggedTensor:... shape=(2, 2), numpy=1>', False)
],
[
"""<tf.Tensor: shape=(2, 2), dtype=int32, numpy=
array([[2, 2],
[3, 5]], dtype=int32)>""",
('\n array([[2, 2],\n [3, 5]], '
'dtype=int32)', True)
],
[
'[<tf.Tensor: shape=(2,), dtype=int32, numpy=array([1, 2], '
'dtype=int32)>, '
'<tf.Tensor: shape=(2,), dtype=int32, numpy=array([3, 4], '
'dtype=int32)>]',
('[array([1, 2], dtype=int32), array([3, 4], dtype=int32)]', True)
],
)
def test_tf_tensor_numpy_output(self, string, expected_output):
output_checker = tf_doctest_lib.TfDoctestOutputChecker()
output = output_checker._tf_tensor_numpy_output(string)
self.assertEqual(expected_output, output)
if __name__ == '__main__':
absltest.main()