chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
License Check / license-check (push) Failing after 2s

This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
+135
View File
@@ -0,0 +1,135 @@
load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
load("//tensorflow:tensorflow.default.bzl", "tf_py_strict_test")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
py_library(
name = "plugin_asset",
srcs = ["plugin_asset.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
"//tensorflow/python/framework:ops",
],
)
py_library(
name = "__init__",
srcs = ["__init__.py"],
strict_deps = True,
visibility = ["//visibility:public"],
)
py_library(
name = "summary_iterator",
srcs = ["summary_iterator.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python/lib/io:tf_record",
"//tensorflow/python/util:tf_export",
],
)
py_library(
name = "summary_py",
srcs = ["summary.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
":tb_summary",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/distribute:summary_op_util",
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:ops",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:logging_ops_gen",
"//tensorflow/python/ops:summary_op_util",
"//tensorflow/python/ops:summary_ops_gen",
"//tensorflow/python/ops:summary_ops_v2",
"//tensorflow/python/summary/writer",
"//tensorflow/python/summary/writer:writer_cache",
"//tensorflow/python/training:training_util",
"//tensorflow/python/util:compat",
"//tensorflow/python/util:tf_export",
],
)
tf_py_strict_test(
name = "plugin_asset_test",
size = "small",
srcs = ["plugin_asset_test.py"],
deps = [
":plugin_asset",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:test",
],
)
tf_py_strict_test(
name = "summary_iterator_test",
size = "small",
srcs = ["summary_iterator_test.py"],
deps = [
":summary_iterator",
":summary_py",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/summary/writer",
],
)
tf_py_strict_test(
name = "summary_test",
size = "small",
srcs = ["summary_test.py"],
deps = [
":summary_py",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:meta_graph",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:summary_ops_v2",
"//tensorflow/python/ops:variables",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:test",
],
)
tf_py_strict_test(
name = "summary_v2_test",
size = "small",
srcs = ["summary_v2_test.py"],
deps = [
":summary_py",
":tb_summary",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:summary_ops_v2",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/training:training_util",
],
)
py_library(
name = "tb_summary",
srcs = ["tb_summary.py"],
strict_deps = True,
visibility = ["//tensorflow:internal"],
deps = ["//tensorflow/python/util:tf_export"],
)
+10
View File
@@ -0,0 +1,10 @@
# TensorFlow Event Processing
This folder contains classes useful for analyzing and visualizing TensorFlow
events files. The code is primarily being developed to support TensorBoard,
but it can be used by anyone who wishes to analyze or visualize TensorFlow
events files.
If you wish to load TensorFlow events, you should use an EventAccumulator
(to load from a single events file) or an EventMultiplexer (to load from
multiple events files).
+136
View File
@@ -0,0 +1,136 @@
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""TensorBoard Plugin asset abstract class.
TensorBoard plugins may need to provide arbitrary assets, such as
configuration information for specific outputs, or vocabulary files, or sprite
images, etc.
This module contains methods that allow plugin assets to be specified at graph
construction time. Plugin authors define a PluginAsset which is treated as a
singleton on a per-graph basis. The PluginAsset has an assets method which
returns a dictionary of asset contents. The tf.compat.v1.summary.FileWriter
(or any other Summary writer) will serialize these assets in such a way that
TensorBoard can retrieve them.
"""
import abc
from tensorflow.python.framework import ops
_PLUGIN_ASSET_PREFIX = "__tensorboard_plugin_asset__"
def get_plugin_asset(plugin_asset_cls, graph=None):
"""Acquire singleton PluginAsset instance from a graph.
PluginAssets are always singletons, and are stored in tf Graph collections.
This way, they can be defined anywhere the graph is being constructed, and
if the same plugin is configured at many different points, the user can always
modify the same instance.
Args:
plugin_asset_cls: The PluginAsset class
graph: (optional) The graph to retrieve the instance from. If not specified,
the default graph is used.
Returns:
An instance of the plugin_asset_class
Raises:
ValueError: If we have a plugin name collision, or if we unexpectedly find
the wrong number of items in a collection.
"""
if graph is None:
graph = ops.get_default_graph()
if not plugin_asset_cls.plugin_name:
raise ValueError("Class %s has no plugin_name" % plugin_asset_cls.__name__)
name = _PLUGIN_ASSET_PREFIX + plugin_asset_cls.plugin_name
container = graph.get_collection(name)
if container:
if len(container) != 1:
raise ValueError("Collection for %s had %d items, expected 1" %
(name, len(container)))
instance = container[0]
if not isinstance(instance, plugin_asset_cls):
raise ValueError("Plugin name collision between classes %s and %s" %
(plugin_asset_cls.__name__, instance.__class__.__name__))
else:
instance = plugin_asset_cls()
graph.add_to_collection(name, instance)
graph.add_to_collection(_PLUGIN_ASSET_PREFIX, plugin_asset_cls.plugin_name)
return instance
def get_all_plugin_assets(graph=None):
"""Retrieve all PluginAssets stored in the graph collection.
Args:
graph: Optionally, the graph to get assets from. If unspecified, the default
graph is used.
Returns:
A list with all PluginAsset instances in the graph.
Raises:
ValueError: if we unexpectedly find a collection with the wrong number of
PluginAssets.
"""
if graph is None:
graph = ops.get_default_graph()
out = []
for name in graph.get_collection(_PLUGIN_ASSET_PREFIX):
collection = graph.get_collection(_PLUGIN_ASSET_PREFIX + name)
if len(collection) != 1:
raise ValueError("Collection for %s had %d items, expected 1" %
(name, len(collection)))
out.append(collection[0])
return out
class PluginAsset(metaclass=abc.ABCMeta):
"""This abstract base class allows TensorBoard to serialize assets to disk.
Plugin authors are expected to extend the PluginAsset class, so that it:
- has a unique plugin_name
- provides an assets method that returns an {asset_name: asset_contents}
dictionary. For now, asset_contents are strings, although we may add
StringIO support later.
LifeCycle of a PluginAsset instance:
- It is constructed when get_plugin_asset is called on the class for
the first time.
- It is configured by code that follows the calls to get_plugin_asset
- When the containing graph is serialized by the
tf.compat.v1.summary.FileWriter, the writer calls assets and the
PluginAsset instance provides its contents to be written to disk.
"""
plugin_name = None
@abc.abstractmethod
def assets(self):
"""Provide all of the assets contained by the PluginAsset instance.
The assets method should return a dictionary structured as
{asset_name: asset_contents}. asset_contents is a string.
This method will be called by the tf.compat.v1.summary.FileWriter when it
is time to write the assets out to disk.
"""
raise NotImplementedError()
@@ -0,0 +1,77 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.platform import googletest
from tensorflow.python.summary import plugin_asset
class _UnnamedPluginAsset(plugin_asset.PluginAsset):
"""An example asset with a dummy serialize method provided, but no name."""
def assets(self):
return {}
class _ExamplePluginAsset(_UnnamedPluginAsset):
"""Simple example asset."""
plugin_name = "_ExamplePluginAsset"
class _OtherExampleAsset(_UnnamedPluginAsset):
"""Simple example asset."""
plugin_name = "_OtherExampleAsset"
class _ExamplePluginThatWillCauseCollision(_UnnamedPluginAsset):
plugin_name = "_ExamplePluginAsset"
class PluginAssetTest(test_util.TensorFlowTestCase):
def testGetPluginAsset(self):
epa = plugin_asset.get_plugin_asset(_ExamplePluginAsset)
self.assertIsInstance(epa, _ExamplePluginAsset)
epa2 = plugin_asset.get_plugin_asset(_ExamplePluginAsset)
self.assertIs(epa, epa2)
opa = plugin_asset.get_plugin_asset(_OtherExampleAsset)
self.assertIsNot(epa, opa)
def testUnnamedPluginFails(self):
with self.assertRaises(ValueError):
plugin_asset.get_plugin_asset(_UnnamedPluginAsset)
def testPluginCollisionDetected(self):
plugin_asset.get_plugin_asset(_ExamplePluginAsset)
with self.assertRaises(ValueError):
plugin_asset.get_plugin_asset(_ExamplePluginThatWillCauseCollision)
def testGetAllPluginAssets(self):
epa = plugin_asset.get_plugin_asset(_ExamplePluginAsset)
opa = plugin_asset.get_plugin_asset(_OtherExampleAsset)
self.assertItemsEqual(plugin_asset.get_all_plugin_assets(), [epa, opa])
def testRespectsGraphArgument(self):
g1 = ops.Graph()
g2 = ops.Graph()
e1 = plugin_asset.get_plugin_asset(_ExamplePluginAsset, g1)
e2 = plugin_asset.get_plugin_asset(_ExamplePluginAsset, g2)
self.assertEqual(e1, plugin_asset.get_all_plugin_assets(g1)[0])
self.assertEqual(e2, plugin_asset.get_all_plugin_assets(g2)[0])
if __name__ == "__main__":
googletest.main()
+852
View File
@@ -0,0 +1,852 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Operations for writing summary data, for use in analysis and visualization.
See the [Summaries and
TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) guide.
API docstring: tensorflow.summary
"""
import contextlib
import warnings
from google.protobuf import json_format as _json_format
# exports Summary, SummaryDescription, Event, TaggedRunMetadata, SessionLog
# pylint: disable=unused-import, g-importing-member
from tensorflow.core.framework.summary_pb2 import Summary
from tensorflow.core.framework.summary_pb2 import SummaryDescription
from tensorflow.core.framework.summary_pb2 import SummaryMetadata as _SummaryMetadata # pylint: enable=unused-import
from tensorflow.core.util.event_pb2 import Event
from tensorflow.core.util.event_pb2 import SessionLog
from tensorflow.core.util.event_pb2 import TaggedRunMetadata
# pylint: enable=unused-import
from tensorflow.python.distribute import summary_op_util as _distribute_summary_op_util
from tensorflow.python.eager import context as _context
from tensorflow.python.framework import constant_op as _constant_op
from tensorflow.python.framework import dtypes as _dtypes
from tensorflow.python.framework import ops as _ops
from tensorflow.python.ops import array_ops as _array_ops
from tensorflow.python.ops import gen_logging_ops as _gen_logging_ops
from tensorflow.python.ops import gen_summary_ops as _gen_summary_ops # pylint: disable=unused-import
from tensorflow.python.ops import summary_op_util as _summary_op_util
from tensorflow.python.ops import summary_ops_v2 as _summary_ops_v2
from tensorflow.python.summary import tb_summary
# exports FileWriter, FileWriterCache
# pylint: disable=unused-import
from tensorflow.python.summary.writer.writer import FileWriter
from tensorflow.python.summary.writer.writer_cache import FileWriterCache
# pylint: enable=unused-import
from tensorflow.python.training import training_util as _training_util
from tensorflow.python.util import compat as _compat
from tensorflow.python.util.tf_export import tf_export
@tf_export(v1=['summary.scalar'])
def scalar(name, tensor, collections=None, family=None):
"""Outputs a `Summary` protocol buffer containing a single scalar value.
The generated Summary has a Tensor.proto containing the input Tensor.
Args:
name: A name for the generated node. Will also serve as the series name in
TensorBoard.
tensor: A real numeric Tensor containing a single value.
collections: Optional list of graph collections keys. The new summary op is
added to these collections. Defaults to `[GraphKeys.SUMMARIES]`.
family: Optional; if provided, used as the prefix of the summary tag name,
which controls the tab name used for display on Tensorboard.
Returns:
A scalar `Tensor` of type `string`. Which contains a `Summary` protobuf.
Raises:
ValueError: If tensor has the wrong shape or type.
@compatibility(TF2)
For compatibility purposes, when invoked in TF2 where the outermost context is
eager mode, this API will check if there is a suitable TF2 summary writer
context available, and if so will forward this call to that writer instead. A
"suitable" writer context means that the writer is set as the default writer,
and there is an associated non-empty value for `step` (see
`tf.summary.SummaryWriter.as_default`, `tf.summary.experimental.set_step` or
alternatively `tf.compat.v1.train.create_global_step`). For the forwarded
call, the arguments here will be passed to the TF2 implementation of
`tf.summary.scalar`, and the return value will be an empty bytestring tensor,
to avoid duplicate summary writing. This forwarding is best-effort and not all
arguments will be preserved.
To migrate to TF2, please use `tf.summary.scalar` instead. Please check
[Migrating tf.summary usage to
TF 2.0](https://www.tensorflow.org/tensorboard/migrate#in_tf_1x) for concrete
steps for migration. `tf.summary.scalar` can also log training metrics in
Keras, you can check [Logging training metrics in
Keras](https://www.tensorflow.org/tensorboard/scalars_and_keras) for details.
#### How to Map Arguments
| TF1 Arg Name | TF2 Arg Name | Note |
| :------------ | :-------------- | :------------------------------------- |
| `name` | `name` | - |
| `tensor` | `data` | - |
| - | `step` | Explicit int64-castable monotonic step |
: : : value. If omitted, this defaults to :
: : : `tf.summary.experimental.get_step()`. :
| `collections` | Not Supported | - |
| `family` | Removed | Please use `tf.name_scope` instead to |
: : : manage summary name prefix. :
| - | `description` | Optional long-form `str` description |
: : : for the summary. Markdown is supported.:
: : : Defaults to empty. :
@end_compatibility
"""
if _distribute_summary_op_util.skip_summary():
return _constant_op.constant('')
# Special case: invoke v2 op for TF2 users who have a v2 writer.
if _should_invoke_v2_op():
# Defer the import to happen inside the symbol to prevent breakage due to
# missing dependency.
with _compat_summary_scope(name, family) as tag:
tb_summary.scalar(name=tag, data=tensor, step=_get_step_for_v2())
# Return an empty Tensor, which will be acceptable as an input to the
# `tf.compat.v1.summary.merge()` API.
return _constant_op.constant(b'')
# Fall back to legacy v1 scalar implementation.
with _summary_op_util.summary_scope(
name, family, values=[tensor]) as (tag, scope):
val = _gen_logging_ops.scalar_summary(tags=tag, values=tensor, name=scope)
_summary_op_util.collect(val, collections, [_ops.GraphKeys.SUMMARIES])
return val
@tf_export(v1=['summary.image'])
def image(name, tensor, max_outputs=3, collections=None, family=None):
"""Outputs a `Summary` protocol buffer with images.
The summary has up to `max_outputs` summary values containing images. The
images are built from `tensor` which must be 4-D with shape `[batch_size,
height, width, channels]` and where `channels` can be:
* 1: `tensor` is interpreted as Grayscale.
* 3: `tensor` is interpreted as RGB.
* 4: `tensor` is interpreted as RGBA.
The images have the same number of channels as the input tensor. For float
input, the values are normalized one image at a time to fit in the range
`[0, 255]`. `uint8` values are unchanged. The op uses two different
normalization algorithms:
* If the input values are all positive, they are rescaled so the largest one
is 255.
* If any input value is negative, the values are shifted so input value 0.0
is at 127. They are then rescaled so that either the smallest value is 0,
or the largest one is 255.
The `tag` in the outputted Summary.Value protobufs is generated based on the
name, with a suffix depending on the max_outputs setting:
* If `max_outputs` is 1, the summary value tag is '*name*/image'.
* If `max_outputs` is greater than 1, the summary value tags are
generated sequentially as '*name*/image/0', '*name*/image/1', etc.
Args:
name: A name for the generated node. Will also serve as a series name in
TensorBoard.
tensor: A 4-D `uint8` or `float32` `Tensor` of shape `[batch_size, height,
width, channels]` where `channels` is 1, 3, or 4.
max_outputs: Max number of batch elements to generate images for.
collections: Optional list of ops.GraphKeys. The collections to add the
summary to. Defaults to [_ops.GraphKeys.SUMMARIES]
family: Optional; if provided, used as the prefix of the summary tag name,
which controls the tab name used for display on Tensorboard.
Returns:
A scalar `Tensor` of type `string`. The serialized `Summary` protocol
buffer.
@compatibility(TF2)
For compatibility purposes, when invoked in TF2 where the outermost context is
eager mode, this API will check if there is a suitable TF2 summary writer
context available, and if so will forward this call to that writer instead. A
"suitable" writer context means that the writer is set as the default writer,
and there is an associated non-empty value for `step` (see
`tf.summary.SummaryWriter.as_default`, `tf.summary.experimental.set_step` or
alternatively `tf.compat.v1.train.create_global_step`). For the forwarded
call, the arguments here will be passed to the TF2 implementation of
`tf.summary.image`, and the return value will be an empty bytestring tensor,
to avoid duplicate summary writing. This forwarding is best-effort and not all
arguments will be preserved. Additionally:
* The TF2 op does not do any of the normalization steps described above.
Rather than rescaling data that's outside the expected range, it simply
clips it.
* The TF2 op just outputs the data under a single tag that contains multiple
samples, rather than multiple tags (i.e. no "/0" or "/1" suffixes).
To migrate to TF2, please use `tf.summary.image` instead. Please check
[Migrating tf.summary usage to
TF 2.0](https://www.tensorflow.org/tensorboard/migrate#in_tf_1x) for concrete
steps for migration.
#### How to Map Arguments
| TF1 Arg Name | TF2 Arg Name | Note |
| :------------ | :-------------- | :------------------------------------- |
| `name` | `name` | - |
| `tensor` | `data` | - |
| - | `step` | Explicit int64-castable monotonic step |
: : : value. If omitted, this defaults to :
: : : `tf.summary.experimental.get_step()`. :
| `max_outputs` | `max_outputs` | - |
| `collections` | Not Supported | - |
| `family` | Removed | Please use `tf.name_scope` instead |
: : : to manage summary name prefix. :
| - | `description` | Optional long-form `str` description |
: : : for the summary. Markdown is supported.:
: : : Defaults to empty. :
@end_compatibility
"""
if _distribute_summary_op_util.skip_summary():
return _constant_op.constant('')
# Special case: invoke v2 op for TF2 users who have a v2 writer.
if _should_invoke_v2_op():
# Defer the import to happen inside the symbol to prevent breakage due to
# missing dependency.
with _compat_summary_scope(name, family) as tag:
tb_summary.image(
name=tag,
data=tensor,
step=_get_step_for_v2(),
max_outputs=max_outputs)
# Return an empty Tensor, which will be acceptable as an input to the
# `tf.compat.v1.summary.merge()` API.
return _constant_op.constant(b'')
# Fall back to legacy v1 image implementation.
with _summary_op_util.summary_scope(
name, family, values=[tensor]) as (tag, scope):
val = _gen_logging_ops.image_summary(
tag=tag, tensor=tensor, max_images=max_outputs, name=scope)
_summary_op_util.collect(val, collections, [_ops.GraphKeys.SUMMARIES])
return val
@tf_export(v1=['summary.histogram'])
def histogram(name, values, collections=None, family=None):
# pylint: disable=line-too-long
"""Outputs a `Summary` protocol buffer with a histogram.
Adding a histogram summary makes it possible to visualize your data's
distribution in TensorBoard. You can see a detailed explanation of the
TensorBoard histogram dashboard
[here](https://www.tensorflow.org/get_started/tensorboard_histograms).
The generated
[`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
has one summary value containing a histogram for `values`.
This op reports an `InvalidArgument` error if any value is not finite.
Args:
name: A name for the generated node. Will also serve as a series name in
TensorBoard.
values: A real numeric `Tensor`. Any shape. Values to use to
build the histogram.
collections: Optional list of graph collections keys. The new summary op is
added to these collections. Defaults to `[GraphKeys.SUMMARIES]`.
family: Optional; if provided, used as the prefix of the summary tag name,
which controls the tab name used for display on Tensorboard.
Returns:
A scalar `Tensor` of type `string`. The serialized `Summary` protocol
buffer.
@compatibility(TF2)
For compatibility purposes, when invoked in TF2 where the outermost context is
eager mode, this API will check if there is a suitable TF2 summary writer
context available, and if so will forward this call to that writer instead. A
"suitable" writer context means that the writer is set as the default writer,
and there is an associated non-empty value for `step` (see
`tf.summary.SummaryWriter.as_default`, `tf.summary.experimental.set_step` or
alternatively `tf.compat.v1.train.create_global_step`). For the forwarded
call, the arguments here will be passed to the TF2 implementation of
`tf.summary.histogram`, and the return value will be an empty bytestring
tensor, to avoid duplicate summary writing. This forwarding is best-effort and
not all arguments will be preserved.
To migrate to TF2, please use `tf.summary.histogram` instead. Please check
[Migrating tf.summary usage to
TF 2.0](https://www.tensorflow.org/tensorboard/migrate#in_tf_1x) for concrete
steps for migration.
#### How to Map Arguments
| TF1 Arg Name | TF2 Arg Name | Note |
| :------------ | :-------------- | :------------------------------------- |
| `name` | `name` | - |
| `values` | `data` | - |
| - | `step` | Explicit int64-castable monotonic step |
: : : value. If omitted, this defaults to :
: : : `tf.summary.experimental.get_step()` :
| - | `buckets` | Optional positive `int` specifying |
: : : the histogram bucket number. :
| `collections` | Not Supported | - |
| `family` | Removed | Please use `tf.name_scope` instead |
: : : to manage summary name prefix. :
| - | `description` | Optional long-form `str` description |
: : : for the summary. Markdown is supported.:
: : : Defaults to empty. :
@end_compatibility
"""
if _distribute_summary_op_util.skip_summary():
return _constant_op.constant('')
# Special case: invoke v2 op for TF2 users who have a v2 writer.
if _should_invoke_v2_op():
# Defer the import to happen inside the symbol to prevent breakage due to
# missing dependency.
with _compat_summary_scope(name, family) as tag:
tb_summary.histogram(name=tag, data=values, step=_get_step_for_v2())
# Return an empty Tensor, which will be acceptable as an input to the
# `tf.compat.v1.summary.merge()` API.
return _constant_op.constant(b'')
# Fall back to legacy v1 histogram implementation.
with _summary_op_util.summary_scope(
name, family, values=[values],
default_name='HistogramSummary') as (tag, scope):
val = _gen_logging_ops.histogram_summary(
tag=tag, values=values, name=scope)
_summary_op_util.collect(val, collections, [_ops.GraphKeys.SUMMARIES])
return val
@tf_export(v1=['summary.audio'])
def audio(name, tensor, sample_rate, max_outputs=3, collections=None,
family=None):
# pylint: disable=line-too-long
"""Outputs a `Summary` protocol buffer with audio.
The summary has up to `max_outputs` summary values containing audio. The
audio is built from `tensor` which must be 3-D with shape `[batch_size,
frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
assumed to be in the range of `[-1.0, 1.0]` with a sample rate of
`sample_rate`.
The `tag` in the outputted Summary.Value protobufs is generated based on the
name, with a suffix depending on the max_outputs setting:
* If `max_outputs` is 1, the summary value tag is '*name*/audio'.
* If `max_outputs` is greater than 1, the summary value tags are
generated sequentially as '*name*/audio/0', '*name*/audio/1', etc
Args:
name: A name for the generated node. Will also serve as a series name in
TensorBoard.
tensor: A 3-D `float32` `Tensor` of shape `[batch_size, frames, channels]`
or a 2-D `float32` `Tensor` of shape `[batch_size, frames]`.
sample_rate: A Scalar `float32` `Tensor` indicating the sample rate of the
signal in hertz.
max_outputs: Max number of batch elements to generate audio for.
collections: Optional list of ops.GraphKeys. The collections to add the
summary to. Defaults to [_ops.GraphKeys.SUMMARIES]
family: Optional; if provided, used as the prefix of the summary tag name,
which controls the tab name used for display on Tensorboard.
Returns:
A scalar `Tensor` of type `string`. The serialized `Summary` protocol
buffer.
@compatibility(TF2)
For compatibility purposes, when invoked in TF2 where the outermost context is
eager mode, this API will check if there is a suitable TF2 summary writer
context available, and if so will forward this call to that writer instead. A
"suitable" writer context means that the writer is set as the default writer,
and there is an associated non-empty value for `step` (see
`tf.summary.SummaryWriter.as_default`, `tf.summary.experimental.set_step` or
alternatively `tf.compat.v1.train.create_global_step`). For the forwarded
call, the arguments here will be passed to the TF2 implementation of
`tf.summary.audio`, and the return value will be an empty bytestring tensor,
to avoid duplicate summary writing. This forwarding is best-effort and not all
arguments will be preserved. Additionally:
* The TF2 op just outputs the data under a single tag that contains multiple
samples, rather than multiple tags (i.e. no "/0" or "/1" suffixes).
To migrate to TF2, please use `tf.summary.audio` instead. Please check
[Migrating tf.summary usage to
TF 2.0](https://www.tensorflow.org/tensorboard/migrate#in_tf_1x) for concrete
steps for migration.
#### How to Map Arguments
| TF1 Arg Name | TF2 Arg Name | Note |
| :------------ | :-------------- | :------------------------------------- |
| `name` | `name` | - |
| `tensor` | `data` | Input for this argument now must be |
: : : three-dimensional `[k, t, c]`, where :
: : : `k` is the number of audio clips, `t` :
: : : is the number of frames, and `c` is :
: : : the number of channels. Two-dimensional:
: : : input is no longer supported. :
| `sample_rate` | `sample_rate` | - |
| - | `step` | Explicit int64-castable monotonic step |
: : : value. If omitted, this defaults to :
: : : `tf.summary.experimental.get_step()`. :
| `max_outputs` | `max_outputs` | - |
| `collections` | Not Supported | - |
| `family` | Removed | Please use `tf.name_scope` instead to |
: : : manage summary name prefix. :
| - | `encoding` | Optional constant str for the desired |
: : : encoding. Check the docs for :
: : : `tf.summary.audio` for latest supported:
: : : audio formats. :
| - | `description` | Optional long-form `str` description |
: : : for the summary. Markdown is supported.:
: : : Defaults to empty. :
@end_compatibility
"""
if _distribute_summary_op_util.skip_summary():
return _constant_op.constant('')
# Special case: invoke v2 op for TF2 users who have a v2 writer.
if _should_invoke_v2_op():
# Defer the import to happen inside the symbol to prevent breakage due to
# missing dependency.
if tensor.shape.rank == 2:
# TF2 op requires 3-D tensor, add the `channels` dimension.
tensor = _array_ops.expand_dims_v2(tensor, axis=2)
with _compat_summary_scope(name, family) as tag:
tb_summary.audio(
name=tag,
data=tensor,
sample_rate=sample_rate,
step=_get_step_for_v2(),
max_outputs=max_outputs,
)
# Return an empty Tensor, which will be acceptable as an input to the
# `tf.compat.v1.summary.merge()` API.
return _constant_op.constant(b'')
# Fall back to legacy v1 audio implementation.
with _summary_op_util.summary_scope(
name, family=family, values=[tensor]) as (tag, scope):
sample_rate = _ops.convert_to_tensor(
sample_rate, dtype=_dtypes.float32, name='sample_rate')
val = _gen_logging_ops.audio_summary_v2(
tag=tag, tensor=tensor, max_outputs=max_outputs,
sample_rate=sample_rate, name=scope)
_summary_op_util.collect(val, collections, [_ops.GraphKeys.SUMMARIES])
return val
@tf_export(v1=['summary.text'])
def text(name, tensor, collections=None):
"""Summarizes textual data.
Text data summarized via this plugin will be visible in the Text Dashboard
in TensorBoard. The standard TensorBoard Text Dashboard will render markdown
in the strings, and will automatically organize 1d and 2d tensors into tables.
If a tensor with more than 2 dimensions is provided, a 2d subarray will be
displayed along with a warning message. (Note that this behavior is not
intrinsic to the text summary api, but rather to the default TensorBoard text
plugin.)
Args:
name: A name for the generated node. Will also serve as a series name in
TensorBoard.
tensor: a string-type Tensor to summarize.
collections: Optional list of ops.GraphKeys. The collections to add the
summary to. Defaults to [_ops.GraphKeys.SUMMARIES]
Returns:
A TensorSummary op that is configured so that TensorBoard will recognize
that it contains textual data. The TensorSummary is a scalar `Tensor` of
type `string` which contains `Summary` protobufs.
Raises:
ValueError: If tensor has the wrong type.
@compatibility(TF2)
For compatibility purposes, when invoked in TF2 where the outermost context is
eager mode, this API will check if there is a suitable TF2 summary writer
context available, and if so will forward this call to that writer instead. A
"suitable" writer context means that the writer is set as the default writer,
and there is an associated non-empty value for `step` (see
`tf.summary.SummaryWriter.as_default`, `tf.summary.experimental.set_step` or
alternatively `tf.compat.v1.train.create_global_step`). For the forwarded
call, the arguments here will be passed to the TF2 implementation of
`tf.summary.text`, and the return value will be an empty bytestring tensor, to
avoid duplicate summary writing. This forwarding is best-effort and not all
arguments will be preserved.
To migrate to TF2, please use `tf.summary.text` instead. Please check
[Migrating tf.summary usage to
TF 2.0](https://www.tensorflow.org/tensorboard/migrate#in_tf_1x) for concrete
steps for migration.
#### How to Map Arguments
| TF1 Arg Name | TF2 Arg Name | Note |
| :------------ | :-------------- | :------------------------------------- |
| `name` | `name` | - |
| `tensor` | `data` | - |
| - | `step` | Explicit int64-castable monotonic step |
: : : value. If omitted, this defaults to :
: : : `tf.summary.experimental.get_step()`. :
| `collections` | Not Supported | - |
| - | `description` | Optional long-form `str` description |
: : : for the summary. Markdown is supported.:
: : : Defaults to empty. :
@end_compatibility
"""
if tensor.dtype != _dtypes.string:
raise ValueError('Expected tensor %s to have dtype string, got %s' %
(tensor.name, tensor.dtype))
# Special case: invoke v2 op for TF2 users who have a v2 writer.
if _should_invoke_v2_op():
# `skip_summary` check for v1 op case is done in `tensor_summary`.
if _distribute_summary_op_util.skip_summary():
return _constant_op.constant('')
# Defer the import to happen inside the symbol to prevent breakage due to
# missing dependency.
tb_summary.text(name=name, data=tensor, step=_get_step_for_v2())
# Return an empty Tensor, which will be acceptable as an input to the
# `tf.compat.v1.summary.merge()` API.
return _constant_op.constant(b'')
# Fall back to legacy v1 text implementation.
summary_metadata = _SummaryMetadata(
plugin_data=_SummaryMetadata.PluginData(plugin_name='text'))
t_summary = tensor_summary(
name=name,
tensor=tensor,
summary_metadata=summary_metadata,
collections=collections)
return t_summary
@tf_export(v1=['summary.tensor_summary'])
def tensor_summary(name,
tensor,
summary_description=None,
collections=None,
summary_metadata=None,
family=None,
display_name=None):
"""Outputs a `Summary` protocol buffer with a serialized tensor.proto.
Args:
name: A name for the generated node. If display_name is not set, it will
also serve as the tag name in TensorBoard. (In that case, the tag
name will inherit tf name scopes.)
tensor: A tensor of any type and shape to serialize.
summary_description: A long description of the summary sequence. Markdown
is supported.
collections: Optional list of graph collections keys. The new summary op is
added to these collections. Defaults to `[GraphKeys.SUMMARIES]`.
summary_metadata: Optional SummaryMetadata proto (which describes which
plugins may use the summary value).
family: Optional; if provided, used as the prefix of the summary tag,
which controls the name used for display on TensorBoard when
display_name is not set.
display_name: A string used to name this data in TensorBoard. If this is
not set, then the node name will be used instead.
Returns:
A scalar `Tensor` of type `string`. The serialized `Summary` protocol
buffer.
"""
if summary_metadata is None:
summary_metadata = _SummaryMetadata()
if summary_description is not None:
summary_metadata.summary_description = summary_description
if display_name is not None:
summary_metadata.display_name = display_name
serialized_summary_metadata = summary_metadata.SerializeToString()
if _distribute_summary_op_util.skip_summary():
return _constant_op.constant('')
with _summary_op_util.summary_scope(
name, family, values=[tensor]) as (tag, scope):
val = _gen_logging_ops.tensor_summary_v2(
tensor=tensor,
tag=tag,
name=scope,
serialized_summary_metadata=serialized_summary_metadata)
_summary_op_util.collect(val, collections, [_ops.GraphKeys.SUMMARIES])
return val
@tf_export(v1=['summary.merge'])
def merge(inputs, collections=None, name=None):
# pylint: disable=line-too-long
"""Merges summaries.
This op creates a
[`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
protocol buffer that contains the union of all the values in the input
summaries.
When the Op is run, it reports an `InvalidArgument` error if multiple values
in the summaries to merge use the same tag.
Args:
inputs: A list of `string` `Tensor` objects containing serialized `Summary`
protocol buffers.
collections: Optional list of graph collections keys. The new summary op is
added to these collections. Defaults to `[]`.
name: A name for the operation (optional).
Returns:
A scalar `Tensor` of type `string`. The serialized `Summary` protocol
buffer resulting from the merging.
Raises:
RuntimeError: If called with eager mode enabled.
@compatibility(TF2)
This API is not compatible with eager execution or `tf.function`. To migrate
to TF2, this API can be omitted entirely, because in TF2 individual summary
ops, like `tf.summary.scalar()`, write directly to the default summary writer
if one is active. Thus, it's not necessary to merge summaries or to manually
add the resulting merged summary output to the writer. See the usage example
shown below.
For a comprehensive `tf.summary` migration guide, please follow
[Migrating tf.summary usage to
TF 2.0](https://www.tensorflow.org/tensorboard/migrate#in_tf_1x).
#### TF1 & TF2 Usage Example
TF1:
```python
dist = tf.compat.v1.placeholder(tf.float32, [100])
tf.compat.v1.summary.histogram(name="distribution", values=dist)
writer = tf.compat.v1.summary.FileWriter("/tmp/tf1_summary_example")
summaries = tf.compat.v1.summary.merge_all()
sess = tf.compat.v1.Session()
for step in range(100):
mean_moving_normal = np.random.normal(loc=step, scale=1, size=[100])
summ = sess.run(summaries, feed_dict={dist: mean_moving_normal})
writer.add_summary(summ, global_step=step)
```
TF2:
```python
writer = tf.summary.create_file_writer("/tmp/tf2_summary_example")
for step in range(100):
mean_moving_normal = np.random.normal(loc=step, scale=1, size=[100])
with writer.as_default(step=step):
tf.summary.histogram(name='distribution', data=mean_moving_normal)
```
@end_compatibility
"""
# pylint: enable=line-too-long
if _context.executing_eagerly():
raise RuntimeError(
'Merging tf.summary.* ops is not compatible with eager execution. '
'Use tf.contrib.summary instead.')
if _distribute_summary_op_util.skip_summary():
return _constant_op.constant('')
name = _summary_op_util.clean_tag(name)
with _ops.name_scope(name, 'Merge', inputs):
val = _gen_logging_ops.merge_summary(inputs=inputs, name=name)
_summary_op_util.collect(val, collections, [])
return val
@tf_export(v1=['summary.merge_all'])
def merge_all(key=_ops.GraphKeys.SUMMARIES, scope=None, name=None):
"""Merges all summaries collected in the default graph.
Args:
key: `GraphKey` used to collect the summaries. Defaults to
`GraphKeys.SUMMARIES`.
scope: Optional scope used to filter the summary ops, using `re.match`.
name: A name for the operation (optional).
Returns:
If no summaries were collected, returns None. Otherwise returns a scalar
`Tensor` of type `string` containing the serialized `Summary` protocol
buffer resulting from the merging.
Raises:
RuntimeError: If called with eager execution enabled.
@compatibility(TF2)
This API is not compatible with eager execution or `tf.function`. To migrate
to TF2, this API can be omitted entirely, because in TF2 individual summary
ops, like `tf.summary.scalar()`, write directly to the default summary writer
if one is active. Thus, it's not necessary to merge summaries or to manually
add the resulting merged summary output to the writer. See the usage example
shown below.
For a comprehensive `tf.summary` migration guide, please follow
[Migrating tf.summary usage to
TF 2.0](https://www.tensorflow.org/tensorboard/migrate#in_tf_1x).
#### TF1 & TF2 Usage Example
TF1:
```python
dist = tf.compat.v1.placeholder(tf.float32, [100])
tf.compat.v1.summary.histogram(name="distribution", values=dist)
writer = tf.compat.v1.summary.FileWriter("/tmp/tf1_summary_example")
summaries = tf.compat.v1.summary.merge_all()
sess = tf.compat.v1.Session()
for step in range(100):
mean_moving_normal = np.random.normal(loc=step, scale=1, size=[100])
summ = sess.run(summaries, feed_dict={dist: mean_moving_normal})
writer.add_summary(summ, global_step=step)
```
TF2:
```python
writer = tf.summary.create_file_writer("/tmp/tf2_summary_example")
for step in range(100):
mean_moving_normal = np.random.normal(loc=step, scale=1, size=[100])
with writer.as_default(step=step):
tf.summary.histogram(name='distribution', data=mean_moving_normal)
```
@end_compatibility
"""
if _context.executing_eagerly():
raise RuntimeError(
'Merging tf.summary.* ops is not compatible with eager execution. '
'Use tf.contrib.summary instead.')
summary_ops = _ops.get_collection(key, scope=scope)
if not summary_ops:
return None
else:
return merge(summary_ops, name=name)
@tf_export(v1=['summary.get_summary_description'])
def get_summary_description(node_def):
"""Given a TensorSummary node_def, retrieve its SummaryDescription.
When a Summary op is instantiated, a SummaryDescription of associated
metadata is stored in its NodeDef. This method retrieves the description.
Args:
node_def: the node_def_pb2.NodeDef of a TensorSummary op
Returns:
a summary_pb2.SummaryDescription
Raises:
ValueError: if the node is not a summary op.
@compatibility(eager)
Not compatible with eager execution. To write TensorBoard
summaries under eager execution, use `tf.contrib.summary` instead.
@end_compatibility
"""
if node_def.op != 'TensorSummary':
raise ValueError("Can't get_summary_description on %s" % node_def.op)
description_str = _compat.as_str_any(node_def.attr['description'].s)
summary_description = SummaryDescription()
_json_format.Parse(description_str, summary_description)
return summary_description
def _get_step_for_v2():
"""Get step for v2 summary invocation in v1.
In order to invoke v2 op in `tf.compat.v1.summary`, global step needs to be
set for the v2 summary writer.
Returns:
The step set by `tf.summary.experimental.set_step` or
`tf.compat.v1.train.create_global_step`, or None is no step has been
set.
"""
step = _summary_ops_v2.get_step()
if step is not None:
return step
return _training_util.get_global_step()
def _should_invoke_v2_op():
"""Check if v2 op can be invoked.
When calling TF1 summary op in eager mode, if the following conditions are
met, v2 op will be invoked:
- The outermost context is eager mode.
- A default TF2 summary writer is present.
- A step is set for the writer (using `tf.summary.SummaryWriter.as_default`,
`tf.summary.experimental.set_step` or
`tf.compat.v1.train.create_global_step`).
Returns:
A boolean indicating whether v2 summary op should be invoked.
"""
# Check if in eager mode.
if not _ops.executing_eagerly_outside_functions():
return False
# Check if a default summary writer is present.
if not _summary_ops_v2.has_default_writer():
warnings.warn(
'Cannot activate TF2 compatibility support for TF1 summary ops: '
'default summary writer not found.')
return False
# Check if a step is set for the writer.
if _get_step_for_v2() is None:
warnings.warn(
'Cannot activate TF2 compatibility support for TF1 summary ops: '
'global step not set. To set step for summary writer, '
'use `tf.summary.SummaryWriter.as_default(step=_)`, '
'`tf.summary.experimental.set_step()` or '
'`tf.compat.v1.train.create_global_step()`.')
return False
return True
@contextlib.contextmanager
def _compat_summary_scope(name, family):
"""Handles `family` argument for v2 op invocation in v1."""
# Get a new summary tag name with the `family` arg.
with _summary_op_util.summary_scope(name, family) as (tag, _):
# Reset the root name scope with an empty summary_scope.
with _summary_op_util.summary_scope(name='', family=None):
yield tag
@@ -0,0 +1,91 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides a method for reading events from an event file via an iterator."""
from tensorflow.core.util import event_pb2
from tensorflow.python.lib.io import tf_record
from tensorflow.python.util.tf_export import tf_export
class _SummaryIterator(object):
"""Yields `Event` protocol buffers from a given path."""
def __init__(self, path):
self._tf_record_iterator = tf_record.tf_record_iterator(path)
def __iter__(self):
return self
def __next__(self):
r = next(self._tf_record_iterator)
return event_pb2.Event.FromString(r)
next = __next__
@tf_export(v1=['train.summary_iterator'])
def summary_iterator(path):
# pylint: disable=line-too-long
"""Returns a iterator for reading `Event` protocol buffers from an event file.
You can use this function to read events written to an event file. It returns
a Python iterator that yields `Event` protocol buffers.
Example: Print the contents of an events file.
```python
for e in tf.compat.v1.train.summary_iterator(path to events file):
print(e)
```
Example: Print selected summary values.
```python
# This example supposes that the events file contains summaries with a
# summary value tag 'loss'. These could have been added by calling
# `add_summary()`, passing the output of a scalar summary op created with
# with: `tf.compat.v1.summary.scalar('loss', loss_tensor)`.
for e in tf.compat.v1.train.summary_iterator(path to events file):
for v in e.summary.value:
if v.tag == 'loss':
print(tf.make_ndarray(v.tensor))
```
Example: Continuously check for new summary values.
```python
summaries = tf.compat.v1.train.summary_iterator(path to events file)
while True:
for e in summaries:
for v in e.summary.value:
if v.tag == 'loss':
print(tf.make_ndarray(v.tensor))
# Wait for a bit before checking the file for any new events
time.sleep(wait time)
```
See the protocol buffer definitions of
[Event](https://www.tensorflow.org/code/tensorflow/core/util/event.proto)
and
[Summary](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
for more information about their attributes.
Args:
path: The path to an event file created by a `SummaryWriter`.
Returns:
A iterator that yields `Event` protocol buffers
"""
return _SummaryIterator(path)
@@ -0,0 +1,57 @@
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tensorflow.python.summary.summary_iterator."""
import glob
import os.path
from tensorflow.core.util import event_pb2
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test
from tensorflow.python.summary import summary_iterator
from tensorflow.python.summary.writer import writer
class SummaryIteratorTestCase(test.TestCase):
@test_util.run_deprecated_v1
def testSummaryIteratorEventsAddedAfterEndOfFile(self):
test_dir = os.path.join(self.get_temp_dir(), "events")
with writer.FileWriter(test_dir) as w:
session_log_start = event_pb2.SessionLog.START
w.add_session_log(event_pb2.SessionLog(status=session_log_start), 1)
w.flush()
path = glob.glob(os.path.join(test_dir, "event*"))[0]
rr = summary_iterator.summary_iterator(path)
# The first event should list the file_version.
ev = next(rr)
self.assertEqual("brain.Event:2", ev.file_version)
# The next event should be the START message.
ev = next(rr)
self.assertEqual(1, ev.step)
self.assertEqual(session_log_start, ev.session_log.status)
# Reached EOF.
self.assertRaises(StopIteration, lambda: next(rr))
w.add_session_log(event_pb2.SessionLog(status=session_log_start), 2)
w.flush()
# The new event is read, after previously seeing EOF.
ev = next(rr)
self.assertEqual(2, ev.step)
self.assertEqual(session_log_start, ev.session_log.status)
# Get EOF again.
self.assertRaises(StopIteration, lambda: next(rr))
if __name__ == "__main__":
test.main()
+251
View File
@@ -0,0 +1,251 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for the API surface of the V1 tf.summary ops.
These tests don't check the actual serialized proto summary value for the
more complex summaries (e.g. audio, image). Those test live separately in
tensorflow/python/kernel_tests/summary_v1_*.py.
"""
from tensorflow.core.framework import summary_pb2
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import meta_graph
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import test
from tensorflow.python.summary import summary as summary_lib
class SummaryTest(test.TestCase):
@test_util.run_deprecated_v1
def testScalarSummary(self):
with self.cached_session() as s:
i = constant_op.constant(3)
with ops.name_scope('outer'):
im = summary_lib.scalar('inner', i)
summary_str = s.run(im)
summary = summary_pb2.Summary()
summary.ParseFromString(summary_str)
values = summary.value
self.assertEqual(len(values), 1)
self.assertEqual(values[0].tag, 'outer/inner')
self.assertEqual(values[0].simple_value, 3.0)
@test_util.run_deprecated_v1
def testScalarSummaryWithFamily(self):
with self.cached_session() as s:
i = constant_op.constant(7)
with ops.name_scope('outer'):
im1 = summary_lib.scalar('inner', i, family='family')
self.assertEqual(im1.op.name, 'outer/family/inner')
im2 = summary_lib.scalar('inner', i, family='family')
self.assertEqual(im2.op.name, 'outer/family/inner_1')
sm1, sm2 = s.run([im1, im2])
summary = summary_pb2.Summary()
summary.ParseFromString(sm1)
values = summary.value
self.assertEqual(len(values), 1)
self.assertEqual(values[0].tag, 'family/outer/family/inner')
self.assertEqual(values[0].simple_value, 7.0)
summary.ParseFromString(sm2)
values = summary.value
self.assertEqual(len(values), 1)
self.assertEqual(values[0].tag, 'family/outer/family/inner_1')
self.assertEqual(values[0].simple_value, 7.0)
@test_util.run_deprecated_v1
def testSummarizingVariable(self):
with self.cached_session() as s:
c = constant_op.constant(42.0)
v = variables.Variable(c)
ss = summary_lib.scalar('summary', v)
init = variables.global_variables_initializer()
s.run(init)
summ_str = s.run(ss)
summary = summary_pb2.Summary()
summary.ParseFromString(summ_str)
self.assertEqual(len(summary.value), 1)
value = summary.value[0]
self.assertEqual(value.tag, 'summary')
self.assertEqual(value.simple_value, 42.0)
@test_util.run_deprecated_v1
def testImageSummary(self):
with self.cached_session() as s:
i = array_ops.ones((5, 4, 4, 3))
with ops.name_scope('outer'):
im = summary_lib.image('inner', i, max_outputs=3)
summary_str = s.run(im)
summary = summary_pb2.Summary()
summary.ParseFromString(summary_str)
values = summary.value
self.assertEqual(len(values), 3)
tags = sorted(v.tag for v in values)
expected = sorted('outer/inner/image/{}'.format(i) for i in range(3))
self.assertEqual(tags, expected)
@test_util.run_deprecated_v1
def testImageSummaryWithFamily(self):
with self.cached_session() as s:
i = array_ops.ones((5, 2, 3, 1))
with ops.name_scope('outer'):
im = summary_lib.image('inner', i, max_outputs=3, family='family')
self.assertEqual(im.op.name, 'outer/family/inner')
summary_str = s.run(im)
summary = summary_pb2.Summary()
summary.ParseFromString(summary_str)
values = summary.value
self.assertEqual(len(values), 3)
tags = sorted(v.tag for v in values)
expected = sorted(
'family/outer/family/inner/image/{}'.format(i) for i in range(3))
self.assertEqual(tags, expected)
@test_util.run_deprecated_v1
def testHistogramSummary(self):
with self.cached_session() as s:
i = array_ops.ones((5, 4, 4, 3))
with ops.name_scope('outer'):
summ_op = summary_lib.histogram('inner', i)
summary_str = s.run(summ_op)
summary = summary_pb2.Summary()
summary.ParseFromString(summary_str)
self.assertEqual(len(summary.value), 1)
self.assertEqual(summary.value[0].tag, 'outer/inner')
@test_util.run_deprecated_v1
def testHistogramSummaryWithFamily(self):
with self.cached_session() as s:
i = array_ops.ones((5, 4, 4, 3))
with ops.name_scope('outer'):
summ_op = summary_lib.histogram('inner', i, family='family')
self.assertEqual(summ_op.op.name, 'outer/family/inner')
summary_str = s.run(summ_op)
summary = summary_pb2.Summary()
summary.ParseFromString(summary_str)
self.assertEqual(len(summary.value), 1)
self.assertEqual(summary.value[0].tag, 'family/outer/family/inner')
def testHistogramSummaryTypes(self):
for dtype in (dtypes.int8, dtypes.uint8, dtypes.int16, dtypes.int32,
dtypes.float32, dtypes.float64):
const = constant_op.constant(10, dtype=dtype)
summary_lib.histogram('h', const)
@test_util.run_deprecated_v1
def testAudioSummary(self):
with self.cached_session() as s:
i = array_ops.ones((5, 3, 4))
with ops.name_scope('outer'):
aud = summary_lib.audio('inner', i, 0.2, max_outputs=3)
summary_str = s.run(aud)
summary = summary_pb2.Summary()
summary.ParseFromString(summary_str)
values = summary.value
self.assertEqual(len(values), 3)
tags = sorted(v.tag for v in values)
expected = sorted('outer/inner/audio/{}'.format(i) for i in range(3))
self.assertEqual(tags, expected)
@test_util.run_deprecated_v1
def testAudioSummaryWithFamily(self):
with self.cached_session() as s:
i = array_ops.ones((5, 3, 4))
with ops.name_scope('outer'):
aud = summary_lib.audio('inner', i, 0.2, max_outputs=3, family='family')
self.assertEqual(aud.op.name, 'outer/family/inner')
summary_str = s.run(aud)
summary = summary_pb2.Summary()
summary.ParseFromString(summary_str)
values = summary.value
self.assertEqual(len(values), 3)
tags = sorted(v.tag for v in values)
expected = sorted(
'family/outer/family/inner/audio/{}'.format(i) for i in range(3))
self.assertEqual(tags, expected)
def testAudioSummaryWithInvalidSampleRate(self):
with self.assertRaises(errors.InvalidArgumentError):
invalid_sample_rate = [22000.0, 22000.0]
self.evaluate(summary_lib.audio('', [[1.0]], invalid_sample_rate))
@test_util.run_deprecated_v1
def testTextSummary(self):
with self.cached_session():
with self.assertRaises(ValueError):
num = array_ops.constant(1)
summary_lib.text('foo', num)
# The API accepts vectors.
arr = array_ops.constant(['one', 'two', 'three'])
summ = summary_lib.text('foo', arr)
self.assertEqual(summ.op.type, 'TensorSummaryV2')
# the API accepts scalars
summ = summary_lib.text('foo', array_ops.constant('one'))
self.assertEqual(summ.op.type, 'TensorSummaryV2')
@test_util.run_deprecated_v1
def testSummaryNameConversion(self):
c = constant_op.constant(3)
s = summary_lib.scalar('name with spaces', c)
self.assertEqual(s.op.name, 'name_with_spaces')
s2 = summary_lib.scalar('name with many $#illegal^: characters!', c)
self.assertEqual(s2.op.name, 'name_with_many___illegal___characters_')
s3 = summary_lib.scalar('/name/with/leading/slash', c)
self.assertEqual(s3.op.name, 'name/with/leading/slash')
@test_util.run_deprecated_v1
def testSummaryWithFamilyMetaGraphExport(self):
with ops.name_scope('outer'):
i = constant_op.constant(11)
summ = summary_lib.scalar('inner', i)
self.assertEqual(summ.op.name, 'outer/inner')
summ_f = summary_lib.scalar('inner', i, family='family')
self.assertEqual(summ_f.op.name, 'outer/family/inner')
metagraph_def, _ = meta_graph.export_scoped_meta_graph(export_scope='outer')
with ops.Graph().as_default() as g:
meta_graph.import_scoped_meta_graph(metagraph_def, graph=g,
import_scope='new_outer')
# The summaries should exist, but with outer scope renamed.
new_summ = g.get_tensor_by_name('new_outer/inner:0')
new_summ_f = g.get_tensor_by_name('new_outer/family/inner:0')
# However, the tags are unaffected.
with self.cached_session() as s:
new_summ_str, new_summ_f_str = s.run([new_summ, new_summ_f])
new_summ_pb = summary_pb2.Summary()
new_summ_pb.ParseFromString(new_summ_str)
self.assertEqual('outer/inner', new_summ_pb.value[0].tag)
new_summ_f_pb = summary_pb2.Summary()
new_summ_f_pb.ParseFromString(new_summ_f_str)
self.assertEqual('family/outer/family/inner',
new_summ_f_pb.value[0].tag)
if __name__ == '__main__':
test.main()
@@ -0,0 +1,311 @@
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for the API surface of the V1 tf.summary ops when TF2 is enabled.
V1 summary ops will invoke V2 TensorBoard summary ops in eager mode.
"""
import unittest
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import summary_ops_v2
from tensorflow.python.platform import test
from tensorflow.python.summary import summary as summary_lib
from tensorflow.python.summary import tb_summary
from tensorflow.python.training import training_util
TENSORBOARD_AVAILABLE = (
hasattr(tb_summary, 'TENSORBOARD_AVAILABLE')
and tb_summary.TENSORBOARD_AVAILABLE
)
@unittest.skipIf(not TENSORBOARD_AVAILABLE, 'Tensorboard not installed.')
class SummaryV2Test(test.TestCase):
@test_util.run_v2_only
def test_scalar_summary_v2__w_writer(self):
"""Tests scalar v2 invocation with a v2 writer."""
with test.mock.patch.object(
tb_summary, 'scalar', autospec=True) as mock_scalar_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default(step=1):
i = constant_op.constant(2.5)
tensor = summary_lib.scalar('float', i)
# Returns empty string.
self.assertEqual(tensor.numpy(), b'')
self.assertEqual(tensor.dtype, dtypes.string)
mock_scalar_v2.assert_called_once_with(name='float', data=i, step=1)
@test_util.run_v2_only
def test_scalar_summary_v2__wo_writer(self):
"""Tests scalar v2 invocation with no writer."""
with self.assertWarnsRegex(
UserWarning, 'default summary writer not found'):
with test.mock.patch.object(
tb_summary, 'scalar', autospec=True) as mock_scalar_v2:
summary_lib.scalar('float', constant_op.constant(2.5))
mock_scalar_v2.assert_not_called()
@test_util.run_v2_only
def test_scalar_summary_v2__global_step_not_set(self):
"""Tests scalar v2 invocation when global step is not set."""
with self.assertWarnsRegex(UserWarning, 'global step not set'):
with test.mock.patch.object(
tb_summary, 'scalar', autospec=True) as mock_scalar_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default():
summary_lib.scalar('float', constant_op.constant(2.5))
mock_scalar_v2.assert_not_called()
@test_util.run_v2_only
def test_tf_summary_scalar_invalid_step_shapes_raise(self):
writer = summary_ops_v2.create_summary_file_writer(self.get_temp_dir())
invalid_steps = [
(), # empty tuple
[], # empty list
[1, 2], # non-scalar list
(1, 2), # non-scalar tuple
array_ops.ones((2,)), # rank-1 tensor
array_ops.ones((1, 1)), # rank-2 tensor
]
with writer.as_default(step=1):
for step in invalid_steps:
with self.assertRaises(ValueError):
summary_ops_v2.scalar('loss', 0.5, step=step)
@test_util.run_v2_only
def test_tf_summary_histogram_invalid_step_raises(self):
writer = summary_ops_v2.create_summary_file_writer(self.get_temp_dir())
with writer.as_default(step=1):
with self.assertRaises(ValueError):
summary_ops_v2.histogram(
'h',
array_ops.ones((10,)),
step=(),
)
@test_util.run_v2_only
def test_tf_summary_image_invalid_step_raises(self):
writer = summary_ops_v2.create_summary_file_writer(self.get_temp_dir())
image = array_ops.ones((1, 4, 4, 3))
with writer.as_default(step=1):
with self.assertRaises(ValueError):
summary_ops_v2.image(
'img',
image,
step=(),
)
@test_util.run_v2_only
def test_tf_summary_audio_invalid_step_raises(self):
writer = summary_ops_v2.create_summary_file_writer(self.get_temp_dir())
audio = array_ops.ones((1, 16000, 1))
with writer.as_default(step=1):
with self.assertRaises(ValueError):
summary_ops_v2.audio(
'audio',
audio,
sample_rate=16000,
max_outputs=1,
step=(),
)
@test_util.run_v2_only
def test_tf_summary_write_invalid_step_raises(self):
writer = summary_ops_v2.create_summary_file_writer(self.get_temp_dir())
with writer.as_default(step=1):
with self.assertRaises(ValueError):
summary_ops_v2.write(
tag='tag',
tensor=constant_op.constant(1.0),
step=(),
)
@test_util.run_v2_only
def test_tf_summary_trace_export_invalid_step_raises(self):
writer = summary_ops_v2.create_summary_file_writer(self.get_temp_dir())
with writer.as_default():
summary_ops_v2.trace_on(graph=True)
with self.assertRaises(ValueError):
summary_ops_v2.trace_export('trace', step=())
@test_util.run_v2_only
def test_scalar_summary_v2__family(self):
"""Tests `family` arg handling when scalar v2 is invoked."""
with test.mock.patch.object(
tb_summary, 'scalar', autospec=True) as mock_scalar_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default(step=1):
tensor = summary_lib.scalar(
'float', constant_op.constant(2.5), family='otter')
# Returns empty string.
self.assertEqual(tensor.numpy(), b'')
self.assertEqual(tensor.dtype, dtypes.string)
mock_scalar_v2.assert_called_once_with(
name='otter/otter/float',
data=constant_op.constant(2.5),
step=1,
)
@test_util.run_v2_only
def test_scalar_summary_v2__family_w_outer_scope(self):
"""Tests `family` arg handling when there is an outer scope."""
with test.mock.patch.object(
tb_summary, 'scalar', autospec=True) as mock_scalar_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default(step=1):
with ops.name_scope_v2('sea'):
tensor = summary_lib.scalar(
'float', constant_op.constant(3.5), family='crabnet')
# Returns empty string.
self.assertEqual(tensor.numpy(), b'')
self.assertEqual(tensor.dtype, dtypes.string)
mock_scalar_v2.assert_called_once_with(
name='crabnet/sea/crabnet/float',
data=constant_op.constant(3.5),
step=1,
)
@test_util.run_v2_only
def test_scalar_summary_v2__v1_set_step(self):
"""Tests scalar v2 invocation when v1 step is set."""
global_step = training_util.create_global_step()
global_step.assign(1024)
with test.mock.patch.object(
tb_summary, 'scalar', autospec=True) as mock_scalar_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default():
i = constant_op.constant(2.5)
tensor = summary_lib.scalar('float', i)
# Returns empty string.
self.assertEqual(tensor.numpy(), b'')
self.assertEqual(tensor.dtype, dtypes.string)
mock_scalar_v2.assert_called_once_with(name='float', data=i, step=1024)
@test_util.run_v2_only
def test_image_summary_v2(self):
"""Tests image v2 invocation."""
with test.mock.patch.object(
tb_summary, 'image', autospec=True) as mock_image_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default(step=2):
i = array_ops.ones((5, 4, 4, 3))
with ops.name_scope_v2('outer'):
tensor = summary_lib.image('image', i, max_outputs=3, family='family')
# Returns empty string.
self.assertEqual(tensor.numpy(), b'')
self.assertEqual(tensor.dtype, dtypes.string)
mock_image_v2.assert_called_once_with(
name='family/outer/family/image',
data=i,
step=2,
max_outputs=3,
)
@test_util.run_v2_only
def test_histogram_summary_v2(self):
"""Tests histogram v2 invocation."""
with test.mock.patch.object(
tb_summary, 'histogram', autospec=True) as mock_histogram_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default(step=3):
i = array_ops.ones((1024,))
tensor = summary_lib.histogram('histogram', i, family='family')
# Returns empty string.
self.assertEqual(tensor.numpy(), b'')
self.assertEqual(tensor.dtype, dtypes.string)
mock_histogram_v2.assert_called_once_with(
name='family/family/histogram',
data=i,
step=3,
)
@test_util.run_v2_only
def test_audio_summary_v2(self):
"""Tests audio v2 invocation."""
with test.mock.patch.object(
tb_summary, 'audio', autospec=True) as mock_audio_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default(step=10):
i = array_ops.ones((5, 3, 4))
with ops.name_scope_v2('dolphin'):
tensor = summary_lib.audio('wave', i, 0.2, max_outputs=3)
# Returns empty string.
self.assertEqual(tensor.numpy(), b'')
self.assertEqual(tensor.dtype, dtypes.string)
mock_audio_v2.assert_called_once_with(
name='dolphin/wave',
data=i,
sample_rate=0.2,
step=10,
max_outputs=3,
)
@test_util.run_v2_only
def test_audio_summary_v2__2d_tensor(self):
"""Tests audio v2 invocation with 2-D tensor input."""
with test.mock.patch.object(
tb_summary, 'audio', autospec=True) as mock_audio_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default(step=11):
input_2d = array_ops.ones((5, 3))
tensor = summary_lib.audio('wave', input_2d, 0.2, max_outputs=3)
# Returns empty string.
self.assertEqual(tensor.numpy(), b'')
self.assertEqual(tensor.dtype, dtypes.string)
mock_audio_v2.assert_called_once_with(
name='wave',
data=test.mock.ANY,
sample_rate=0.2,
step=11,
max_outputs=3,
)
input_3d = array_ops.ones((5, 3, 1)) # 3-D input tensor
self.assertAllEqual(mock_audio_v2.call_args[1]['data'], input_3d)
@test_util.run_v2_only
def test_text_summary_v2(self):
"""Tests text v2 invocation."""
with test.mock.patch.object(
tb_summary, 'text', autospec=True) as mock_text_v2:
with summary_ops_v2.create_summary_file_writer(
self.get_temp_dir()).as_default(step=22):
i = constant_op.constant('lorem ipsum', dtype=dtypes.string)
tensor = summary_lib.text('text', i)
# Returns empty string.
self.assertEqual(tensor.numpy(), b'')
self.assertEqual(tensor.dtype, dtypes.string)
mock_text_v2.assert_called_once_with(name='text', data=i, step=22)
if __name__ == '__main__':
test.main()
+378
View File
@@ -0,0 +1,378 @@
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Re-exports the APIs of TF2 summary that live in TensorBoard."""
from tensorflow.python.util.tf_export import tf_export
try:
from tensorboard.summary.v2 import scalar as scalar_v2_lib # pylint: disable=g-import-not-at-top
TENSORBOARD_AVAILABLE = True
del scalar_v2_lib
except ImportError:
TENSORBOARD_AVAILABLE = False
_TENSORBOARD_NOT_INSTALLED_ERROR = (
"TensorBoard is not installed, missing implementation for {}. "
"Please install TensorBoard via `pip install tensorboard`."
)
class TBNotInstalledError(ImportError):
def __init__(self, summary_api):
self.error_message = _TENSORBOARD_NOT_INSTALLED_ERROR.format(summary_api)
super().__init__(self.error_message)
@tf_export("summary.audio", v1=[])
def audio(
name,
data,
sample_rate,
step=None,
max_outputs=3,
encoding=None,
description=None,
):
"""Write an audio summary.
Arguments:
name: A name for this summary. The summary tag used for TensorBoard will be
this name prefixed by any active name scopes.
data: A `Tensor` representing audio data with shape `[k, t, c]`, where `k`
is the number of audio clips, `t` is the number of frames, and `c` is the
number of channels. Elements should be floating-point values in `[-1.0,
1.0]`. Any of the dimensions may be statically unknown (i.e., `None`).
sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the sample
rate, in Hz. Must be positive.
step: Explicit `int64`-castable monotonic step value for this summary. If
omitted, this defaults to `tf.summary.experimental.get_step()`, which must
not be None.
max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this many
audio clips will be emitted at each step. When more than `max_outputs`
many clips are provided, the first `max_outputs` many clips will be used
and the rest silently discarded.
encoding: Optional constant `str` for the desired encoding. Only "wav" is
currently supported, but this is not guaranteed to remain the default, so
if you want "wav" in particular, set this explicitly.
description: Optional long-form description for this summary, as a constant
`str`. Markdown is supported. Defaults to empty.
Returns:
True on success, or false if no summary was emitted because no default
summary writer was available.
Raises:
ValueError: if a default writer exists, but no step was provided and
`tf.summary.experimental.get_step()` is None.
"""
try:
from tensorboard.summary.v2 import audio as audio_v2 # pylint: disable=g-import-not-at-top, g-importing-member
except ImportError as exc:
raise TBNotInstalledError("tf.summary.audio") from exc
return audio_v2(
name=name,
data=data,
sample_rate=sample_rate,
step=step,
max_outputs=max_outputs,
encoding=encoding,
description=description,
)
@tf_export("summary.histogram", v1=[])
def histogram(name, data, step=None, buckets=None, description=None):
"""Write a histogram summary.
See also `tf.summary.scalar`, `tf.summary.SummaryWriter`.
Writes a histogram to the current default summary writer, for later analysis
in TensorBoard's 'Histograms' and 'Distributions' dashboards (data written
using this API will appear in both places). Like `tf.summary.scalar` points,
each histogram is associated with a `step` and a `name`. All the histograms
with the same `name` constitute a time series of histograms.
The histogram is calculated over all the elements of the given `Tensor`
without regard to its shape or rank.
This example writes 2 histograms:
```python
w = tf.summary.create_file_writer('test/logs')
with w.as_default():
tf.summary.histogram("activations", tf.random.uniform([100, 50]), step=0)
tf.summary.histogram("initial_weights", tf.random.normal([1000]), step=0)
```
A common use case is to examine the changing activation patterns (or lack
thereof) at specific layers in a neural network, over time.
```python
w = tf.summary.create_file_writer('test/logs')
with w.as_default():
for step in range(100):
# Generate fake "activations".
activations = [
tf.random.normal([1000], mean=step, stddev=1),
tf.random.normal([1000], mean=step, stddev=10),
tf.random.normal([1000], mean=step, stddev=100),
]
tf.summary.histogram("layer1/activate", activations[0], step=step)
tf.summary.histogram("layer2/activate", activations[1], step=step)
tf.summary.histogram("layer3/activate", activations[2], step=step)
```
Arguments:
name: A name for this summary. The summary tag used for TensorBoard will be
this name prefixed by any active name scopes.
data: A `Tensor` of any shape. The histogram is computed over its elements,
which must be castable to `float64`.
step: Explicit `int64`-castable monotonic step value for this summary. If
omitted, this defaults to `tf.summary.experimental.get_step()`, which must
not be None.
buckets: Optional positive `int`. The output will have this many buckets,
except in two edge cases. If there is no data, then there are no buckets.
If there is data but all points have the same value, then all buckets'
left and right endpoints are the same and only the last bucket has nonzero
count. Defaults to 30 if not specified.
description: Optional long-form description for this summary, as a constant
`str`. Markdown is supported. Defaults to empty.
Returns:
True on success, or false if no summary was emitted because no default
summary writer was available.
Raises:
ValueError: if a default writer exists, but no step was provided and
`tf.summary.experimental.get_step()` is None.
"""
try:
from tensorboard.summary.v2 import histogram as histogram_v2 # pylint: disable=g-import-not-at-top, g-importing-member
except ImportError as exc:
raise TBNotInstalledError("tf.summary.histogram") from exc
return histogram_v2(
name=name, data=data, step=step, buckets=buckets, description=description
)
@tf_export("summary.image", v1=[])
def image(name, data, step=None, max_outputs=3, description=None):
"""Write an image summary.
See also `tf.summary.scalar`, `tf.summary.SummaryWriter`.
Writes a collection of images to the current default summary writer. Data
appears in TensorBoard's 'Images' dashboard. Like `tf.summary.scalar` points,
each collection of images is associated with a `step` and a `name`. All the
image collections with the same `name` constitute a time series of image
collections.
This example writes 2 random grayscale images:
```python
w = tf.summary.create_file_writer('test/logs')
with w.as_default():
image1 = tf.random.uniform(shape=[8, 8, 1])
image2 = tf.random.uniform(shape=[8, 8, 1])
tf.summary.image("grayscale_noise", [image1, image2], step=0)
```
To avoid clipping, data should be converted to one of the following:
- floating point values in the range [0,1], or
- uint8 values in the range [0,255]
```python
# Convert the original dtype=int32 `Tensor` into `dtype=float64`.
rgb_image_float = tf.constant([
[[1000, 0, 0], [0, 500, 1000]],
]) / 1000
tf.summary.image("picture", [rgb_image_float], step=0)
# Convert original dtype=uint8 `Tensor` into proper range.
rgb_image_uint8 = tf.constant([
[[1, 1, 0], [0, 0, 1]],
], dtype=tf.uint8) * 255
tf.summary.image("picture", [rgb_image_uint8], step=1)
```
Arguments:
name: A name for this summary. The summary tag used for TensorBoard will be
this name prefixed by any active name scopes.
data: A `Tensor` representing pixel data with shape `[k, h, w, c]`, where
`k` is the number of images, `h` and `w` are the height and width of the
images, and `c` is the number of channels, which should be 1, 2, 3, or 4
(grayscale, grayscale with alpha, RGB, RGBA). Any of the dimensions may be
statically unknown (i.e., `None`). Floating point data will be clipped to
the range [0,1]. Other data types will be clipped into an allowed range
for safe casting to uint8, using `tf.image.convert_image_dtype`.
step: Explicit `int64`-castable monotonic step value for this summary. If
omitted, this defaults to `tf.summary.experimental.get_step()`, which must
not be None.
max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this many
images will be emitted at each step. When more than `max_outputs` many
images are provided, the first `max_outputs` many images will be used and
the rest silently discarded.
description: Optional long-form description for this summary, as a constant
`str`. Markdown is supported. Defaults to empty.
Returns:
True on success, or false if no summary was emitted because no default
summary writer was available.
Raises:
ValueError: if a default writer exists, but no step was provided and
`tf.summary.experimental.get_step()` is None.
"""
try:
from tensorboard.summary.v2 import image as image_v2 # pylint: disable=g-import-not-at-top, g-importing-member
except ImportError as exc:
raise TBNotInstalledError("tf.summary.image") from exc
return image_v2(
name=name,
data=data,
step=step,
max_outputs=max_outputs,
description=description,
)
@tf_export("summary.scalar", v1=[])
def scalar(name, data, step=None, description=None):
"""Write a scalar summary.
See also `tf.summary.image`, `tf.summary.histogram`,
`tf.summary.SummaryWriter`.
Writes simple numeric values for later analysis in TensorBoard. Writes go to
the current default summary writer. Each summary point is associated with an
integral `step` value. This enables the incremental logging of time series
data. A common usage of this API is to log loss during training to produce
a loss curve.
For example:
```python
test_summary_writer = tf.summary.create_file_writer('test/logdir')
with test_summary_writer.as_default():
tf.summary.scalar('loss', 0.345, step=1)
tf.summary.scalar('loss', 0.234, step=2)
tf.summary.scalar('loss', 0.123, step=3)
```
Multiple independent time series may be logged by giving each series a unique
`name` value.
See [Get started with
TensorBoard](https://www.tensorflow.org/tensorboard/get_started)
for more examples of effective usage of `tf.summary.scalar`.
In general, this API expects that data points are logged with a monotonically
increasing step value. Duplicate points for a single step or points logged out
of order by step are not guaranteed to display as desired in TensorBoard.
Arguments:
name: A name for this summary. The summary tag used for TensorBoard will be
this name prefixed by any active name scopes.
data: A real numeric scalar value, convertible to a `float32` Tensor.
step: Explicit `int64`-castable monotonic step value for this summary. If
omitted, this defaults to `tf.summary.experimental.get_step()`, which must
not be None.
description: Optional long-form description for this summary, as a constant
`str`. Markdown is supported. Defaults to empty.
Returns:
True on success, or false if no summary was written because no default
summary writer was available.
Raises:
ValueError: if a default writer exists, but no step was provided and
`tf.summary.experimental.get_step()` is None.
"""
try:
from tensorboard.summary.v2 import scalar as scalar_v2 # pylint: disable=g-import-not-at-top, g-importing-member
except ImportError as exc:
raise TBNotInstalledError("tf.summary.scalar") from exc
return scalar_v2(name=name, data=data, step=step, description=description)
@tf_export("summary.text", v1=[])
def text(name, data, step=None, description=None):
r"""Write a text summary.
See also `tf.summary.scalar`, `tf.summary.SummaryWriter`, `tf.summary.image`.
Writes text Tensor values for later visualization and analysis in TensorBoard.
Writes go to the current default summary writer. Like `tf.summary.scalar`
points, text points are each associated with a `step` and a `name`.
All the points with the same `name` constitute a time series of text values.
For Example:
```python
test_summary_writer = tf.summary.create_file_writer('test/logdir')
with test_summary_writer.as_default():
tf.summary.text('first_text', 'hello world!', step=0)
tf.summary.text('first_text', 'nice to meet you!', step=1)
```
The text summary can also contain Markdown, and TensorBoard will render the
text
as such.
```python
with test_summary_writer.as_default():
text_data = '''
| *hello* | *there* |
|---------|---------|
| this | is |
| a | table |
'''
text_data = '\n'.join(l.strip() for l in text_data.splitlines())
tf.summary.text('markdown_text', text_data, step=0)
```
Since text is Tensor valued, each text point may be a Tensor of string values.
rank-1 and rank-2 Tensors are rendered as tables in TensorBoard. For higher
ranked
Tensors, you'll see just a 2D slice of the data. To avoid this, reshape the
Tensor
to at most rank-2 prior to passing it to this function.
Demo notebook at
["Displaying text data in
TensorBoard"](https://www.tensorflow.org/tensorboard/text_summaries).
Arguments:
name: A name for this summary. The summary tag used for TensorBoard will be
this name prefixed by any active name scopes.
data: A UTF-8 string Tensor value.
step: Explicit `int64`-castable monotonic step value for this summary. If
omitted, this defaults to `tf.summary.experimental.get_step()`, which must
not be None.
description: Optional long-form description for this summary, as a constant
`str`. Markdown is supported. Defaults to empty.
Returns:
True on success, or false if no summary was emitted because no default
summary writer was available.
Raises:
ValueError: if a default writer exists, but no step was provided and
`tf.summary.experimental.get_step()` is None.
"""
try:
from tensorboard.summary.v2 import text as text_v2 # pylint: disable=g-import-not-at-top, g-importing-member
except ImportError as exc:
raise TBNotInstalledError("tf.summary.text") from exc
return text_v2(name=name, data=data, step=step, description=description)
+106
View File
@@ -0,0 +1,106 @@
load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
load("//tensorflow:tensorflow.default.bzl", "tf_py_strict_test")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
py_library(
name = "fake_summary_writer",
testonly = 1,
srcs = ["fake_summary_writer.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
":writer",
":writer_cache",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/framework:test_lib",
],
)
py_library(
name = "writer",
srcs = ["writer.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
":event_file_writer",
":event_file_writer_v2",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/eager:context",
"//tensorflow/python/framework:meta_graph",
"//tensorflow/python/framework:ops",
"//tensorflow/python/platform:gfile",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/summary:plugin_asset",
"//tensorflow/python/util:tf_export",
],
)
py_library(
name = "event_file_writer",
srcs = ["event_file_writer.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
"//tensorflow/python/client:_pywrap_events_writer",
"//tensorflow/python/platform:gfile",
"//tensorflow/python/platform:tf_logging",
"//tensorflow/python/util:compat",
],
)
py_library(
name = "writer_cache",
srcs = ["writer_cache.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
":writer",
"//tensorflow/python/framework:ops",
"//tensorflow/python/util:tf_export",
],
)
py_library(
name = "event_file_writer_v2",
srcs = ["event_file_writer_v2.py"],
strict_deps = True,
visibility = ["//visibility:public"],
deps = [
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:ops",
"//tensorflow/python/ops:array_ops",
"//tensorflow/python/ops:summary_ops_v2",
"//tensorflow/python/platform:gfile",
],
)
tf_py_strict_test(
name = "writer_test",
size = "small",
srcs = [
"writer_test.py",
],
deps = [
":writer",
":writer_cache",
"//tensorflow/core:protos_all_py",
"//tensorflow/python/client:session",
"//tensorflow/python/framework:constant_op",
"//tensorflow/python/framework:dtypes",
"//tensorflow/python/framework:errors",
"//tensorflow/python/framework:meta_graph",
"//tensorflow/python/framework:ops",
"//tensorflow/python/framework:test_lib",
"//tensorflow/python/ops:summary_ops_v2",
"//tensorflow/python/platform:client_testlib",
"//tensorflow/python/platform:gfile",
"//tensorflow/python/summary:plugin_asset",
"//tensorflow/python/summary:summary_iterator",
"//tensorflow/python/util:compat",
],
)
@@ -0,0 +1,299 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Writes events to disk in a logdir."""
import collections
import os.path
import sys
import threading
import time
from tensorflow.python.client import _pywrap_events_writer
from tensorflow.python.platform import gfile
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import compat
class EventFileWriter:
"""Writes `Event` protocol buffers to an event file.
The `EventFileWriter` class creates an event file in the specified directory,
and asynchronously writes Event protocol buffers to the file. The Event file
is encoded using the tfrecord format, which is similar to RecordIO.
This class is not thread-safe.
"""
def __init__(self, logdir, max_queue=10, flush_secs=120,
filename_suffix=None):
"""Creates a `EventFileWriter` and an event file to write to.
On construction the summary writer creates a new event file in `logdir`.
This event file will contain `Event` protocol buffers, which are written to
disk via the add_event method.
The other arguments to the constructor control the asynchronous writes to
the event file:
* `flush_secs`: How often, in seconds, to flush the added summaries
and events to disk.
* `max_queue`: Maximum number of summaries or events pending to be
written to disk before one of the 'add' calls block.
Args:
logdir: A string. Directory where event file will be written.
max_queue: Integer. Size of the queue for pending events and summaries.
flush_secs: Number. How often, in seconds, to flush the
pending events and summaries to disk.
filename_suffix: A string. Every event file's name is suffixed with
`filename_suffix`.
"""
self._logdir = str(logdir)
gfile.MakeDirs(self._logdir)
self._max_queue = max_queue
self._flush_secs = flush_secs
self._flush_complete = threading.Event()
self._flush_sentinel = object()
self._close_sentinel = object()
self._ev_writer = _pywrap_events_writer.EventsWriter(
compat.as_bytes(os.path.join(self._logdir, "events")))
if filename_suffix:
self._ev_writer.InitWithSuffix(compat.as_bytes(filename_suffix))
self._initialize()
self._closed = False
def _initialize(self):
"""Initializes or re-initializes the queue and writer thread.
The EventsWriter itself does not need to be re-initialized explicitly,
because it will auto-initialize itself if used after being closed.
"""
self._event_queue = CloseableQueue(self._max_queue)
self._worker = _EventLoggerThread(self._event_queue, self._ev_writer,
self._flush_secs, self._flush_complete,
self._flush_sentinel,
self._close_sentinel)
self._worker.start()
def get_logdir(self):
"""Returns the directory where event file will be written."""
return self._logdir
def reopen(self):
"""Reopens the EventFileWriter.
Can be called after `close()` to add more events in the same directory.
The events will go into a new events file.
Does nothing if the EventFileWriter was not closed.
"""
if self._closed:
self._initialize()
self._closed = False
def add_event(self, event):
"""Adds an event to the event file.
Args:
event: An `Event` protocol buffer.
"""
if not self._closed:
self._try_put(event)
def _try_put(self, item):
"""Attempts to enqueue an item to the event queue.
If the queue is closed, this will close the EventFileWriter and reraise the
exception that caused the queue closure, if one exists.
Args:
item: the item to enqueue
"""
try:
self._event_queue.put(item)
except QueueClosedError:
self._internal_close()
if self._worker.failure_exc_info:
_, exception, _ = self._worker.failure_exc_info
raise exception from None
def flush(self):
"""Flushes the event file to disk.
Call this method to make sure that all pending events have been written to
disk.
"""
if not self._closed:
# Request a flush operation by enqueuing a sentinel and then waiting for
# the writer thread to mark the flush as complete.
self._flush_complete.clear()
self._try_put(self._flush_sentinel)
self._flush_complete.wait()
if self._worker.failure_exc_info:
self._internal_close()
_, exception, _ = self._worker.failure_exc_info
raise exception
def close(self):
"""Flushes the event file to disk and close the file.
Call this method when you do not need the summary writer anymore.
"""
if not self._closed:
self.flush()
self._try_put(self._close_sentinel)
self._internal_close()
def _internal_close(self):
self._closed = True
self._worker.join()
self._ev_writer.Close()
class _EventLoggerThread(threading.Thread):
"""Thread that logs events."""
def __init__(self, queue, ev_writer, flush_secs, flush_complete,
flush_sentinel, close_sentinel):
"""Creates an _EventLoggerThread.
Args:
queue: A CloseableQueue from which to dequeue events. The queue will be
closed just before the thread exits, whether due to `close_sentinel` or
any exception raised in the writing loop.
ev_writer: An event writer. Used to log brain events for
the visualizer.
flush_secs: How often, in seconds, to flush the
pending file to disk.
flush_complete: A threading.Event that will be set whenever a flush
operation requested via `flush_sentinel` has been completed.
flush_sentinel: A sentinel element in queue that tells this thread to
flush the writer and mark the current flush operation complete.
close_sentinel: A sentinel element in queue that tells this thread to
terminate and close the queue.
"""
threading.Thread.__init__(self, name="EventLoggerThread")
self.daemon = True
self._queue = queue
self._ev_writer = ev_writer
self._flush_secs = flush_secs
# The first event will be flushed immediately.
self._next_event_flush_time = 0
self._flush_complete = flush_complete
self._flush_sentinel = flush_sentinel
self._close_sentinel = close_sentinel
# Populated when writing logic raises an exception and kills the thread.
self.failure_exc_info = ()
def run(self):
try:
while True:
event = self._queue.get()
if event is self._close_sentinel:
return
elif event is self._flush_sentinel:
self._ev_writer.Flush()
self._flush_complete.set()
else:
self._ev_writer.WriteEvent(event)
# Flush the event writer every so often.
now = time.time()
if now > self._next_event_flush_time:
self._ev_writer.Flush()
self._next_event_flush_time = now + self._flush_secs
except Exception as e:
logging.error("EventFileWriter writer thread error: %s", e)
self.failure_exc_info = sys.exc_info()
raise
finally:
# When exiting the thread, always complete any pending flush operation
# (to unblock flush() calls) and close the queue (to unblock add_event()
# calls, including those used by flush() and close()), which ensures that
# code using EventFileWriter doesn't deadlock if this thread dies.
self._flush_complete.set()
self._queue.close()
class CloseableQueue:
"""Stripped-down fork of the standard library Queue that is closeable."""
def __init__(self, maxsize=0):
"""Create a queue object with a given maximum size.
Args:
maxsize: int size of queue. If <= 0, the queue size is infinite.
"""
self._maxsize = maxsize
self._queue = collections.deque()
self._closed = False
# Mutex must be held whenever queue is mutating; shared by conditions.
self._mutex = threading.Lock()
# Notify not_empty whenever an item is added to the queue; a
# thread waiting to get is notified then.
self._not_empty = threading.Condition(self._mutex)
# Notify not_full whenever an item is removed from the queue;
# a thread waiting to put is notified then.
self._not_full = threading.Condition(self._mutex)
def get(self):
"""Remove and return an item from the queue.
If the queue is empty, blocks until an item is available.
Returns:
an item from the queue
"""
with self._not_empty:
while not self._queue:
self._not_empty.wait()
item = self._queue.popleft()
self._not_full.notify()
return item
def put(self, item):
"""Put an item into the queue.
If the queue is closed, fails immediately.
If the queue is full, blocks until space is available or until the queue
is closed by a call to close(), at which point this call fails.
Args:
item: an item to add to the queue
Raises:
QueueClosedError: if insertion failed because the queue is closed
"""
with self._not_full:
if self._closed:
raise QueueClosedError()
if self._maxsize > 0:
while len(self._queue) == self._maxsize:
self._not_full.wait()
if self._closed:
raise QueueClosedError()
self._queue.append(item)
self._not_empty.notify()
def close(self):
"""Closes the queue, causing any pending or future `put()` calls to fail."""
with self._not_full:
self._closed = True
self._not_full.notify_all()
class QueueClosedError(Exception):
"""Raised when CloseableQueue.put() fails because the queue is closed."""
@@ -0,0 +1,136 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Writes events to disk in a logdir."""
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import summary_ops_v2
from tensorflow.python.platform import gfile
class EventFileWriterV2(object):
"""Writes `Event` protocol buffers to an event file via the graph.
The `EventFileWriterV2` class is backed by the summary file writer in the v2
summary API (currently in tf.contrib.summary), so it uses a shared summary
writer resource and graph ops to write events.
As with the original EventFileWriter, this class will asynchronously write
Event protocol buffers to the backing file. The Event file is encoded using
the tfrecord format, which is similar to RecordIO.
"""
def __init__(self, session, logdir, max_queue=10, flush_secs=120,
filename_suffix=''):
"""Creates an `EventFileWriterV2` and an event file to write to.
On construction, this calls `tf.contrib.summary.create_file_writer` within
the graph from `session.graph` to look up a shared summary writer resource
for `logdir` if one exists, and create one if not. Creating the summary
writer resource in turn creates a new event file in `logdir` to be filled
with `Event` protocol buffers passed to `add_event`. Graph ops to control
this writer resource are added to `session.graph` during this init call;
stateful methods on this class will call `session.run()` on these ops.
Note that because the underlying resource is shared, it is possible that
other parts of the code using the same session may interact independently
with the resource, e.g. by flushing or even closing it. It is the caller's
responsibility to avoid any undesirable sharing in this regard.
The remaining arguments to the constructor (`flush_secs`, `max_queue`, and
`filename_suffix`) control the construction of the shared writer resource
if one is created. If an existing resource is reused, these arguments have
no effect. See `tf.contrib.summary.create_file_writer` for details.
Args:
session: A `tf.compat.v1.Session`. Session that will hold shared writer
resource. The writer ops will be added to session.graph during this
init call.
logdir: A string. Directory where event file will be written.
max_queue: Integer. Size of the queue for pending events and summaries.
flush_secs: Number. How often, in seconds, to flush the
pending events and summaries to disk.
filename_suffix: A string. Every event file's name is suffixed with
`filename_suffix`.
"""
self._session = session
self._logdir = logdir
self._closed = False
gfile.MakeDirs(self._logdir)
with self._session.graph.as_default():
with ops.name_scope('filewriter'):
file_writer = summary_ops_v2.create_file_writer(
logdir=self._logdir,
max_queue=max_queue,
flush_millis=flush_secs * 1000,
filename_suffix=filename_suffix)
with summary_ops_v2.always_record_summaries(), file_writer.as_default():
self._event_placeholder = array_ops.placeholder_with_default(
constant_op.constant('unused', dtypes.string),
shape=[])
self._add_event_op = summary_ops_v2.import_event(
self._event_placeholder)
self._init_op = file_writer.init() # pylint: disable=assignment-from-no-return
self._flush_op = file_writer.flush() # pylint: disable=assignment-from-no-return
self._close_op = file_writer.close() # pylint: disable=assignment-from-no-return
self._session.run(self._init_op)
def get_logdir(self):
"""Returns the directory where event file will be written."""
return self._logdir
def reopen(self):
"""Reopens the EventFileWriter.
Can be called after `close()` to add more events in the same directory.
The events will go into a new events file.
Does nothing if the EventFileWriter was not closed.
"""
if self._closed:
self._closed = False
self._session.run(self._init_op)
def add_event(self, event):
"""Adds an event to the event file.
Args:
event: An `Event` protocol buffer.
"""
if not self._closed:
event_pb = event.SerializeToString()
self._session.run(
self._add_event_op, feed_dict={self._event_placeholder: event_pb})
def flush(self):
"""Flushes the event file to disk.
Call this method to make sure that all pending events have been written to
disk.
"""
self._session.run(self._flush_op)
def close(self):
"""Flushes the event file to disk and close the file.
Call this method when you do not need the summary writer anymore.
"""
if not self._closed:
self.flush()
self._session.run(self._close_op)
self._closed = True
@@ -0,0 +1,139 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Fake summary writer for unit tests."""
from tensorflow.core.framework import summary_pb2
from tensorflow.python.framework import test_util
from tensorflow.python.summary.writer import writer
from tensorflow.python.summary.writer import writer_cache
# TODO(ptucker): Replace with mock framework.
class FakeSummaryWriter(object):
"""Fake summary writer."""
_replaced_summary_writer = None
@classmethod
def install(cls):
if cls._replaced_summary_writer:
raise ValueError('FakeSummaryWriter already installed.')
cls._replaced_summary_writer = writer.FileWriter
writer.FileWriter = FakeSummaryWriter
writer_cache.FileWriter = FakeSummaryWriter
@classmethod
def uninstall(cls):
if not cls._replaced_summary_writer:
raise ValueError('FakeSummaryWriter not installed.')
writer.FileWriter = cls._replaced_summary_writer
writer_cache.FileWriter = cls._replaced_summary_writer
cls._replaced_summary_writer = None
def __init__(self, logdir, graph=None):
self._logdir = logdir
self._graph = graph
self._summaries = {}
self._added_graphs = []
self._added_meta_graphs = []
self._added_session_logs = []
self._added_run_metadata = {}
@property
def summaries(self):
return self._summaries
def assert_summaries(self,
test_case,
expected_logdir=None,
expected_graph=None,
expected_summaries=None,
expected_added_graphs=None,
expected_added_meta_graphs=None,
expected_session_logs=None):
"""Assert expected items have been added to summary writer."""
if expected_logdir is not None:
test_case.assertEqual(expected_logdir, self._logdir)
if expected_graph is not None:
test_case.assertTrue(expected_graph is self._graph)
expected_summaries = expected_summaries or {}
for step in expected_summaries:
test_case.assertTrue(
step in self._summaries,
msg='Missing step %s from %s.' % (step, self._summaries.keys()))
actual_simple_values = {}
for step_summary in self._summaries[step]:
for v in step_summary.value:
# Ignore global_step/sec since it's written by Supervisor in a
# separate thread, so it's non-deterministic how many get written.
if 'global_step/sec' != v.tag:
actual_simple_values[v.tag] = v.simple_value
test_case.assertEqual(expected_summaries[step], actual_simple_values)
if expected_added_graphs is not None:
test_case.assertEqual(expected_added_graphs, self._added_graphs)
if expected_added_meta_graphs is not None:
test_case.assertEqual(len(expected_added_meta_graphs),
len(self._added_meta_graphs))
for expected, actual in zip(expected_added_meta_graphs,
self._added_meta_graphs):
test_util.assert_meta_graph_protos_equal(test_case, expected, actual)
if expected_session_logs is not None:
test_case.assertEqual(expected_session_logs, self._added_session_logs)
def add_summary(self, summ, current_global_step):
"""Add summary."""
if isinstance(summ, bytes):
summary_proto = summary_pb2.Summary()
summary_proto.ParseFromString(summ)
summ = summary_proto
if current_global_step in self._summaries:
step_summaries = self._summaries[current_global_step]
else:
step_summaries = []
self._summaries[current_global_step] = step_summaries
step_summaries.append(summ)
# NOTE: Ignore global_step since its value is non-deterministic.
def add_graph(self, graph, global_step=None, graph_def=None):
"""Add graph."""
if (global_step is not None) and (global_step < 0):
raise ValueError('Invalid global_step %s.' % global_step)
if graph_def is not None:
raise ValueError('Unexpected graph_def %s.' % graph_def)
self._added_graphs.append(graph)
def add_meta_graph(self, meta_graph_def, global_step=None):
"""Add metagraph."""
if (global_step is not None) and (global_step < 0):
raise ValueError('Invalid global_step %s.' % global_step)
self._added_meta_graphs.append(meta_graph_def)
# NOTE: Ignore global_step since its value is non-deterministic.
def add_session_log(self, session_log, global_step=None):
# pylint: disable=unused-argument
self._added_session_logs.append(session_log)
def add_run_metadata(self, run_metadata, tag, global_step=None):
if (global_step is not None) and (global_step < 0):
raise ValueError('Invalid global_step %s.' % global_step)
self._added_run_metadata[tag] = run_metadata
def flush(self):
pass
def reopen(self):
pass
def close(self):
pass
+483
View File
@@ -0,0 +1,483 @@
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Provides an API for generating Event protocol buffers."""
import os.path
import time
import warnings
from tensorflow.core.framework import graph_pb2
from tensorflow.core.framework import summary_pb2
from tensorflow.core.protobuf import meta_graph_pb2
from tensorflow.core.util import event_pb2
from tensorflow.python.eager import context
from tensorflow.python.framework import meta_graph
from tensorflow.python.framework import ops
from tensorflow.python.platform import gfile
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.summary import plugin_asset
from tensorflow.python.summary.writer.event_file_writer import EventFileWriter
from tensorflow.python.summary.writer.event_file_writer_v2 import EventFileWriterV2
from tensorflow.python.util.tf_export import tf_export
_PLUGINS_DIR = "plugins"
class SummaryToEventTransformer(object):
"""Abstractly implements the SummaryWriter API.
This API basically implements a number of endpoints (add_summary,
add_session_log, etc). The endpoints all generate an event protobuf, which is
passed to the contained event_writer.
"""
def __init__(self, event_writer, graph=None, graph_def=None):
"""Creates a `SummaryWriter` and an event file.
On construction the summary writer creates a new event file in `logdir`.
This event file will contain `Event` protocol buffers constructed when you
call one of the following functions: `add_summary()`, `add_session_log()`,
`add_event()`, or `add_graph()`.
If you pass a `Graph` to the constructor it is added to
the event file. (This is equivalent to calling `add_graph()` later).
TensorBoard will pick the graph from the file and display it graphically so
you can interactively explore the graph you built. You will usually pass
the graph from the session in which you launched it:
```python
...create a graph...
# Launch the graph in a session.
sess = tf.compat.v1.Session()
# Create a summary writer, add the 'graph' to the event file.
writer = tf.compat.v1.summary.FileWriter(<some-directory>, sess.graph)
```
Args:
event_writer: An EventWriter. Implements add_event and get_logdir.
graph: A `Graph` object, such as `sess.graph`.
graph_def: DEPRECATED: Use the `graph` argument instead.
"""
self.event_writer = event_writer
# For storing used tags for session.run() outputs.
self._session_run_tags = {}
if graph is not None or graph_def is not None:
# Calling it with both graph and graph_def for backward compatibility.
self.add_graph(graph=graph, graph_def=graph_def)
# Also export the meta_graph_def in this case.
# graph may itself be a graph_def due to positional arguments
maybe_graph_as_def = (graph.as_graph_def(add_shapes=True)
if isinstance(graph, ops.Graph) else graph)
self.add_meta_graph(
meta_graph.create_meta_graph_def(graph_def=graph_def or
maybe_graph_as_def))
# This set contains tags of Summary Values that have been encountered
# already. The motivation here is that the SummaryWriter only keeps the
# metadata property (which is a SummaryMetadata proto) of the first Summary
# Value encountered for each tag. The SummaryWriter strips away the
# SummaryMetadata for all subsequent Summary Values with tags seen
# previously. This saves space.
self._seen_summary_tags = set()
def add_summary(self, summary, global_step=None):
"""Adds a `Summary` protocol buffer to the event file.
This method wraps the provided summary in an `Event` protocol buffer
and adds it to the event file.
You can pass the result of evaluating any summary op, using
`tf.Session.run` or
`tf.Tensor.eval`, to this
function. Alternatively, you can pass a `tf.compat.v1.Summary` protocol
buffer that you populate with your own data. The latter is
commonly done to report evaluation results in event files.
Args:
summary: A `Summary` protocol buffer, optionally serialized as a string.
global_step: Number. Optional global step value to record with the
summary.
"""
if isinstance(summary, bytes):
summ = summary_pb2.Summary()
summ.ParseFromString(summary)
summary = summ
# We strip metadata from values with tags that we have seen before in order
# to save space - we just store the metadata on the first value with a
# specific tag.
for value in summary.value:
if not value.metadata:
continue
if value.tag in self._seen_summary_tags:
# This tag has been encountered before. Strip the metadata.
value.ClearField("metadata")
continue
# We encounter a value with a tag we have not encountered previously. And
# it has metadata. Remember to strip metadata from future values with this
# tag string.
self._seen_summary_tags.add(value.tag)
event = event_pb2.Event(summary=summary)
self._add_event(event, global_step)
def add_session_log(self, session_log, global_step=None):
"""Adds a `SessionLog` protocol buffer to the event file.
This method wraps the provided session in an `Event` protocol buffer
and adds it to the event file.
Args:
session_log: A `SessionLog` protocol buffer.
global_step: Number. Optional global step value to record with the
summary.
"""
event = event_pb2.Event(session_log=session_log)
self._add_event(event, global_step)
def _add_graph_def(self, graph_def, global_step=None):
graph_bytes = graph_def.SerializeToString()
event = event_pb2.Event(graph_def=graph_bytes)
self._add_event(event, global_step)
def add_graph(self, graph, global_step=None, graph_def=None):
"""Adds a `Graph` to the event file.
The graph described by the protocol buffer will be displayed by
TensorBoard. Most users pass a graph in the constructor instead.
Args:
graph: A `Graph` object, such as `sess.graph`.
global_step: Number. Optional global step counter to record with the
graph.
graph_def: DEPRECATED. Use the `graph` parameter instead.
Raises:
ValueError: If both graph and graph_def are passed to the method.
"""
if graph is not None and graph_def is not None:
raise ValueError("Please pass only graph, or graph_def (deprecated), "
"but not both.")
if isinstance(graph, ops.Graph) or isinstance(graph_def, ops.Graph):
# The user passed a `Graph`.
# Check if the user passed it via the graph or the graph_def argument and
# correct for that.
if not isinstance(graph, ops.Graph):
logging.warning("When passing a `Graph` object, please use the `graph`"
" named argument instead of `graph_def`.")
graph = graph_def
# Serialize the graph with additional info.
true_graph_def = graph.as_graph_def(add_shapes=True)
self._write_plugin_assets(graph)
elif (isinstance(graph, graph_pb2.GraphDef) or
isinstance(graph_def, graph_pb2.GraphDef)):
# The user passed a `GraphDef`.
logging.warning("Passing a `GraphDef` to the SummaryWriter is deprecated."
" Pass a `Graph` object instead, such as `sess.graph`.")
# Check if the user passed it via the graph or the graph_def argument and
# correct for that.
if isinstance(graph, graph_pb2.GraphDef):
true_graph_def = graph
else:
true_graph_def = graph_def
else:
# The user passed neither `Graph`, nor `GraphDef`.
raise TypeError("The passed graph must be an instance of `Graph` "
"or the deprecated `GraphDef`")
# Finally, add the graph_def to the summary writer.
self._add_graph_def(true_graph_def, global_step)
def _write_plugin_assets(self, graph):
plugin_assets = plugin_asset.get_all_plugin_assets(graph)
logdir = self.event_writer.get_logdir()
for asset_container in plugin_assets:
plugin_name = asset_container.plugin_name
plugin_dir = os.path.join(logdir, _PLUGINS_DIR, plugin_name)
gfile.MakeDirs(plugin_dir)
assets = asset_container.assets()
for (asset_name, content) in assets.items():
asset_path = os.path.join(plugin_dir, asset_name)
with gfile.Open(asset_path, "w") as f:
f.write(content)
def add_meta_graph(self, meta_graph_def, global_step=None):
"""Adds a `MetaGraphDef` to the event file.
The `MetaGraphDef` allows running the given graph via
`saver.import_meta_graph()`.
Args:
meta_graph_def: A `MetaGraphDef` object, often as returned by
`saver.export_meta_graph()`.
global_step: Number. Optional global step counter to record with the
graph.
Raises:
TypeError: If both `meta_graph_def` is not an instance of `MetaGraphDef`.
"""
if not isinstance(meta_graph_def, meta_graph_pb2.MetaGraphDef):
raise TypeError("meta_graph_def must be type MetaGraphDef, saw type: %s" %
type(meta_graph_def))
meta_graph_bytes = meta_graph_def.SerializeToString()
event = event_pb2.Event(meta_graph_def=meta_graph_bytes)
self._add_event(event, global_step)
def add_run_metadata(self, run_metadata, tag, global_step=None):
"""Adds a metadata information for a single session.run() call.
Args:
run_metadata: A `RunMetadata` protobuf object.
tag: The tag name for this metadata.
global_step: Number. Optional global step counter to record with the
StepStats.
Raises:
ValueError: If the provided tag was already used for this type of event.
"""
if tag in self._session_run_tags:
raise ValueError("The provided tag was already used for this event type")
self._session_run_tags[tag] = True
tagged_metadata = event_pb2.TaggedRunMetadata()
tagged_metadata.tag = tag
# Store the `RunMetadata` object as bytes in order to have postponed
# (lazy) deserialization when used later.
tagged_metadata.run_metadata = run_metadata.SerializeToString()
event = event_pb2.Event(tagged_run_metadata=tagged_metadata)
self._add_event(event, global_step)
def _add_event(self, event, step):
event.wall_time = time.time()
if step is not None:
event.step = int(step)
self.event_writer.add_event(event)
@tf_export(v1=["summary.FileWriter"])
class FileWriter(SummaryToEventTransformer):
"""Writes `Summary` protocol buffers to event files.
The `FileWriter` class provides a mechanism to create an event file in a
given directory and add summaries and events to it. The class updates the
file contents asynchronously. This allows a training program to call methods
to add data to the file directly from the training loop, without slowing down
training.
When constructed with a `tf.compat.v1.Session` parameter, a `FileWriter`
instead forms a compatibility layer over new graph-based summaries
to facilitate the use of new summary writing with
pre-existing code that expects a `FileWriter` instance.
This class is not thread-safe.
@compatibility(TF2)
This API is not compatible with eager execution or `tf.function`. To migrate
to TF2, please use `tf.summary.create_file_writer` instead for summary
management. To specify the summary step, you can manage the context with
`tf.summary.SummaryWriter`, which is returned by
`tf.summary.create_file_writer()`. Or, you can also use the `step` argument
of summary functions such as `tf.summary.histogram`.
See the usage example shown below.
For a comprehensive `tf.summary` migration guide, please follow
[Migrating tf.summary usage to
TF 2.0](https://www.tensorflow.org/tensorboard/migrate#in_tf_1x).
#### How to Map Arguments
| TF1 Arg Name | TF2 Arg Name | Note |
| :---------------- | :---------------- | :-------------------------------- |
| `logdir` | `logdir` | - |
| `graph` | Not supported | - |
| `max_queue` | `max_queue` | - |
| `flush_secs` | `flush_millis` | The unit of time is changed |
: : : from seconds to milliseconds. :
| `graph_def` | Not supported | - |
| `filename_suffix` | `filename_suffix` | - |
| `name` | `name` | - |
#### TF1 & TF2 Usage Example
TF1:
```python
dist = tf.compat.v1.placeholder(tf.float32, [100])
tf.compat.v1.summary.histogram(name="distribution", values=dist)
writer = tf.compat.v1.summary.FileWriter("/tmp/tf1_summary_example")
summaries = tf.compat.v1.summary.merge_all()
sess = tf.compat.v1.Session()
for step in range(100):
mean_moving_normal = np.random.normal(loc=step, scale=1, size=[100])
summ = sess.run(summaries, feed_dict={dist: mean_moving_normal})
writer.add_summary(summ, global_step=step)
```
TF2:
```python
writer = tf.summary.create_file_writer("/tmp/tf2_summary_example")
for step in range(100):
mean_moving_normal = np.random.normal(loc=step, scale=1, size=[100])
with writer.as_default(step=step):
tf.summary.histogram(name='distribution', data=mean_moving_normal)
```
@end_compatibility
"""
def __init__(self,
logdir,
graph=None,
max_queue=10,
flush_secs=120,
graph_def=None,
filename_suffix=None,
session=None):
"""Creates a `FileWriter`, optionally shared within the given session.
Typically, constructing a file writer creates a new event file in `logdir`.
This event file will contain `Event` protocol buffers constructed when you
call one of the following functions: `add_summary()`, `add_session_log()`,
`add_event()`, or `add_graph()`.
If you pass a `Graph` to the constructor it is added to
the event file. (This is equivalent to calling `add_graph()` later).
TensorBoard will pick the graph from the file and display it graphically so
you can interactively explore the graph you built. You will usually pass
the graph from the session in which you launched it:
```python
...create a graph...
# Launch the graph in a session.
sess = tf.compat.v1.Session()
# Create a summary writer, add the 'graph' to the event file.
writer = tf.compat.v1.summary.FileWriter(<some-directory>, sess.graph)
```
The `session` argument to the constructor makes the returned `FileWriter` a
compatibility layer over new graph-based summaries (`tf.summary`).
Crucially, this means the underlying writer resource and events file will
be shared with any other `FileWriter` using the same `session` and `logdir`.
In either case, ops will be added to `session.graph` to control the
underlying file writer resource.
Args:
logdir: A string. Directory where event file will be written.
graph: A `Graph` object, such as `sess.graph`.
max_queue: Integer. Size of the queue for pending events and summaries.
flush_secs: Number. How often, in seconds, to flush the
pending events and summaries to disk.
graph_def: DEPRECATED: Use the `graph` argument instead.
filename_suffix: A string. Every event file's name is suffixed with
`suffix`.
session: A `tf.compat.v1.Session` object. See details above.
Raises:
RuntimeError: If called with eager execution enabled.
@compatibility(eager)
`v1.summary.FileWriter` is not compatible with eager execution.
To write TensorBoard summaries under eager execution,
use `tf.summary.create_file_writer` or
a `with v1.Graph().as_default():` context.
@end_compatibility
"""
if context.executing_eagerly():
raise RuntimeError(
"v1.summary.FileWriter is not compatible with eager execution. "
"Use `tf.summary.create_file_writer`,"
"or a `with v1.Graph().as_default():` context")
if session is not None:
event_writer = EventFileWriterV2(
session, logdir, max_queue, flush_secs, filename_suffix)
else:
event_writer = EventFileWriter(logdir, max_queue, flush_secs,
filename_suffix)
self._closed = False
super(FileWriter, self).__init__(event_writer, graph, graph_def)
def __enter__(self):
"""Make usable with "with" statement."""
return self
def __exit__(self, unused_type, unused_value, unused_traceback):
"""Make usable with "with" statement."""
self.close()
def get_logdir(self):
"""Returns the directory where event file will be written."""
return self.event_writer.get_logdir()
def _warn_if_event_writer_is_closed(self):
if self._closed:
warnings.warn("Attempting to use a closed FileWriter. "
"The operation will be a noop unless the FileWriter "
"is explicitly reopened.")
def _add_event(self, event, step):
self._warn_if_event_writer_is_closed()
super(FileWriter, self)._add_event(event, step)
def add_event(self, event):
"""Adds an event to the event file.
Args:
event: An `Event` protocol buffer.
"""
self._warn_if_event_writer_is_closed()
self.event_writer.add_event(event)
def flush(self):
"""Flushes the event file to disk.
Call this method to make sure that all pending events have been written to
disk.
"""
# Flushing a closed EventFileWriterV2 raises an exception. It is,
# however, a noop for EventFileWriter.
self._warn_if_event_writer_is_closed()
self.event_writer.flush()
def close(self):
"""Flushes the event file to disk and close the file.
Call this method when you do not need the summary writer anymore.
"""
self.event_writer.close()
self._closed = True
def reopen(self):
"""Reopens the EventFileWriter.
Can be called after `close()` to add more events in the same directory.
The events will go into a new events file.
Does nothing if the EventFileWriter was not closed.
"""
self.event_writer.reopen()
self._closed = False
@@ -0,0 +1,60 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A cache for FileWriters."""
import threading
from tensorflow.python.framework import ops
from tensorflow.python.summary.writer.writer import FileWriter
from tensorflow.python.util.tf_export import tf_export
@tf_export(v1=['summary.FileWriterCache'])
class FileWriterCache(object):
"""Cache for file writers.
This class caches file writers, one per directory.
"""
# Cache, keyed by directory.
_cache = {}
# Lock protecting _FILE_WRITERS.
_lock = threading.RLock()
@staticmethod
def clear():
"""Clear cached summary writers. Currently only used for unit tests."""
with FileWriterCache._lock:
# Make sure all the writers are closed now (otherwise open file handles
# may hang around, blocking deletions on Windows).
for item in FileWriterCache._cache.values():
item.close()
FileWriterCache._cache = {}
@staticmethod
def get(logdir):
"""Returns the FileWriter for the specified directory.
Args:
logdir: str, name of the directory.
Returns:
A `FileWriter`.
"""
with FileWriterCache._lock:
if logdir not in FileWriterCache._cache:
FileWriterCache._cache[logdir] = FileWriter(
logdir, graph=ops.get_default_graph())
return FileWriterCache._cache[logdir]
@@ -0,0 +1,748 @@
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for tensorflow.python.summary.writer."""
import glob
import os.path
import shutil
import threading
import time
import warnings
from tensorflow.core.framework import graph_pb2
from tensorflow.core.framework import summary_pb2
from tensorflow.core.protobuf import config_pb2
from tensorflow.core.protobuf import meta_graph_pb2
from tensorflow.core.util import event_pb2
from tensorflow.core.util.event_pb2 import SessionLog
from tensorflow.python.client import session
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import meta_graph
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import summary_ops_v2
from tensorflow.python.platform import gfile
from tensorflow.python.platform import test
from tensorflow.python.summary import plugin_asset
from tensorflow.python.summary import summary_iterator
from tensorflow.python.summary.writer import writer
from tensorflow.python.summary.writer import writer_cache
from tensorflow.python.util import compat
class FileWriterTestBase:
def _FileWriter(self, *args, **kwargs):
return writer.FileWriter(*args, **kwargs)
def _TestDir(self, test_name):
test_dir = os.path.join(self.get_temp_dir(), test_name)
return test_dir
def _CleanTestDir(self, test_name):
test_dir = self._TestDir(test_name)
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
return test_dir
def _EventsReader(self, test_dir):
event_paths = glob.glob(os.path.join(test_dir, "event*"))
# If the tests runs multiple times in the same directory we can have
# more than one matching event file. We only want to read the last one.
self.assertTrue(event_paths)
return summary_iterator.summary_iterator(event_paths[-1])
def assertRecent(self, t):
# We want to ensure the timestamp is something plausible, and aren't able
# to mock out the actual clock used through many layers of the stack, so
# just assert that it's within the past hour, which should always be true.
self.assertLessEqual(t, time.time())
self.assertLess(abs(t - time.time()), 3600)
def assertEventsWithGraph(self, test_dir, g, has_shapes):
meta_graph_def = meta_graph.create_meta_graph_def(
graph_def=g.as_graph_def(add_shapes=has_shapes))
rr = self._EventsReader(test_dir)
# The first event should list the file_version.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual("brain.Event:2", ev.file_version)
# The next event should have the graph.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(0, ev.step)
ev_graph = graph_pb2.GraphDef()
ev_graph.ParseFromString(ev.graph_def)
self.assertProtoEquals(g.as_graph_def(add_shapes=has_shapes), ev_graph)
# The next event should have the metagraph.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(0, ev.step)
ev_meta_graph = meta_graph_pb2.MetaGraphDef()
ev_meta_graph.ParseFromString(ev.meta_graph_def)
self.assertProtoEquals(meta_graph_def, ev_meta_graph)
# We should be done.
self.assertRaises(StopIteration, lambda: next(rr))
@test_util.run_deprecated_v1
def testAddingSummaryGraphAndRunMetadata(self):
test_dir = self._CleanTestDir("basics")
sw = self._FileWriter(test_dir)
sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 1)
sw.add_summary(
summary_pb2.Summary(
value=[summary_pb2.Summary.Value(
tag="mee", simple_value=10.0)]),
10)
sw.add_summary(
summary_pb2.Summary(
value=[summary_pb2.Summary.Value(
tag="boo", simple_value=20.0)]),
20)
with ops.Graph().as_default() as g:
constant_op.constant([0], name="zero")
sw.add_graph(g, global_step=30)
run_metadata = config_pb2.RunMetadata()
device_stats = run_metadata.step_stats.dev_stats.add()
device_stats.device = "test"
sw.add_run_metadata(run_metadata, "test run", global_step=40)
sw.close()
rr = self._EventsReader(test_dir)
# The first event should list the file_version.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual("brain.Event:2", ev.file_version)
# The next event should be the START message.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(1, ev.step)
self.assertEqual(SessionLog.START, ev.session_log.status)
# The next event should have the value 'mee=10.0'.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(10, ev.step)
self.assertProtoEquals("""
value { tag: 'mee' simple_value: 10.0 }
""", ev.summary)
# The next event should have the value 'boo=20.0'.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(20, ev.step)
self.assertProtoEquals("""
value { tag: 'boo' simple_value: 20.0 }
""", ev.summary)
# The next event should have the graph_def.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(30, ev.step)
ev_graph = graph_pb2.GraphDef()
ev_graph.ParseFromString(ev.graph_def)
self.assertProtoEquals(g.as_graph_def(add_shapes=True), ev_graph)
# The next event should have metadata for the run.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(40, ev.step)
self.assertEqual("test run", ev.tagged_run_metadata.tag)
parsed_run_metadata = config_pb2.RunMetadata()
parsed_run_metadata.ParseFromString(ev.tagged_run_metadata.run_metadata)
self.assertProtoEquals(run_metadata, parsed_run_metadata)
# We should be done.
self.assertRaises(StopIteration, lambda: next(rr))
@test_util.run_deprecated_v1
def testGraphAsNamed(self):
test_dir = self._CleanTestDir("basics_named_graph")
with ops.Graph().as_default() as g:
constant_op.constant([12], name="douze")
sw = self._FileWriter(test_dir, graph=g)
sw.close()
self.assertEventsWithGraph(test_dir, g, True)
@test_util.run_deprecated_v1
def testGraphAsPositional(self):
test_dir = self._CleanTestDir("basics_positional_graph")
with ops.Graph().as_default() as g:
constant_op.constant([12], name="douze")
sw = self._FileWriter(test_dir, g)
sw.close()
self.assertEventsWithGraph(test_dir, g, True)
@test_util.run_deprecated_v1
def testGraphDefAsNamed(self):
test_dir = self._CleanTestDir("basics_named_graph_def")
with ops.Graph().as_default() as g:
constant_op.constant([12], name="douze")
gd = g.as_graph_def()
sw = self._FileWriter(test_dir, graph_def=gd)
sw.close()
self.assertEventsWithGraph(test_dir, g, False)
@test_util.run_deprecated_v1
def testGraphDefAsPositional(self):
test_dir = self._CleanTestDir("basics_positional_graph_def")
with ops.Graph().as_default() as g:
constant_op.constant([12], name="douze")
gd = g.as_graph_def()
sw = self._FileWriter(test_dir, gd)
sw.close()
self.assertEventsWithGraph(test_dir, g, False)
@test_util.run_deprecated_v1
def testGraphAndGraphDef(self):
with self.assertRaises(ValueError):
test_dir = self._CleanTestDir("basics_graph_and_graph_def")
with ops.Graph().as_default() as g:
constant_op.constant([12], name="douze")
gd = g.as_graph_def()
sw = self._FileWriter(test_dir, graph=g, graph_def=gd)
sw.close()
@test_util.run_deprecated_v1
def testNeitherGraphNorGraphDef(self):
with self.assertRaises(TypeError):
test_dir = self._CleanTestDir("basics_string_instead_of_graph")
sw = self._FileWriter(test_dir, "string instead of graph object")
sw.close()
@test_util.run_deprecated_v1
def testCloseAndReopen(self):
test_dir = self._CleanTestDir("close_and_reopen")
sw = self._FileWriter(test_dir)
sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 1)
sw.close()
# Sleep at least one second to make sure we get a new event file name.
time.sleep(1.2)
sw.reopen()
sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 2)
sw.close()
# We should now have 2 events files.
event_paths = sorted(glob.glob(os.path.join(test_dir, "event*")))
self.assertEqual(2, len(event_paths))
# Check the first file contents.
rr = summary_iterator.summary_iterator(event_paths[0])
# The first event should list the file_version.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual("brain.Event:2", ev.file_version)
# The next event should be the START message.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(1, ev.step)
self.assertEqual(SessionLog.START, ev.session_log.status)
# We should be done.
self.assertRaises(StopIteration, lambda: next(rr))
# Check the second file contents.
rr = summary_iterator.summary_iterator(event_paths[1])
# The first event should list the file_version.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual("brain.Event:2", ev.file_version)
# The next event should be the START message.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(2, ev.step)
self.assertEqual(SessionLog.START, ev.session_log.status)
# We should be done.
self.assertRaises(StopIteration, lambda: next(rr))
@test_util.run_deprecated_v1
def testNonBlockingClose(self):
test_dir = self._CleanTestDir("non_blocking_close")
sw = self._FileWriter(test_dir)
# Sleep 1.2 seconds to make sure event queue is empty.
time.sleep(1.2)
time_before_close = time.time()
sw.close()
self.assertRecent(time_before_close)
@test_util.run_deprecated_v1
def testUseAfterClose(self):
test_dir = self._CleanTestDir("use_after_close")
sw = self._FileWriter(test_dir)
sw.close()
with warnings.catch_warnings(record=True) as triggered:
warnings.simplefilter("always")
self.assertFalse(triggered)
sw.add_summary(summary_pb2.Summary())
sw.add_session_log(event_pb2.SessionLog())
sw.add_graph(ops.Graph())
self.assertEqual(len(triggered), 3)
for w in triggered:
self.assertEqual(w.category, UserWarning)
@test_util.run_deprecated_v1
def testWithStatement(self):
test_dir = self._CleanTestDir("with_statement")
with self._FileWriter(test_dir) as sw:
sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 1)
event_paths = sorted(glob.glob(os.path.join(test_dir, "event*")))
self.assertEqual(1, len(event_paths))
# Checks that values returned from session Run() calls are added correctly to
# summaries. These are numpy types so we need to check they fit in the
# protocol buffers correctly.
@test_util.run_deprecated_v1
def testAddingSummariesFromSessionRunCalls(self):
test_dir = self._CleanTestDir("global_step")
sw = self._FileWriter(test_dir)
with self.cached_session():
i = constant_op.constant(1, dtype=dtypes.int32, shape=[])
l = constant_op.constant(2, dtype=dtypes.int64, shape=[])
# Test the summary can be passed serialized.
summ = summary_pb2.Summary(
value=[summary_pb2.Summary.Value(
tag="i", simple_value=1.0)])
sw.add_summary(summ.SerializeToString(), self.evaluate(i))
sw.add_summary(
summary_pb2.Summary(
value=[summary_pb2.Summary.Value(tag="l", simple_value=2.0)]),
self.evaluate(l))
sw.close()
rr = self._EventsReader(test_dir)
# File_version.
ev = next(rr)
self.assertTrue(ev)
self.assertRecent(ev.wall_time)
self.assertEqual("brain.Event:2", ev.file_version)
# Summary passed serialized.
ev = next(rr)
self.assertTrue(ev)
self.assertRecent(ev.wall_time)
self.assertEqual(1, ev.step)
self.assertProtoEquals("""
value { tag: 'i' simple_value: 1.0 }
""", ev.summary)
# Summary passed as SummaryObject.
ev = next(rr)
self.assertTrue(ev)
self.assertRecent(ev.wall_time)
self.assertEqual(2, ev.step)
self.assertProtoEquals("""
value { tag: 'l' simple_value: 2.0 }
""", ev.summary)
# We should be done.
self.assertRaises(StopIteration, lambda: next(rr))
@test_util.run_deprecated_v1
def testPluginMetadataStrippedFromSubsequentEvents(self):
test_dir = self._CleanTestDir("basics")
sw = self._FileWriter(test_dir)
sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 1)
# We add 2 summaries with the same tags. They both have metadata. The writer
# should strip the metadata from the second one.
value = summary_pb2.Summary.Value(tag="foo", simple_value=10.0)
value.metadata.plugin_data.plugin_name = "bar"
value.metadata.plugin_data.content = compat.as_bytes("... content ...")
sw.add_summary(summary_pb2.Summary(value=[value]), 10)
value = summary_pb2.Summary.Value(tag="foo", simple_value=10.0)
value.metadata.plugin_data.plugin_name = "bar"
value.metadata.plugin_data.content = compat.as_bytes("... content ...")
sw.add_summary(summary_pb2.Summary(value=[value]), 10)
sw.close()
rr = self._EventsReader(test_dir)
# The first event should list the file_version.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual("brain.Event:2", ev.file_version)
# The next event should be the START message.
ev = next(rr)
self.assertRecent(ev.wall_time)
self.assertEqual(1, ev.step)
self.assertEqual(SessionLog.START, ev.session_log.status)
# This is the first event with tag foo. It should contain SummaryMetadata.
ev = next(rr)
self.assertProtoEquals("""
value {
tag: "foo"
simple_value: 10.0
metadata {
plugin_data {
plugin_name: "bar"
content: "... content ..."
}
}
}
""", ev.summary)
# This is the second event with tag foo. It should lack SummaryMetadata
# because the file writer should have stripped it.
ev = next(rr)
self.assertProtoEquals("""
value {
tag: "foo"
simple_value: 10.0
}
""", ev.summary)
# We should be done.
self.assertRaises(StopIteration, lambda: next(rr))
@test_util.run_deprecated_v1
def testFileWriterWithSuffix(self):
test_dir = self._CleanTestDir("test_suffix")
sw = self._FileWriter(test_dir, filename_suffix="_test_suffix")
for _ in range(10):
sw.add_summary(
summary_pb2.Summary(value=[
summary_pb2.Summary.Value(tag="float_ten", simple_value=10.0)
]),
10)
sw.close()
sw.reopen()
sw.close()
event_filenames = glob.glob(os.path.join(test_dir, "event*"))
for filename in event_filenames:
self.assertTrue(filename.endswith("_test_suffix"))
def testPluginAssetSerialized(self):
class ExamplePluginAsset(plugin_asset.PluginAsset):
plugin_name = "example"
def assets(self):
return {"foo.txt": "foo!", "bar.txt": "bar!"}
with ops.Graph().as_default() as g:
plugin_asset.get_plugin_asset(ExamplePluginAsset)
logdir = self.get_temp_dir()
fw = self._FileWriter(logdir)
fw.add_graph(g)
plugin_dir = os.path.join(logdir, writer._PLUGINS_DIR, "example")
with gfile.Open(os.path.join(plugin_dir, "foo.txt"), "r") as f:
content = f.read()
self.assertEqual(content, "foo!")
with gfile.Open(os.path.join(plugin_dir, "bar.txt"), "r") as f:
content = f.read()
self.assertEqual(content, "bar!")
class FakeWriteError(Exception):
pass
class FileWriterTestCase(FileWriterTestBase, test.TestCase):
@test_util.run_deprecated_v1
def testWriterException_raisedFromFlush(self):
test_dir = self.get_temp_dir()
sw = self._FileWriter(test_dir)
writer_thread = sw.event_writer._worker
with test.mock.patch.object(
writer_thread, "_ev_writer", autospec=True) as mock_writer:
# Coordinate threads to ensure both events are added before the writer
# thread dies, to avoid the second add_event() failing instead of flush().
second_event_added = threading.Event()
def _FakeWriteEvent(event):
del event # unused
second_event_added.wait()
raise FakeWriteError()
mock_writer.WriteEvent.side_effect = _FakeWriteEvent
sw.add_event(event_pb2.Event())
sw.add_event(event_pb2.Event())
second_event_added.set()
with self.assertRaises(FakeWriteError):
sw.flush()
@test_util.run_deprecated_v1
def testWriterException_raisedFromClose(self):
test_dir = self.get_temp_dir()
sw = self._FileWriter(test_dir)
writer_thread = sw.event_writer._worker
with test.mock.patch.object(
writer_thread, "_ev_writer", autospec=True) as mock_writer:
mock_writer.WriteEvent.side_effect = FakeWriteError()
sw.add_event(event_pb2.Event())
with self.assertRaises(FakeWriteError):
sw.close()
@test_util.run_deprecated_v1
def testWriterException_raisedFromAddEvent(self):
test_dir = self.get_temp_dir()
sw = self._FileWriter(test_dir)
writer_thread = sw.event_writer._worker
with test.mock.patch.object(
writer_thread, "_ev_writer", autospec=True) as mock_writer:
mock_writer.WriteEvent.side_effect = FakeWriteError()
sw.add_event(event_pb2.Event())
# Wait for writer thread to exit first, then try to add a new event.
writer_thread.join()
with self.assertRaises(FakeWriteError):
sw.add_event(event_pb2.Event())
@test_util.run_deprecated_v1
def testWriterException_raisedFromPendingAddEvent(self):
test_dir = self.get_temp_dir()
# Set max_queue=1 to allow the third add_event() call to block (first event
# is consumed immediately, the second fills the queue, the third blocks).
sw = self._FileWriter(test_dir, max_queue=1)
writer_thread = sw.event_writer._worker
with test.mock.patch.object(
writer_thread, "_ev_writer", autospec=True) as mock_writer:
# Coordinate threads to ensure the first two events are added and then
# the writer thread sleeps briefly before exiting, to maximize the chance
# that the third add_event() reaches the pending blocked state before the
# queue closes on writer thread exit, since that's what we want to test.
second_event_added = threading.Event()
def _FakeWriteEvent(event):
del event # unused
second_event_added.wait()
time.sleep(0.1)
raise FakeWriteError()
mock_writer.WriteEvent.side_effect = _FakeWriteEvent
sw.add_event(event_pb2.Event())
sw.add_event(event_pb2.Event())
second_event_added.set()
with self.assertRaises(FakeWriteError):
sw.add_event(event_pb2.Event())
class SessionBasedFileWriterTestCase(FileWriterTestBase, test.TestCase):
"""Tests for FileWriter behavior when passed a Session argument."""
def _FileWriter(self, *args, **kwargs):
if "session" not in kwargs:
# Pass in test_session() as the session. It will be cached during this
# test method invocation so that any other use of test_session() with no
# graph should result in re-using the same underlying Session.
with self.cached_session() as sess:
kwargs["session"] = sess
return writer.FileWriter(*args, **kwargs)
return writer.FileWriter(*args, **kwargs)
def _createTaggedSummary(self, tag):
summary = summary_pb2.Summary()
summary.value.add(tag=tag)
return summary
def testSharing_withOtherSessionBasedFileWriters(self):
logdir = self.get_temp_dir()
with session.Session() as sess:
# Initial file writer
writer1 = writer.FileWriter(session=sess, logdir=logdir)
writer1.add_summary(self._createTaggedSummary("one"), 1)
writer1.flush()
# File writer, should share file with writer1
writer2 = writer.FileWriter(session=sess, logdir=logdir)
writer2.add_summary(self._createTaggedSummary("two"), 2)
writer2.flush()
# File writer with different logdir (shouldn't be in this logdir at all)
writer3 = writer.FileWriter(session=sess, logdir=logdir + "-other")
writer3.add_summary(self._createTaggedSummary("three"), 3)
writer3.flush()
# File writer in a different session (should be in separate file)
time.sleep(1.1) # Ensure filename has a different timestamp
with session.Session() as other_sess:
writer4 = writer.FileWriter(session=other_sess, logdir=logdir)
writer4.add_summary(self._createTaggedSummary("four"), 4)
writer4.flush()
# One more file writer, should share file with writer1
writer5 = writer.FileWriter(session=sess, logdir=logdir)
writer5.add_summary(self._createTaggedSummary("five"), 5)
writer5.flush()
event_paths = iter(sorted(glob.glob(os.path.join(logdir, "event*"))))
# First file should have tags "one", "two", and "five"
events = summary_iterator.summary_iterator(next(event_paths))
self.assertEqual("brain.Event:2", next(events).file_version)
self.assertEqual("one", next(events).summary.value[0].tag)
self.assertEqual("two", next(events).summary.value[0].tag)
self.assertEqual("five", next(events).summary.value[0].tag)
self.assertRaises(StopIteration, lambda: next(events))
# Second file should have just "four"
events = summary_iterator.summary_iterator(next(event_paths))
self.assertEqual("brain.Event:2", next(events).file_version)
self.assertEqual("four", next(events).summary.value[0].tag)
self.assertRaises(StopIteration, lambda: next(events))
# No more files
self.assertRaises(StopIteration, lambda: next(event_paths))
# Just check that the other logdir file exists to be sure we wrote it
self.assertTrue(glob.glob(os.path.join(logdir + "-other", "event*")))
def testSharing_withExplicitSummaryFileWriters(self):
logdir = self.get_temp_dir()
with session.Session() as sess:
# Initial file writer via FileWriter(session=?)
writer1 = writer.FileWriter(session=sess, logdir=logdir)
writer1.add_summary(self._createTaggedSummary("one"), 1)
writer1.flush()
# Next one via create_file_writer(), should use same file
writer2 = summary_ops_v2.create_file_writer(logdir=logdir)
with summary_ops_v2.always_record_summaries(), writer2.as_default():
summary2 = summary_ops_v2.scalar("two", 2.0, step=2)
sess.run(writer2.init())
sess.run(summary2)
sess.run(writer2.flush())
# Next has different shared name, should be in separate file
time.sleep(1.1) # Ensure filename has a different timestamp
writer3 = summary_ops_v2.create_file_writer(logdir=logdir, name="other")
with summary_ops_v2.always_record_summaries(), writer3.as_default():
summary3 = summary_ops_v2.scalar("three", 3.0, step=3)
sess.run(writer3.init())
sess.run(summary3)
sess.run(writer3.flush())
# Next uses a second session, should be in separate file
time.sleep(1.1) # Ensure filename has a different timestamp
with session.Session() as other_sess:
writer4 = summary_ops_v2.create_file_writer(logdir=logdir)
with summary_ops_v2.always_record_summaries(), writer4.as_default():
summary4 = summary_ops_v2.scalar("four", 4.0, step=4)
other_sess.run(writer4.init())
other_sess.run(summary4)
other_sess.run(writer4.flush())
# Next via FileWriter(session=?) uses same second session, should be in
# same separate file. (This checks sharing in the other direction)
writer5 = writer.FileWriter(session=other_sess, logdir=logdir)
writer5.add_summary(self._createTaggedSummary("five"), 5)
writer5.flush()
# One more via create_file_writer(), should use same file
writer6 = summary_ops_v2.create_file_writer(logdir=logdir)
with summary_ops_v2.always_record_summaries(), writer6.as_default():
summary6 = summary_ops_v2.scalar("six", 6.0, step=6)
sess.run(writer6.init())
sess.run(summary6)
sess.run(writer6.flush())
event_paths = iter(sorted(glob.glob(os.path.join(logdir, "event*"))))
# First file should have tags "one", "two", and "six"
events = summary_iterator.summary_iterator(next(event_paths))
self.assertEqual("brain.Event:2", next(events).file_version)
self.assertEqual("one", next(events).summary.value[0].tag)
self.assertEqual("two", next(events).summary.value[0].tag)
self.assertEqual("six", next(events).summary.value[0].tag)
self.assertRaises(StopIteration, lambda: next(events))
# Second file should have just "three"
events = summary_iterator.summary_iterator(next(event_paths))
self.assertEqual("brain.Event:2", next(events).file_version)
self.assertEqual("three", next(events).summary.value[0].tag)
self.assertRaises(StopIteration, lambda: next(events))
# Third file should have "four" and "five"
events = summary_iterator.summary_iterator(next(event_paths))
self.assertEqual("brain.Event:2", next(events).file_version)
self.assertEqual("four", next(events).summary.value[0].tag)
self.assertEqual("five", next(events).summary.value[0].tag)
self.assertRaises(StopIteration, lambda: next(events))
# No more files
self.assertRaises(StopIteration, lambda: next(event_paths))
def testSummaryFileWritersInvalidInput(self):
# Test case for GitHub issue 46909
logdir = self.get_temp_dir()
with session.Session() as sess:
with self.assertRaises(errors_impl.InvalidArgumentError):
writer = summary_ops_v2.create_file_writer(
logdir=logdir, flush_millis=[1, 2])
sess.run(writer.init())
sess.run(writer.flush())
class FileWriterCacheTest(test.TestCase):
"""FileWriterCache tests."""
def _test_dir(self, test_name):
"""Create an empty dir to use for tests.
Args:
test_name: Name of the test.
Returns:
Absolute path to the test directory.
"""
test_dir = os.path.join(self.get_temp_dir(), test_name)
if os.path.isdir(test_dir):
for f in glob.glob("%s/*" % test_dir):
os.remove(f)
else:
os.makedirs(test_dir)
return test_dir
def test_cache(self):
with ops.Graph().as_default():
dir1 = self._test_dir("test_cache_1")
dir2 = self._test_dir("test_cache_2")
sw1 = writer_cache.FileWriterCache.get(dir1)
sw2 = writer_cache.FileWriterCache.get(dir2)
sw3 = writer_cache.FileWriterCache.get(dir1)
self.assertEqual(sw1, sw3)
self.assertFalse(sw1 == sw2)
sw1.close()
sw2.close()
events1 = glob.glob(os.path.join(dir1, "event*"))
self.assertTrue(events1)
events2 = glob.glob(os.path.join(dir2, "event*"))
self.assertTrue(events2)
events3 = glob.glob(os.path.join("nowriter", "event*"))
self.assertFalse(events3)
def test_clear(self):
with ops.Graph().as_default():
dir1 = self._test_dir("test_clear")
sw1 = writer_cache.FileWriterCache.get(dir1)
writer_cache.FileWriterCache.clear()
sw2 = writer_cache.FileWriterCache.get(dir1)
self.assertFalse(sw1 == sw2)
if __name__ == "__main__":
test.main()