wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/core/summary/BUILD
+166
View File
@@ -0,0 +1,166 @@
# Description:
# C++ implementation code for the summary writing APIs.
load(
"//tensorflow:tensorflow.bzl",
"tf_cc_binary",
"tf_cc_test",
"tf_copts",
)
load("//tensorflow/core/platform:rules_cc.bzl", "cc_library")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:internal"],
licenses = ["notice"],
)
cc_library(
name = "schema",
srcs = ["schema.cc"],
hdrs = ["schema.h"],
copts = tf_copts(),
deps = [
"//tensorflow/core:lib",
"//tensorflow/core/lib/db:sqlite",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
],
)
tf_cc_test(
name = "schema_test",
size = "small",
srcs = ["schema_test.cc"],
deps = [
":schema",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
],
)
cc_library(
name = "summary_db_writer",
srcs = ["summary_db_writer.cc"],
hdrs = ["summary_db_writer.h"],
copts = tf_copts(),
deps = [
":summary_converter",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/kernels:summary_interface",
"//tensorflow/core/lib/db:sqlite",
"@com_google_absl//absl/log",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:string_view",
"@xla//xla/tsl/protobuf:error_codes_proto_impl_cc",
"@xla//xla/tsl/protobuf:histogram_proto_cc",
],
)
tf_cc_test(
name = "summary_db_writer_test",
size = "small",
srcs = ["summary_db_writer_test.cc"],
deps = [
":schema",
":summary_db_writer",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core/lib/db:sqlite",
"@com_google_absl//absl/log",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@xla//xla/tsl/protobuf:histogram_proto_cc",
],
)
cc_library(
name = "summary_file_writer",
srcs = ["summary_file_writer.cc"],
hdrs = ["summary_file_writer.h"],
copts = tf_copts(),
deps = [
":summary_converter",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/kernels:summary_interface",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@xla//xla/tsl/protobuf:error_codes_proto_impl_cc",
],
)
tf_cc_test(
name = "summary_file_writer_test",
size = "medium", # file i/o
timeout = "short",
srcs = ["summary_file_writer_test.cc"],
deps = [
":summary_file_writer",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "summary_converter",
srcs = ["summary_converter.cc"],
hdrs = ["summary_converter.h"],
copts = tf_copts(),
visibility = ["//visibility:private"],
deps = [
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/lib/png:png_io",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
],
)
tf_cc_binary(
name = "loader",
srcs = ["loader.cc"],
linkstatic = 1,
deps = [
":schema",
":summary_db_writer",
"//tensorflow/core:framework",
"//tensorflow/core:framework_internal",
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/lib/db:sqlite",
"@com_google_absl//absl/log",
"@com_google_absl//absl/status",
"@xla//xla/tsl/protobuf:error_codes_proto_impl_cc",
],
)
tf_cc_binary(
name = "vacuum",
srcs = ["vacuum.cc"],
linkstatic = 1,
deps = [
"//tensorflow/core:framework_internal",
"//tensorflow/core:lib",
"//tensorflow/core/lib/db:sqlite",
"@com_google_absl//absl/log",
],
)
tensorflow/core/summary/loader.cc
+133
View File
@@ -0,0 +1,133 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <cstdint>
#include <iostream>
#include <memory>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
#include "absl/log/log.h"
#include "absl/status/status.h"
#include "xla/tsl/protobuf/error_codes.pb.h"
#include "tensorflow/core/lib/db/sqlite.h"
#include "tensorflow/core/lib/io/record_reader.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/summary/schema.h"
#include "tensorflow/core/summary/summary_db_writer.h"
#include "tensorflow/core/util/command_line_flags.h"
#include "tensorflow/core/util/event.pb.h"
namespace tensorflow {
namespace {
template <typename T>
std::string AddCommas(T n) {
static_assert(std::is_integral<T>::value, "is_integral");
std::string s = strings::StrCat(n);
if (s.size() > 3) {
int extra = s.size() / 3 - (s.size() % 3 == 0 ? 1 : 0);
s.append(extra, 'X');
int c = 0;
for (int i = s.size() - 1; i > 0; --i) {
s[i] = s[i - extra];
if (++c % 3 == 0) {
s[--i] = ',';
--extra;
}
}
}
return s;
}
int main(int argc, char* argv[]) {
std::string path;
std::string events;
std::string experiment_name;
std::string run_name;
std::string user_name;
std::vector<Flag> flag_list = {
Flag("db", &path, "Path of SQLite DB file"),
Flag("events", &events, "TensorFlow record proto event log file"),
Flag("experiment_name", &experiment_name, "The DB experiment_name value"),
Flag("run_name", &run_name, "The DB run_name value"),
Flag("user_name", &user_name, "The DB user_name value"),
};
std::string usage = Flags::Usage(argv[0], flag_list);
bool parse_result = Flags::Parse(&argc, argv, flag_list);
if (!parse_result || path.empty()) {
std::cerr << "The loader tool imports tf.Event record files, created by\n"
<< "SummaryFileWriter, into the sorts of SQLite database files\n"
<< "created by SummaryDbWriter.\n\n"
<< "In addition to the flags below, the environment variables\n"
<< "defined by core/lib/db/sqlite.cc can also be set.\n\n"
<< usage;
return -1;
}
port::InitMain(argv[0], &argc, &argv);
Env* env = Env::Default();
LOG(INFO) << "Opening SQLite file: " << path;
Sqlite* db;
TF_CHECK_OK(Sqlite::Open(
path, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX,
&db));
core::ScopedUnref unref_db(db);
LOG(INFO) << "Initializing TensorBoard schema";
TF_CHECK_OK(SetupTensorboardSqliteDb(db));
LOG(INFO) << "Creating SummaryDbWriter";
SummaryWriterInterface* db_writer;
TF_CHECK_OK(CreateSummaryDbWriter(db, experiment_name, run_name, user_name,
env, &db_writer));
core::ScopedUnref unref(db_writer);
LOG(INFO) << "Loading TF event log: " << events;
std::unique_ptr<RandomAccessFile> file;
TF_CHECK_OK(env->NewRandomAccessFile(events, &file));
io::RecordReader reader(file.get());
uint64_t start = env->NowMicros();
uint64_t records = 0;
uint64_t offset = 0;
tstring record;
while (true) {
std::unique_ptr<Event> event = std::unique_ptr<Event>(new Event);
absl::Status s = reader.ReadRecord(&offset, &record);
if (s.code() == error::OUT_OF_RANGE) break;
TF_CHECK_OK(s);
if (!ParseProtoUnlimited(event.get(), record)) {
LOG(FATAL) << "Corrupt tf.Event record"
<< " offset=" << (offset - record.size())
<< " size=" << static_cast<int>(record.size());
}
TF_CHECK_OK(db_writer->WriteEvent(std::move(event)));
++records;
}
uint64_t elapsed = env->NowMicros() - start;
uint64_t bps =
(elapsed == 0 ? offset
: static_cast<uint64_t>(offset / (elapsed / 1000000.0)));
LOG(INFO) << "Loaded " << AddCommas(offset) << " bytes with "
<< AddCommas(records) << " records at " << AddCommas(bps) << " bps";
return 0;
}
} // namespace
} // namespace tensorflow
int main(int argc, char* argv[]) { return tensorflow::main(argc, argv); }
tensorflow/core/summary/schema.cc
+445
View File
@@ -0,0 +1,445 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/summary/schema.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "tensorflow/core/lib/core/errors.h"
namespace tensorflow {
namespace {
absl::Status Run(Sqlite* db, const char* sql) {
SqliteStatement stmt;
TF_RETURN_IF_ERROR(db->Prepare(sql, &stmt));
return stmt.StepAndReset();
}
} // namespace
absl::Status SetupTensorboardSqliteDb(Sqlite* db) {
// Note: GCC raw strings macros are broken.
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55971
TF_RETURN_IF_ERROR(
db->PrepareOrDie(absl::StrCat("PRAGMA application_id=",
kTensorboardSqliteApplicationId))
.StepAndReset());
db->PrepareOrDie("PRAGMA user_version=0").StepAndResetOrDie();
absl::Status s;
// Ids identify resources.
//
// This table can be used to efficiently generate Permanent IDs in
// conjunction with a random number generator. Unlike rowids these
// IDs safe to use in URLs and unique across tables.
//
// Within any given system, there can't be any foo_id == bar_id for
// all rows of any two (Foos, Bars) tables. A row should only be
// deleted from this table if there's a very high level of confidence
// it exists nowhere else in the system.
//
// Fields:
// id: The system-wide ID. This must be in the range [1,2**47). 0
// is assigned the same meaning as NULL and shouldn't be stored
// and all other int64 values are reserved for future use. Please
// note that id is also the rowid.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS Ids (
id INTEGER PRIMARY KEY
)
)sql"));
// Descriptions are Markdown text that can be associated with any
// resource that has a Permanent ID.
//
// Fields:
// id: The foo_id of the associated row in Foos.
// description: Arbitrary NUL-terminated Markdown text.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS Descriptions (
id INTEGER PRIMARY KEY,
description TEXT
)
)sql"));
// Tensors are 0..n-dimensional numbers or strings.
//
// Fields:
// rowid: Ephemeral b-tree ID.
// series: The Permanent ID of a different resource, e.g. tag_id. A
// tensor will be vacuumed if no series == foo_id exists for all
// rows of all Foos. When series is NULL this tensor may serve
// undefined purposes. This field should be set on placeholders.
// step: Arbitrary number to uniquely order tensors within series.
// The meaning of step is undefined when series is NULL. This may
// be set on placeholders to prepopulate index pages.
// computed_time: Float UNIX timestamp with microsecond precision.
// In the old summaries system that uses FileWriter, this is the
// wall time around when tf.Session.run finished. In the new
// summaries system, it is the wall time of when the tensor was
// computed. On systems with monotonic clocks, it is calculated
// by adding the monotonic run duration to Run.started_time.
// dtype: The tensorflow::DataType ID. For example, DT_INT64 is 9.
// When NULL or 0 this must be treated as a placeholder row that
// does not officially exist.
// shape: A comma-delimited list of int64 >=0 values representing
// length of each dimension in the tensor. This must be a valid
// shape. That means no -1 values and, in the case of numeric
// tensors, length(data) == product(shape) * sizeof(dtype). Empty
// means this is a scalar a.k.a. 0-dimensional tensor.
// data: Little-endian raw tensor memory. If dtype is DT_STRING and
// shape is empty, the nullness of this field indicates whether or
// not it contains the tensor contents; otherwise TensorStrings
// must be queried. If dtype is NULL then ZEROBLOB can be used on
// this field to reserve row space to be updated later.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS Tensors (
rowid INTEGER PRIMARY KEY,
series INTEGER,
step INTEGER,
dtype INTEGER,
computed_time REAL,
shape TEXT,
data BLOB
)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS
TensorSeriesStepIndex
ON
Tensors (series, step)
WHERE
series IS NOT NULL
AND step IS NOT NULL
)sql"));
// TensorStrings are the flat contents of 1..n dimensional DT_STRING
// Tensors.
//
// The number of rows associated with a Tensor must be equal to the
// product of its Tensors.shape.
//
// Fields:
// rowid: Ephemeral b-tree ID.
// tensor_rowid: References Tensors.rowid.
// idx: Index in flattened tensor, starting at 0.
// data: The string value at a particular index. NUL characters are
// permitted.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS TensorStrings (
rowid INTEGER PRIMARY KEY,
tensor_rowid INTEGER NOT NULL,
idx INTEGER NOT NULL,
data BLOB
)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS TensorStringIndex
ON TensorStrings (tensor_rowid, idx)
)sql"));
// Tags are series of Tensors.
//
// Fields:
// rowid: Ephemeral b-tree ID.
// tag_id: The Permanent ID of the Tag.
// run_id: Optional ID of associated Run.
// inserted_time: Float UNIX timestamp with µs precision. This is
// always the wall time of when the row was inserted into the
// DB. It may be used as a hint for an archival job.
// tag_name: The tag field in summary.proto, unique across Run.
// display_name: Optional for GUI and defaults to tag_name.
// plugin_name: Arbitrary TensorBoard plugin name for dispatch.
// plugin_data: Arbitrary data that plugin wants.
//
// TODO(jart): Maybe there should be a Plugins table?
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS Tags (
rowid INTEGER PRIMARY KEY,
run_id INTEGER,
tag_id INTEGER NOT NULL,
inserted_time DOUBLE,
tag_name TEXT,
display_name TEXT,
plugin_name TEXT,
plugin_data BLOB
)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS TagIdIndex
ON Tags (tag_id)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS
TagRunNameIndex
ON
Tags (run_id, tag_name)
WHERE
run_id IS NOT NULL
AND tag_name IS NOT NULL
)sql"));
// Runs are groups of Tags.
//
// Each Run usually represents a single attempt at training or testing
// a TensorFlow model, with a given set of hyper-parameters, whose
// summaries are written out to a single event logs directory with a
// monotonic step counter.
//
// Fields:
// rowid: Ephemeral b-tree ID.
// run_id: The Permanent ID of the Run. This has a 1:1 mapping
// with a SummaryWriter instance. If two writers spawn for a
// given (user_name, run_name, run_name) then each should
// allocate its own run_id and whichever writer puts it in the
// database last wins. The Tags / Tensors associated with the
// previous invocations will then enter limbo, where they may be
// accessible for certain operations, but should be garbage
// collected eventually.
// run_name: User-supplied string, unique across Experiment.
// experiment_id: Optional ID of associated Experiment.
// inserted_time: Float UNIX timestamp with µs precision. This is
// always the time the row was inserted into the database. It
// does not change.
// started_time: Float UNIX timestamp with µs precision. In the
// old summaries system that uses FileWriter, this is
// approximated as the first tf.Event.wall_time. In the new
// summaries system, it is the wall time of when summary writing
// started, from the perspective of whichever machine talks to
// the database. This field will be mutated if the run is
// restarted.
// finished_time: Float UNIX timestamp with µs precision of when
// SummaryWriter resource that created this run was destroyed.
// Once this value becomes non-NULL a Run and its Tags and
// Tensors should be regarded as immutable.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS Runs (
rowid INTEGER PRIMARY KEY,
experiment_id INTEGER,
run_id INTEGER NOT NULL,
inserted_time REAL,
started_time REAL,
finished_time REAL,
run_name TEXT
)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS RunIdIndex
ON Runs (run_id)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS RunNameIndex
ON Runs (experiment_id, run_name)
WHERE run_name IS NOT NULL
)sql"));
// Experiments are groups of Runs.
//
// Fields:
// rowid: Ephemeral b-tree ID.
// user_id: Optional ID of associated User.
// experiment_id: The Permanent ID of the Experiment.
// experiment_name: User-supplied string, unique across User.
// inserted_time: Float UNIX timestamp with µs precision. This is
// always the time the row was inserted into the database. It
// does not change.
// started_time: Float UNIX timestamp with µs precision. This is
// the MIN(experiment.started_time, run.started_time) of each
// Run added to the database, including Runs which have since
// been overwritten.
// is_watching: A boolean indicating if someone is actively
// looking at this Experiment in the TensorBoard GUI. Tensor
// writers that do reservoir sampling can query this value to
// decide if they want the "keep last" behavior. This improves
// the performance of long running training while allowing low
// latency feedback in TensorBoard.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS Experiments (
rowid INTEGER PRIMARY KEY,
user_id INTEGER,
experiment_id INTEGER NOT NULL,
inserted_time REAL,
started_time REAL,
is_watching INTEGER,
experiment_name TEXT
)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS ExperimentIdIndex
ON Experiments (experiment_id)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS ExperimentNameIndex
ON Experiments (user_id, experiment_name)
WHERE experiment_name IS NOT NULL
)sql"));
// Users are people who love TensorBoard.
//
// Fields:
// rowid: Ephemeral b-tree ID.
// user_id: The Permanent ID of the User.
// user_name: Unique user name.
// email: Optional unique email address.
// inserted_time: Float UNIX timestamp with µs precision. This is
// always the time the row was inserted into the database. It
// does not change.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS Users (
rowid INTEGER PRIMARY KEY,
user_id INTEGER NOT NULL,
inserted_time REAL,
user_name TEXT,
email TEXT
)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS UserIdIndex
ON Users (user_id)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS UserNameIndex
ON Users (user_name)
WHERE user_name IS NOT NULL
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS UserEmailIndex
ON Users (email)
WHERE email IS NOT NULL
)sql"));
// Graphs define how Tensors flowed in Runs.
//
// Fields:
// rowid: Ephemeral b-tree ID.
// run_id: The Permanent ID of the associated Run. Only one Graph
// can be associated with a Run.
// graph_id: The Permanent ID of the Graph.
// inserted_time: Float UNIX timestamp with µs precision. This is
// always the wall time of when the row was inserted into the
// DB. It may be used as a hint for an archival job.
// graph_def: Contains the tf.GraphDef proto parts leftover which
// haven't been defined in SQL yet.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS Graphs (
rowid INTEGER PRIMARY KEY,
run_id INTEGER,
graph_id INTEGER NOT NULL,
inserted_time REAL,
graph_def BLOB
)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS GraphIdIndex
ON Graphs (graph_id)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS GraphRunIndex
ON Graphs (run_id)
WHERE run_id IS NOT NULL
)sql"));
// Nodes are the vertices in Graphs.
//
// Fields:
// rowid: Ephemeral b-tree ID.
// graph_id: The Permanent ID of the associated Graph.
// node_id: ID for this node. This is more like a 0-index within
// the Graph. Please note indexes are allowed to be removed.
// node_name: Unique name for this Node within Graph. This is
// copied from the proto so it can be indexed. This is allowed
// to be NULL to save space on the index, in which case the
// node_def.name proto field must not be cleared.
// op: Copied from tf.NodeDef proto.
// device: Copied from tf.NodeDef proto.
// node_def: Contains the tf.NodeDef proto parts leftover which
// haven't been defined in SQL yet.
//
// TODO(jart): Make separate tables for op and device strings.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS Nodes (
rowid INTEGER PRIMARY KEY,
graph_id INTEGER NOT NULL,
node_id INTEGER NOT NULL,
node_name TEXT,
op TEXT,
device TEXT,
node_def BLOB
)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS NodeIdIndex
ON Nodes (graph_id, node_id)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS NodeNameIndex
ON Nodes (graph_id, node_name)
WHERE node_name IS NOT NULL
)sql"));
// NodeInputs are directed edges between Nodes in Graphs.
//
// Fields:
// rowid: Ephemeral b-tree ID.
// graph_id: The Permanent ID of the associated Graph.
// node_id: Index of Node in question. This can be considered the
// 'to' vertex.
// idx: Used for ordering inputs on a given Node.
// input_node_id: Nodes.node_id of the corresponding input node.
// This can be considered the 'from' vertex.
// input_node_idx: Since a Node can output multiple Tensors, this
// is the integer index of which of those outputs is our input.
// NULL is treated as 0.
// is_control: If non-zero, indicates this input is a controlled
// dependency, which means this isn't an edge through which
// tensors flow. NULL means 0.
//
// TODO(jart): Rename to NodeEdges.
s.Update(Run(db, R"sql(
CREATE TABLE IF NOT EXISTS NodeInputs (
rowid INTEGER PRIMARY KEY,
graph_id INTEGER NOT NULL,
node_id INTEGER NOT NULL,
idx INTEGER NOT NULL,
input_node_id INTEGER NOT NULL,
input_node_idx INTEGER,
is_control INTEGER
)
)sql"));
s.Update(Run(db, R"sql(
CREATE UNIQUE INDEX IF NOT EXISTS NodeInputsIndex
ON NodeInputs (graph_id, node_id, idx)
)sql"));
return s;
}
} // namespace tensorflow
tensorflow/core/summary/schema.h
+34
View File
@@ -0,0 +1,34 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_SUMMARY_SCHEMA_H_
#define TENSORFLOW_CORE_SUMMARY_SCHEMA_H_
#include "absl/status/status.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/db/sqlite.h"
namespace tensorflow {
constexpr uint32_t kTensorboardSqliteApplicationId = 0xfeedabee;
/// \brief Creates TensorBoard SQLite tables and indexes.
///
/// If they are already created, this has no effect. If schema
/// migrations are necessary, they will be performed with logging.
absl::Status SetupTensorboardSqliteDb(Sqlite* db);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_SUMMARY_SCHEMA_H_
tensorflow/core/summary/schema_test.cc
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/summary/schema.h"
#include "tensorflow/core/lib/core/status_test_util.h"
#include "tensorflow/core/platform/test.h"
namespace tensorflow {
namespace {
TEST(SchemaTest, SmokeTestTensorboardSchema) {
Sqlite* db;
TF_ASSERT_OK(Sqlite::Open(":memory:", SQLITE_OPEN_READWRITE, &db));
core::ScopedUnref unref_db(db);
TF_ASSERT_OK(SetupTensorboardSqliteDb(db));
}
} // namespace
} // namespace tensorflow
tensorflow/core/summary/summary_converter.cc
+338
View File
@@ -0,0 +1,338 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/summary/summary_converter.h"
#include <algorithm>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <functional>
#include <limits>
#include <string>
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/lib/histogram/histogram.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/png/png_io.h"
#include "tensorflow/core/lib/wav/wav_io.h"
namespace tensorflow {
namespace {
template <typename T>
absl::Status TensorValueAt(Tensor t, int64_t i, T* out) {
#define CASE(I) \
case DataTypeToEnum<I>::value: \
*out = static_cast<T>(t.flat<I>()(i)); \
break;
#define COMPLEX_CASE(I) \
case DataTypeToEnum<I>::value: \
*out = static_cast<T>(t.flat<I>()(i).real()); \
break;
// clang-format off
switch (t.dtype()) {
TF_CALL_bool(CASE)
TF_CALL_half(CASE)
TF_CALL_float(CASE)
TF_CALL_double(CASE)
TF_CALL_int8(CASE)
TF_CALL_int16(CASE)
TF_CALL_int32(CASE)
TF_CALL_int64(CASE)
TF_CALL_uint8(CASE)
TF_CALL_uint16(CASE)
TF_CALL_uint32(CASE)
TF_CALL_uint64(CASE)
TF_CALL_complex64(COMPLEX_CASE)
TF_CALL_complex128(COMPLEX_CASE)
default:
return absl::UnimplementedError(absl::StrCat("SummaryFileWriter ", DataTypeString(t.dtype()), " not supported."));
}
// clang-format on
return absl::OkStatus();
#undef CASE
#undef COMPLEX_CASE
}
typedef Eigen::Tensor<uint8_t, 2, Eigen::RowMajor> Uint8Image;
// Add the sequence of images specified by ith_image to the summary.
//
// Factoring this loop out into a helper function lets ith_image behave
// differently in the float and uint8 cases: the float case needs a temporary
// buffer which can be shared across calls to ith_image, but the uint8 case
// does not.
absl::Status AddImages(const std::string& tag, int max_images, int batch_size,
int w, int h, int depth,
const std::function<Uint8Image(int)>& ith_image,
Summary* s) {
const int N = std::min<int>(max_images, batch_size);
for (int i = 0; i < N; ++i) {
Summary::Value* v = s->add_value();
// The tag depends on the number of requested images (not the number
// produced.)
//
// Note that later on avisu uses "/" to figure out a consistent naming
// convention for display, so we append "/image" to guarantee that the
// image(s) won't be displayed in the global scope with no name.
if (max_images > 1) {
v->set_tag(absl::StrCat(tag, "/image/", i));
} else {
v->set_tag(absl::StrCat(tag, "/image"));
}
const auto image = ith_image(i);
Summary::Image* si = v->mutable_image();
si->set_height(h);
si->set_width(w);
si->set_colorspace(depth);
const int channel_bits = 8;
const int compression = -1; // Use zlib default
if (!png::WriteImageToBuffer(image.data(), w, h, w * depth, depth,
channel_bits, compression,
si->mutable_encoded_image_string(), nullptr)) {
return absl::InternalError("PNG encoding failed");
}
}
return absl::OkStatus();
}
template <class T>
void NormalizeFloatImage(int hw, int depth,
typename TTypes<T>::ConstMatrix values,
typename TTypes<uint8_t>::ConstVec bad_color,
Uint8Image* image) {
if (!image->size()) return; // Nothing to do for empty images
// Rescale the image to uint8 range.
//
// We are trying to generate an RGB image from a float/half tensor. We do
// not have any info about the expected range of values in the tensor
// but the generated image needs to have all RGB values within [0, 255].
//
// We use two different algorithms to generate these values. If the
// tensor has only positive values we scale them all by 255/max(values).
// If the tensor has both negative and positive values we scale them by
// the max of their absolute values and center them around 127.
//
// This works for most cases, but does not respect the relative dynamic
// range across different instances of the tensor.
// Compute min and max ignoring nonfinite pixels
float image_min = std::numeric_limits<float>::infinity();
float image_max = -image_min;
for (int i = 0; i < hw; i++) {
bool finite = true;
for (int j = 0; j < depth; j++) {
if (!Eigen::numext::isfinite(values(i, j))) {
finite = false;
break;
}
}
if (finite) {
for (int j = 0; j < depth; j++) {
float value(values(i, j));
image_min = std::min(image_min, value);
image_max = std::max(image_max, value);
}
}
}
// Pick an affine transform into uint8
const float kZeroThreshold = 1e-6;
T scale, offset;
if (image_min < 0) {
const float max_val = std::max(std::abs(image_min), std::abs(image_max));
scale = T(max_val < kZeroThreshold ? 0.0f : 127.0f / max_val);
offset = T(128.0f);
} else {
scale = T(image_max < kZeroThreshold ? 0.0f : 255.0f / image_max);
offset = T(0.0f);
}
// Transform image, turning nonfinite values to bad_color
for (int i = 0; i < hw; i++) {
bool finite = true;
for (int j = 0; j < depth; j++) {
if (!Eigen::numext::isfinite(values(i, j))) {
finite = false;
break;
}
}
if (finite) {
image->chip<0>(i) = (values.template chip<0>(i) * scale + offset)
.template cast<uint8_t>();
} else {
image->chip<0>(i) = bad_color;
}
}
}
template <class T>
absl::Status NormalizeAndAddImages(const Tensor& tensor, int max_images, int h,
int w, int hw, int depth, int batch_size,
const std::string& base_tag,
Tensor bad_color_tensor, Summary* s) {
// For float and half images, nans and infs are replaced with bad_color.
if (bad_color_tensor.dim_size(0) < depth) {
return absl::InvalidArgumentError(
absl::StrCat("expected depth <= bad_color.size, got depth = ", depth,
", bad_color.size = ", bad_color_tensor.dim_size(0)));
}
auto bad_color_full = bad_color_tensor.vec<uint8_t>();
typename TTypes<uint8_t>::ConstVec bad_color(bad_color_full.data(), depth);
// Float images must be scaled and translated.
Uint8Image image(hw, depth);
auto ith_image = [&tensor, &image, bad_color, batch_size, hw, depth](int i) {
auto tensor_eigen = tensor.template shaped<T, 3>({batch_size, hw, depth});
typename TTypes<T>::ConstMatrix values(
&tensor_eigen(i, 0, 0), Eigen::DSizes<Eigen::DenseIndex, 2>(hw, depth));
NormalizeFloatImage<T>(hw, depth, values, bad_color, &image);
return image;
};
return AddImages(base_tag, max_images, batch_size, w, h, depth, ith_image, s);
}
} // namespace
absl::Status AddTensorAsScalarToSummary(const Tensor& t, const std::string& tag,
Summary* s) {
Summary::Value* v = s->add_value();
v->set_tag(tag);
float value;
TF_RETURN_IF_ERROR(TensorValueAt<float>(t, 0, &value));
v->set_simple_value(value);
return absl::OkStatus();
}
absl::Status AddTensorAsHistogramToSummary(const Tensor& t,
const std::string& tag, Summary* s) {
Summary::Value* v = s->add_value();
v->set_tag(tag);
histogram::Histogram histo;
for (int64_t i = 0; i < t.NumElements(); i++) {
double double_val;
TF_RETURN_IF_ERROR(TensorValueAt<double>(t, i, &double_val));
if (Eigen::numext::isnan(double_val)) {
return absl::InvalidArgumentError(
absl::StrCat("Nan in summary histogram for: ", tag));
} else if (Eigen::numext::isinf(double_val)) {
return absl::InvalidArgumentError(
absl::StrCat("Infinity in summary histogram for: ", tag));
}
histo.Add(double_val);
}
histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */);
return absl::OkStatus();
}
absl::Status AddTensorAsImageToSummary(const Tensor& tensor,
const std::string& tag, int max_images,
const Tensor& bad_color, Summary* s) {
if (!(tensor.dims() == 4 &&
(tensor.dim_size(3) == 1 || tensor.dim_size(3) == 3 ||
tensor.dim_size(3) == 4))) {
return absl::InvalidArgumentError(
absl::StrCat("Tensor must be 4-D with last dim 1, 3, or 4, not ",
tensor.shape().DebugString()));
}
if (!(tensor.dim_size(0) < (1LL << 31) && tensor.dim_size(1) < (1LL << 31) &&
tensor.dim_size(2) < (1LL << 31) &&
(tensor.dim_size(1) * tensor.dim_size(2)) < (1LL << 29))) {
return absl::InvalidArgumentError(absl::StrCat(
"Tensor too large for summary ", tensor.shape().DebugString()));
}
// The casts and h * w cannot overflow because of the limits above.
const int batch_size = static_cast<int>(tensor.dim_size(0));
const int h = static_cast<int>(tensor.dim_size(1));
const int w = static_cast<int>(tensor.dim_size(2));
const int hw = h * w; // Compact these two dims for simplicity
const int depth = static_cast<int>(tensor.dim_size(3));
if (tensor.dtype() == DT_UINT8) {
// For uint8 input, no normalization is necessary
auto ith_image = [&tensor, batch_size, hw, depth](int i) {
auto values = tensor.shaped<uint8_t, 3>({batch_size, hw, depth});
return typename TTypes<uint8_t>::ConstMatrix(
&values(i, 0, 0), Eigen::DSizes<Eigen::DenseIndex, 2>(hw, depth));
};
TF_RETURN_IF_ERROR(
AddImages(tag, max_images, batch_size, w, h, depth, ith_image, s));
} else if (tensor.dtype() == DT_HALF) {
TF_RETURN_IF_ERROR(NormalizeAndAddImages<Eigen::half>(
tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, s));
} else if (tensor.dtype() == DT_FLOAT) {
TF_RETURN_IF_ERROR(NormalizeAndAddImages<float>(
tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, s));
} else if (tensor.dtype() == DT_DOUBLE) {
TF_RETURN_IF_ERROR(NormalizeAndAddImages<double>(
tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, s));
} else {
return absl::InvalidArgumentError(absl::StrCat(
"Only DT_INT8, DT_HALF, DT_DOUBLE, and DT_FLOAT images are supported. "
"Got ",
DataTypeString(tensor.dtype())));
}
return absl::OkStatus();
}
absl::Status AddTensorAsAudioToSummary(const Tensor& tensor,
const std::string& tag, int max_outputs,
float sample_rate, Summary* s) {
if (sample_rate <= 0.0f) {
return absl::InvalidArgumentError("sample_rate must be > 0");
}
const int batch_size = tensor.dim_size(0);
const int64_t length_frames = tensor.dim_size(1);
const int64_t num_channels =
tensor.dims() == 2 ? 1 : tensor.dim_size(tensor.dims() - 1);
const int N = std::min<int>(max_outputs, batch_size);
for (int i = 0; i < N; ++i) {
Summary::Value* v = s->add_value();
if (max_outputs > 1) {
v->set_tag(absl::StrCat(tag, "/audio/", i));
} else {
v->set_tag(absl::StrCat(tag, "/audio"));
}
Summary::Audio* sa = v->mutable_audio();
sa->set_sample_rate(sample_rate);
sa->set_num_channels(num_channels);
sa->set_length_frames(length_frames);
sa->set_content_type("audio/wav");
auto values =
tensor.shaped<float, 3>({batch_size, length_frames, num_channels});
auto channels_by_frames = typename TTypes<float>::ConstMatrix(
&values(i, 0, 0),
Eigen::DSizes<Eigen::DenseIndex, 2>(length_frames, num_channels));
size_t sample_rate_truncated = lrintf(sample_rate);
if (sample_rate_truncated == 0) {
sample_rate_truncated = 1;
}
TF_RETURN_IF_ERROR(wav::EncodeAudioAsS16LEWav(
channels_by_frames.data(), sample_rate_truncated, num_channels,
length_frames, sa->mutable_encoded_audio_string()));
}
return absl::OkStatus();
}
} // namespace tensorflow
tensorflow/core/summary/summary_converter.h
+39
View File
@@ -0,0 +1,39 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_SUMMARY_SUMMARY_CONVERTER_H_
#define TENSORFLOW_CORE_SUMMARY_SUMMARY_CONVERTER_H_
#include "absl/status/status.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/lib/core/status.h"
namespace tensorflow {
// TODO(jart): Delete these methods in favor of new Python implementation.
absl::Status AddTensorAsScalarToSummary(const Tensor& t, const std::string& tag,
Summary* s);
absl::Status AddTensorAsHistogramToSummary(const Tensor& t,
const std::string& tag, Summary* s);
absl::Status AddTensorAsImageToSummary(const Tensor& tensor,
const std::string& tag, int max_images,
const Tensor& bad_color, Summary* s);
absl::Status AddTensorAsAudioToSummary(const Tensor& tensor,
const std::string& tag, int max_outputs,
float sample_rate, Summary* s);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_SUMMARY_SUMMARY_CONVERTER_H_
tensorflow/core/summary/summary_db_writer.cc
+1188
View File
File diff suppressed because it is too large Load Diff
tensorflow/core/summary/summary_db_writer.h
+45
View File
@@ -0,0 +1,45 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_SUMMARY_SUMMARY_DB_WRITER_H_
#define TENSORFLOW_CORE_SUMMARY_SUMMARY_DB_WRITER_H_
#include "absl/status/status.h"
#include "tensorflow/core/kernels/summary_interface.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/db/sqlite.h"
#include "tensorflow/core/platform/env.h"
namespace tensorflow {
/// \brief Creates SQLite SummaryWriterInterface.
///
/// This can be used to write tensors from the execution graph directly
/// to a database. The schema must be created beforehand. Entries in
/// Users, Experiments, and Runs tables will be created automatically
/// if they don't already exist.
///
/// Please note that the type signature of this function may change in
/// the future if support for other DBs is added to core.
///
/// The result holds a new reference to db.
absl::Status CreateSummaryDbWriter(Sqlite* db,
const std::string& experiment_name,
const std::string& run_name,
const std::string& user_name, Env* env,
SummaryWriterInterface** result);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_SUMMARY_SUMMARY_DB_WRITER_H_
tensorflow/core/summary/summary_db_writer_test.cc
+400
View File
@@ -0,0 +1,400 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/summary/summary_db_writer.h"
#include <cstdint>
#include <limits>
#include <memory>
#include <string>
#include <utility>
#include "absl/log/log.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "xla/tsl/protobuf/histogram.pb.h"
#include "tensorflow/core/framework/function.pb.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/node_def.pb.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/lib/core/status_test_util.h"
#include "tensorflow/core/lib/db/sqlite.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/summary/schema.h"
#include "tensorflow/core/util/event.pb.h"
namespace tensorflow {
namespace {
Tensor MakeScalarInt64(int64_t x) {
Tensor t(DT_INT64, TensorShape({}));
t.scalar<int64_t>()() = x;
return t;
}
class FakeClockEnv : public EnvWrapper {
public:
FakeClockEnv() : EnvWrapper(Env::Default()), current_millis_(0) {}
void AdvanceByMillis(const uint64_t millis) { current_millis_ += millis; }
uint64_t NowMicros() const override { return current_millis_ * 1000; }
uint64_t NowSeconds() const override { return current_millis_ * 1000; }
private:
uint64_t current_millis_;
};
class SummaryDbWriterTest : public ::testing::Test {
protected:
void SetUp() override {
TF_ASSERT_OK(Sqlite::Open(":memory:", SQLITE_OPEN_READWRITE, &db_));
TF_ASSERT_OK(SetupTensorboardSqliteDb(db_));
}
void TearDown() override {
if (writer_ != nullptr) {
writer_->Unref();
writer_ = nullptr;
}
db_->Unref();
db_ = nullptr;
}
int64_t QueryInt(const std::string& sql) {
SqliteStatement stmt = db_->PrepareOrDie(sql);
bool is_done;
absl::Status s = stmt.Step(&is_done);
if (!s.ok() || is_done) {
LOG(ERROR) << s << " due to " << sql;
return -1;
}
return stmt.ColumnInt(0);
}
double QueryDouble(const std::string& sql) {
SqliteStatement stmt = db_->PrepareOrDie(sql);
bool is_done;
absl::Status s = stmt.Step(&is_done);
if (!s.ok() || is_done) {
LOG(ERROR) << s << " due to " << sql;
return -1;
}
return stmt.ColumnDouble(0);
}
std::string QueryString(const std::string& sql) {
SqliteStatement stmt = db_->PrepareOrDie(sql);
bool is_done;
absl::Status s = stmt.Step(&is_done);
if (!s.ok() || is_done) {
LOG(ERROR) << s << " due to " << sql;
return "MISSINGNO";
}
return stmt.ColumnString(0);
}
FakeClockEnv env_;
Sqlite* db_ = nullptr;
SummaryWriterInterface* writer_ = nullptr;
};
TEST_F(SummaryDbWriterTest, WriteHistogram_VerifyTensorValues) {
TF_ASSERT_OK(CreateSummaryDbWriter(db_, "histtest", "test1", "user1", &env_,
&writer_));
int step = 0;
std::unique_ptr<Event> e{new Event};
e->set_step(step);
e->set_wall_time(123);
Summary::Value* s = e->mutable_summary()->add_value();
s->set_tag("normal/myhisto");
double dummy_value = 10.123;
HistogramProto* proto = s->mutable_histo();
proto->Clear();
proto->set_min(dummy_value);
proto->set_max(dummy_value);
proto->set_num(dummy_value);
proto->set_sum(dummy_value);
proto->set_sum_squares(dummy_value);
int size = 3;
double bucket_limits[] = {-30.5, -10.5, -5.5};
double bucket[] = {-10, 10, 20};
for (int i = 0; i < size; i++) {
proto->add_bucket_limit(bucket_limits[i]);
proto->add_bucket(bucket[i]);
}
TF_ASSERT_OK(writer_->WriteEvent(std::move(e)));
TF_ASSERT_OK(writer_->Flush());
writer_->Unref();
writer_ = nullptr;
// TODO(nickfelt): implement QueryTensor() to encapsulate this
// Verify the data
std::string result = QueryString("SELECT data FROM Tensors");
const double* val = reinterpret_cast<const double*>(result.data());
double histarray[] = {std::numeric_limits<double>::min(),
-30.5,
-10,
-30.5,
-10.5,
10,
-10.5,
-5.5,
20};
int histarray_size = 9;
for (int i = 0; i < histarray_size; i++) {
EXPECT_EQ(histarray[i], val[i]);
}
}
TEST_F(SummaryDbWriterTest, NothingWritten_NoRowsCreated) {
TF_ASSERT_OK(CreateSummaryDbWriter(db_, "mad-science", "train", "jart", &env_,
&writer_));
TF_ASSERT_OK(writer_->Flush());
writer_->Unref();
writer_ = nullptr;
EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Ids"));
EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Users"));
EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Experiments"));
EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Runs"));
EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Tags"));
EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Tensors"));
}
TEST_F(SummaryDbWriterTest, TensorsWritten_RowsGetInitialized) {
SummaryMetadata metadata;
metadata.set_display_name("display_name");
metadata.set_summary_description("description");
metadata.mutable_plugin_data()->set_plugin_name("plugin_name");
metadata.mutable_plugin_data()->set_content("plugin_data");
SummaryMetadata metadata_nope;
metadata_nope.set_display_name("nope");
metadata_nope.set_summary_description("nope");
metadata_nope.mutable_plugin_data()->set_plugin_name("nope");
metadata_nope.mutable_plugin_data()->set_content("nope");
TF_ASSERT_OK(CreateSummaryDbWriter(db_, "mad-science", "train", "jart", &env_,
&writer_));
env_.AdvanceByMillis(23);
TF_ASSERT_OK(writer_->WriteTensor(1, MakeScalarInt64(123LL), "taggy",
metadata.SerializeAsString()));
env_.AdvanceByMillis(23);
TF_ASSERT_OK(writer_->WriteTensor(2, MakeScalarInt64(314LL), "taggy",
metadata_nope.SerializeAsString()));
TF_ASSERT_OK(writer_->Flush());
ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Users"));
ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Experiments"));
ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Runs"));
ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Tags"));
ASSERT_EQ(1000LL, QueryInt("SELECT COUNT(*) FROM Tensors"));
int64_t user_id = QueryInt("SELECT user_id FROM Users");
int64_t experiment_id = QueryInt("SELECT experiment_id FROM Experiments");
int64_t run_id = QueryInt("SELECT run_id FROM Runs");
int64_t tag_id = QueryInt("SELECT tag_id FROM Tags");
EXPECT_LT(0LL, user_id);
EXPECT_LT(0LL, experiment_id);
EXPECT_LT(0LL, run_id);
EXPECT_LT(0LL, tag_id);
EXPECT_EQ("jart", QueryString("SELECT user_name FROM Users"));
EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Users"));
EXPECT_EQ(user_id, QueryInt("SELECT user_id FROM Experiments"));
EXPECT_EQ("mad-science",
QueryString("SELECT experiment_name FROM Experiments"));
EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Experiments"));
EXPECT_EQ(experiment_id, QueryInt("SELECT experiment_id FROM Runs"));
EXPECT_EQ("train", QueryString("SELECT run_name FROM Runs"));
EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Runs"));
EXPECT_EQ(run_id, QueryInt("SELECT run_id FROM Tags"));
EXPECT_EQ("taggy", QueryString("SELECT tag_name FROM Tags"));
EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Tags"));
EXPECT_EQ("display_name", QueryString("SELECT display_name FROM Tags"));
EXPECT_EQ("plugin_name", QueryString("SELECT plugin_name FROM Tags"));
EXPECT_EQ("plugin_data", QueryString("SELECT plugin_data FROM Tags"));
EXPECT_EQ("description", QueryString("SELECT description FROM Descriptions"));
EXPECT_EQ(tag_id, QueryInt("SELECT series FROM Tensors WHERE step = 1"));
EXPECT_EQ(0.023,
QueryDouble("SELECT computed_time FROM Tensors WHERE step = 1"));
EXPECT_EQ(tag_id, QueryInt("SELECT series FROM Tensors WHERE step = 2"));
EXPECT_EQ(0.046,
QueryDouble("SELECT computed_time FROM Tensors WHERE step = 2"));
}
TEST_F(SummaryDbWriterTest, EmptyParentNames_NoParentsCreated) {
TF_ASSERT_OK(CreateSummaryDbWriter(db_, "", "", "", &env_, &writer_));
TF_ASSERT_OK(writer_->WriteTensor(1, MakeScalarInt64(123LL), "taggy", ""));
TF_ASSERT_OK(writer_->Flush());
ASSERT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Users"));
ASSERT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Experiments"));
ASSERT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Runs"));
ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Tags"));
ASSERT_EQ(1000LL, QueryInt("SELECT COUNT(*) FROM Tensors"));
}
TEST_F(SummaryDbWriterTest, WriteEvent_Scalar) {
TF_ASSERT_OK(CreateSummaryDbWriter(db_, "", "", "", &env_, &writer_));
std::unique_ptr<Event> e{new Event};
e->set_step(7);
e->set_wall_time(123.456);
Summary::Value* s = e->mutable_summary()->add_value();
s->set_tag("π");
s->set_simple_value(3.14f);
s = e->mutable_summary()->add_value();
s->set_tag("φ");
s->set_simple_value(1.61f);
TF_ASSERT_OK(writer_->WriteEvent(std::move(e)));
TF_ASSERT_OK(writer_->Flush());
ASSERT_EQ(2LL, QueryInt("SELECT COUNT(*) FROM Tags"));
ASSERT_EQ(2000LL, QueryInt("SELECT COUNT(*) FROM Tensors"));
int64_t tag1_id = QueryInt("SELECT tag_id FROM Tags WHERE tag_name = 'π'");
int64_t tag2_id = QueryInt("SELECT tag_id FROM Tags WHERE tag_name = 'φ'");
EXPECT_GT(tag1_id, 0LL);
EXPECT_GT(tag2_id, 0LL);
EXPECT_EQ(123.456, QueryDouble(absl::StrCat(
"SELECT computed_time FROM Tensors WHERE series = ",
tag1_id, " AND step = 7")));
EXPECT_EQ(123.456, QueryDouble(absl::StrCat(
"SELECT computed_time FROM Tensors WHERE series = ",
tag2_id, " AND step = 7")));
}
TEST_F(SummaryDbWriterTest, WriteGraph) {
TF_ASSERT_OK(CreateSummaryDbWriter(db_, "", "R", "", &env_, &writer_));
env_.AdvanceByMillis(23);
GraphDef graph;
graph.mutable_library()->add_gradient()->set_function_name("funk");
NodeDef* node = graph.add_node();
node->set_name("x");
node->set_op("Placeholder");
node = graph.add_node();
node->set_name("y");
node->set_op("Placeholder");
node = graph.add_node();
node->set_name("z");
node->set_op("Love");
node = graph.add_node();
node->set_name("+");
node->set_op("Add");
node->add_input("x");
node->add_input("y");
node->add_input("^z");
node->set_device("tpu/lol");
std::unique_ptr<Event> e{new Event};
graph.SerializeToString(e->mutable_graph_def());
TF_ASSERT_OK(writer_->WriteEvent(std::move(e)));
TF_ASSERT_OK(writer_->Flush());
ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Runs"));
ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Graphs"));
ASSERT_EQ(4LL, QueryInt("SELECT COUNT(*) FROM Nodes"));
ASSERT_EQ(3LL, QueryInt("SELECT COUNT(*) FROM NodeInputs"));
ASSERT_EQ(QueryInt("SELECT run_id FROM Runs"),
QueryInt("SELECT run_id FROM Graphs"));
int64_t graph_id = QueryInt("SELECT graph_id FROM Graphs");
EXPECT_GT(graph_id, 0LL);
EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Graphs"));
GraphDef graph2;
graph2.ParseFromString(QueryString("SELECT graph_def FROM Graphs"));
EXPECT_EQ(0, graph2.node_size());
EXPECT_EQ("funk", graph2.library().gradient(0).function_name());
EXPECT_EQ("x", QueryString("SELECT node_name FROM Nodes WHERE node_id = 0"));
EXPECT_EQ("y", QueryString("SELECT node_name FROM Nodes WHERE node_id = 1"));
EXPECT_EQ("z", QueryString("SELECT node_name FROM Nodes WHERE node_id = 2"));
EXPECT_EQ("+", QueryString("SELECT node_name FROM Nodes WHERE node_id = 3"));
EXPECT_EQ("Placeholder",
QueryString("SELECT op FROM Nodes WHERE node_id = 0"));
EXPECT_EQ("Placeholder",
QueryString("SELECT op FROM Nodes WHERE node_id = 1"));
EXPECT_EQ("Love", QueryString("SELECT op FROM Nodes WHERE node_id = 2"));
EXPECT_EQ("Add", QueryString("SELECT op FROM Nodes WHERE node_id = 3"));
EXPECT_EQ("", QueryString("SELECT device FROM Nodes WHERE node_id = 0"));
EXPECT_EQ("", QueryString("SELECT device FROM Nodes WHERE node_id = 1"));
EXPECT_EQ("", QueryString("SELECT device FROM Nodes WHERE node_id = 2"));
EXPECT_EQ("tpu/lol",
QueryString("SELECT device FROM Nodes WHERE node_id = 3"));
EXPECT_EQ(graph_id,
QueryInt("SELECT graph_id FROM NodeInputs WHERE idx = 0"));
EXPECT_EQ(graph_id,
QueryInt("SELECT graph_id FROM NodeInputs WHERE idx = 1"));
EXPECT_EQ(graph_id,
QueryInt("SELECT graph_id FROM NodeInputs WHERE idx = 2"));
EXPECT_EQ(3LL, QueryInt("SELECT node_id FROM NodeInputs WHERE idx = 0"));
EXPECT_EQ(3LL, QueryInt("SELECT node_id FROM NodeInputs WHERE idx = 1"));
EXPECT_EQ(3LL, QueryInt("SELECT node_id FROM NodeInputs WHERE idx = 2"));
EXPECT_EQ(0LL,
QueryInt("SELECT input_node_id FROM NodeInputs WHERE idx = 0"));
EXPECT_EQ(1LL,
QueryInt("SELECT input_node_id FROM NodeInputs WHERE idx = 1"));
EXPECT_EQ(2LL,
QueryInt("SELECT input_node_id FROM NodeInputs WHERE idx = 2"));
EXPECT_EQ(0LL, QueryInt("SELECT is_control FROM NodeInputs WHERE idx = 0"));
EXPECT_EQ(0LL, QueryInt("SELECT is_control FROM NodeInputs WHERE idx = 1"));
EXPECT_EQ(1LL, QueryInt("SELECT is_control FROM NodeInputs WHERE idx = 2"));
}
TEST_F(SummaryDbWriterTest, UsesIdsTable) {
SummaryMetadata metadata;
TF_ASSERT_OK(CreateSummaryDbWriter(db_, "mad-science", "train", "jart", &env_,
&writer_));
env_.AdvanceByMillis(23);
TF_ASSERT_OK(writer_->WriteTensor(1, MakeScalarInt64(123LL), "taggy",
metadata.SerializeAsString()));
TF_ASSERT_OK(writer_->Flush());
ASSERT_EQ(4LL, QueryInt("SELECT COUNT(*) FROM Ids"));
EXPECT_EQ(4LL, QueryInt(strings::StrCat(
"SELECT COUNT(*) FROM Ids WHERE id IN (",
QueryInt("SELECT user_id FROM Users"), ", ",
QueryInt("SELECT experiment_id FROM Experiments"), ", ",
QueryInt("SELECT run_id FROM Runs"), ", ",
QueryInt("SELECT tag_id FROM Tags"), ")")));
}
TEST_F(SummaryDbWriterTest, SetsRunFinishedTime) {
SummaryMetadata metadata;
TF_ASSERT_OK(CreateSummaryDbWriter(db_, "mad-science", "train", "jart", &env_,
&writer_));
env_.AdvanceByMillis(23);
TF_ASSERT_OK(writer_->WriteTensor(1, MakeScalarInt64(123LL), "taggy",
metadata.SerializeAsString()));
TF_ASSERT_OK(writer_->Flush());
ASSERT_EQ(0.023, QueryDouble("SELECT started_time FROM Runs"));
ASSERT_EQ(0.0, QueryDouble("SELECT finished_time FROM Runs"));
env_.AdvanceByMillis(23);
writer_->Unref();
writer_ = nullptr;
ASSERT_EQ(0.023, QueryDouble("SELECT started_time FROM Runs"));
ASSERT_EQ(0.046, QueryDouble("SELECT finished_time FROM Runs"));
}
} // namespace
} // namespace tensorflow
tensorflow/core/summary/summary_file_writer.cc
+224
View File
@@ -0,0 +1,224 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/summary/summary_file_writer.h"
#include <atomic>
#include <cstdint>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/status/status.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "xla/tsl/protobuf/error_codes.pb.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/resource_mgr.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/summary/summary_converter.h"
#include "tensorflow/core/util/event.pb.h"
#include "tensorflow/core/util/events_writer.h"
namespace tensorflow {
namespace {
class SummaryFileWriter : public SummaryWriterInterface {
public:
SummaryFileWriter(int max_queue, int flush_millis, Env* env)
: SummaryWriterInterface(),
is_initialized_(false),
max_queue_(max_queue),
flush_millis_(flush_millis),
env_(env) {}
absl::Status Initialize(const std::string& logdir,
const std::string& filename_suffix) {
const absl::Status is_dir = env_->IsDirectory(logdir);
if (!is_dir.ok()) {
if (is_dir.code() != tensorflow::error::NOT_FOUND) {
return is_dir;
}
TF_RETURN_IF_ERROR(env_->RecursivelyCreateDir(logdir));
}
// Embed PID plus a unique counter as the leading portion of the filename
// suffix to help prevent filename collisions between and within processes.
int32_t pid = env_->GetProcessId();
static std::atomic<int64_t> file_id_counter(0);
// Precede filename_suffix with "." if it doesn't already start with one.
std::string sep = absl::StartsWith(filename_suffix, ".") ? "" : ".";
const std::string uniquified_filename_suffix = absl::StrCat(
".", pid, ".", file_id_counter.fetch_add(1), sep, filename_suffix);
mutex_lock ml(mu_);
events_writer_ =
std::make_unique<EventsWriter>(io::JoinPath(logdir, "events"));
TF_RETURN_WITH_CONTEXT_IF_ERROR(
events_writer_->InitWithSuffix(uniquified_filename_suffix),
"Could not initialize events writer.");
last_flush_ = env_->NowMicros();
is_initialized_ = true;
return absl::OkStatus();
}
absl::Status Flush() override {
mutex_lock ml(mu_);
if (!is_initialized_) {
return absl::FailedPreconditionError(
"Class was not properly initialized.");
}
return InternalFlush();
}
~SummaryFileWriter() override {
(void)Flush(); // Ignore errors.
}
absl::Status WriteTensor(int64_t global_step, Tensor t,
const std::string& tag,
const std::string& serialized_metadata) override {
std::unique_ptr<Event> e{new Event};
e->set_step(global_step);
e->set_wall_time(GetWallTime());
Summary::Value* v = e->mutable_summary()->add_value();
if (t.dtype() == DT_STRING) {
// Treat DT_STRING specially, so that tensor_util.MakeNdarray in Python
// can convert the TensorProto to string-type numpy array. MakeNdarray
// does not work with strings encoded by AsProtoTensorContent() in
// tensor_content.
t.AsProtoField(v->mutable_tensor());
} else {
t.AsProtoTensorContent(v->mutable_tensor());
}
v->set_tag(tag);
if (!serialized_metadata.empty()) {
v->mutable_metadata()->ParseFromString(serialized_metadata);
}
return WriteEvent(std::move(e));
}
absl::Status WriteScalar(int64_t global_step, Tensor t,
const std::string& tag) override {
std::unique_ptr<Event> e{new Event};
e->set_step(global_step);
e->set_wall_time(GetWallTime());
TF_RETURN_IF_ERROR(
AddTensorAsScalarToSummary(t, tag, e->mutable_summary()));
return WriteEvent(std::move(e));
}
absl::Status WriteHistogram(int64_t global_step, Tensor t,
const std::string& tag) override {
std::unique_ptr<Event> e{new Event};
e->set_step(global_step);
e->set_wall_time(GetWallTime());
TF_RETURN_IF_ERROR(
AddTensorAsHistogramToSummary(t, tag, e->mutable_summary()));
return WriteEvent(std::move(e));
}
absl::Status WriteImage(int64_t global_step, Tensor t, const std::string& tag,
int max_images, Tensor bad_color) override {
std::unique_ptr<Event> e{new Event};
e->set_step(global_step);
e->set_wall_time(GetWallTime());
TF_RETURN_IF_ERROR(AddTensorAsImageToSummary(t, tag, max_images, bad_color,
e->mutable_summary()));
return WriteEvent(std::move(e));
}
absl::Status WriteAudio(int64_t global_step, Tensor t, const std::string& tag,
int max_outputs, float sample_rate) override {
std::unique_ptr<Event> e{new Event};
e->set_step(global_step);
e->set_wall_time(GetWallTime());
TF_RETURN_IF_ERROR(AddTensorAsAudioToSummary(
t, tag, max_outputs, sample_rate, e->mutable_summary()));
return WriteEvent(std::move(e));
}
absl::Status WriteGraph(int64_t global_step,
std::unique_ptr<GraphDef> graph) override {
std::unique_ptr<Event> e{new Event};
e->set_step(global_step);
e->set_wall_time(GetWallTime());
graph->SerializeToString(e->mutable_graph_def());
return WriteEvent(std::move(e));
}
absl::Status WriteEvent(std::unique_ptr<Event> event) override {
mutex_lock ml(mu_);
queue_.emplace_back(std::move(event));
if (queue_.size() > max_queue_ ||
env_->NowMicros() - last_flush_ > 1000 * flush_millis_) {
return InternalFlush();
}
return absl::OkStatus();
}
std::string DebugString() const override { return "SummaryFileWriter"; }
private:
double GetWallTime() {
return static_cast<double>(env_->NowMicros()) / 1.0e6;
}
absl::Status InternalFlush() TF_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
for (const std::unique_ptr<Event>& e : queue_) {
events_writer_->WriteEvent(*e);
}
queue_.clear();
TF_RETURN_WITH_CONTEXT_IF_ERROR(events_writer_->Flush(),
"Could not flush events file.");
last_flush_ = env_->NowMicros();
return absl::OkStatus();
}
bool is_initialized_;
const int max_queue_;
const int flush_millis_;
uint64_t last_flush_;
Env* env_;
mutex mu_;
std::vector<std::unique_ptr<Event>> queue_ TF_GUARDED_BY(mu_);
// A pointer to allow deferred construction.
std::unique_ptr<EventsWriter> events_writer_ TF_GUARDED_BY(mu_);
std::vector<std::pair<std::string, SummaryMetadata>> registered_summaries_
TF_GUARDED_BY(mu_);
};
} // namespace
absl::Status CreateSummaryFileWriter(int max_queue, int flush_millis,
const std::string& logdir,
const std::string& filename_suffix,
Env* env,
SummaryWriterInterface** result) {
SummaryFileWriter* w = new SummaryFileWriter(max_queue, flush_millis, env);
const absl::Status s = w->Initialize(logdir, filename_suffix);
if (!s.ok()) {
w->Unref();
*result = nullptr;
return s;
}
*result = w;
return absl::OkStatus();
}
} // namespace tensorflow
tensorflow/core/summary/summary_file_writer.h
+44
View File
@@ -0,0 +1,44 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_SUMMARY_SUMMARY_FILE_WRITER_H_
#define TENSORFLOW_CORE_SUMMARY_SUMMARY_FILE_WRITER_H_
#include "absl/status/status.h"
#include "tensorflow/core/kernels/summary_interface.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
/// \brief Creates SummaryWriterInterface which writes to a file.
///
/// The file is an append-only records file of tf.Event protos. That
/// makes this summary writer suitable for file systems like GCS.
///
/// It will enqueue up to max_queue summaries, and flush at least every
/// flush_millis milliseconds. The summaries will be written to the
/// directory specified by logdir and with the filename suffixed by
/// filename_suffix. The caller owns a reference to result if the
/// returned status is ok. The Env object must not be destroyed until
/// after the returned writer.
absl::Status CreateSummaryFileWriter(int max_queue, int flush_millis,
const std::string& logdir,
const std::string& filename_suffix,
Env* env, SummaryWriterInterface** result);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_SUMMARY_SUMMARY_FILE_WRITER_H_
tensorflow/core/summary/summary_file_writer_test.cc
+300
View File
@@ -0,0 +1,300 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/summary/summary_file_writer.h"
#include <algorithm>
#include <cstdint>
#include <functional>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>
#include "absl/log/check.h"
#include "absl/status/status.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/framework/tensor.pb.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/refcount.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/io/record_reader.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/util/event.pb.h"
namespace tensorflow {
namespace {
class FakeClockEnv : public EnvWrapper {
public:
FakeClockEnv() : EnvWrapper(Env::Default()), current_millis_(0) {}
void AdvanceByMillis(const uint64_t millis) { current_millis_ += millis; }
uint64_t NowMicros() const override { return current_millis_ * 1000; }
uint64_t NowSeconds() const override { return current_millis_ * 1000; }
private:
uint64_t current_millis_;
};
class SummaryFileWriterTest : public ::testing::Test {
protected:
absl::Status SummaryTestHelper(
const std::string& test_name,
const std::function<absl::Status(SummaryWriterInterface*)>& writer_fn,
const std::function<void(const Event&)>& test_fn) {
static std::set<std::string>* tests = new std::set<std::string>();
CHECK(tests->insert(test_name).second) << ": " << test_name;
SummaryWriterInterface* writer;
TF_CHECK_OK(CreateSummaryFileWriter(1, 1, testing::TmpDir(), test_name,
&env_, &writer));
core::ScopedUnref deleter(writer);
TF_CHECK_OK(writer_fn(writer));
TF_CHECK_OK(writer->Flush());
std::vector<std::string> files;
TF_CHECK_OK(env_.GetChildren(testing::TmpDir(), &files));
bool found = false;
for (const std::string& f : files) {
if (absl::StrContains(f, test_name)) {
if (found) {
return absl::UnknownError(
absl::StrCat("Found more than one file for ", test_name));
}
found = true;
std::unique_ptr<RandomAccessFile> read_file;
TF_CHECK_OK(env_.NewRandomAccessFile(io::JoinPath(testing::TmpDir(), f),
&read_file));
io::RecordReader reader(read_file.get(), io::RecordReaderOptions());
tstring record;
uint64_t offset = 0;
TF_CHECK_OK(
reader.ReadRecord(&offset,
&record)); // The first event is irrelevant
TF_CHECK_OK(reader.ReadRecord(&offset, &record));
Event e;
e.ParseFromString(record);
test_fn(e);
}
}
if (!found) {
return absl::UnknownError(absl::StrCat("Found no file for ", test_name));
}
return absl::OkStatus();
}
FakeClockEnv env_;
};
TEST_F(SummaryFileWriterTest, WriteTensor) {
TF_CHECK_OK(SummaryTestHelper("tensor_test",
[](SummaryWriterInterface* writer) {
Tensor one(DT_FLOAT, TensorShape({}));
one.scalar<float>()() = 1.0;
TF_RETURN_IF_ERROR(writer->WriteTensor(
2, one, "name",
SummaryMetadata().SerializeAsString()));
TF_RETURN_IF_ERROR(writer->Flush());
return absl::OkStatus();
},
[](const Event& e) {
EXPECT_EQ(e.step(), 2);
CHECK_EQ(e.summary().value_size(), 1);
EXPECT_EQ(e.summary().value(0).tag(), "name");
}));
TF_CHECK_OK(SummaryTestHelper(
"string_tensor_test",
[](SummaryWriterInterface* writer) {
Tensor hello(DT_STRING, TensorShape({}));
hello.scalar<tstring>()() = "hello";
TF_RETURN_IF_ERROR(writer->WriteTensor(
2, hello, "name", SummaryMetadata().SerializeAsString()));
TF_RETURN_IF_ERROR(writer->Flush());
return absl::OkStatus();
},
[](const Event& e) {
EXPECT_EQ(e.step(), 2);
CHECK_EQ(e.summary().value_size(), 1);
EXPECT_EQ(e.summary().value(0).tag(), "name");
EXPECT_EQ(e.summary().value(0).tensor().dtype(), DT_STRING);
EXPECT_EQ(e.summary().value(0).tensor().string_val()[0], "hello");
}));
}
TEST_F(SummaryFileWriterTest, WriteScalar) {
TF_CHECK_OK(SummaryTestHelper(
"scalar_test",
[](SummaryWriterInterface* writer) {
Tensor one(DT_FLOAT, TensorShape({}));
one.scalar<float>()() = 1.0;
TF_RETURN_IF_ERROR(writer->WriteScalar(2, one, "name"));
TF_RETURN_IF_ERROR(writer->Flush());
return absl::OkStatus();
},
[](const Event& e) {
EXPECT_EQ(e.step(), 2);
CHECK_EQ(e.summary().value_size(), 1);
EXPECT_EQ(e.summary().value(0).tag(), "name");
EXPECT_EQ(e.summary().value(0).simple_value(), 1.0);
}));
}
TEST_F(SummaryFileWriterTest, WriteHistogram) {
TF_CHECK_OK(SummaryTestHelper("hist_test",
[](SummaryWriterInterface* writer) {
Tensor one(DT_FLOAT, TensorShape({}));
one.scalar<float>()() = 1.0;
TF_RETURN_IF_ERROR(
writer->WriteHistogram(2, one, "name"));
TF_RETURN_IF_ERROR(writer->Flush());
return absl::OkStatus();
},
[](const Event& e) {
EXPECT_EQ(e.step(), 2);
CHECK_EQ(e.summary().value_size(), 1);
EXPECT_EQ(e.summary().value(0).tag(), "name");
EXPECT_TRUE(e.summary().value(0).has_histo());
}));
}
namespace {
// Create a 1x1 monochrome image consisting of a single pixel oof the given
// type.
template <typename T>
static absl::Status CreateImage(SummaryWriterInterface* writer) {
Tensor bad_color(DT_UINT8, TensorShape({1}));
bad_color.scalar<uint8_t>()() = 0;
Tensor one(DataTypeToEnum<T>::v(), TensorShape({1, 1, 1, 1}));
one.scalar<T>()() = T(1);
TF_RETURN_IF_ERROR(writer->WriteImage(2, one, "name", 1, bad_color));
TF_RETURN_IF_ERROR(writer->Flush());
return absl::OkStatus();
}
// Verify that the event contains an image generated by CreateImage above.
static void CheckImage(const Event& e) {
EXPECT_EQ(e.step(), 2);
CHECK_EQ(e.summary().value_size(), 1);
EXPECT_EQ(e.summary().value(0).tag(), "name/image");
CHECK(e.summary().value(0).has_image());
EXPECT_EQ(e.summary().value(0).image().height(), 1);
EXPECT_EQ(e.summary().value(0).image().width(), 1);
EXPECT_EQ(e.summary().value(0).image().colorspace(), 1);
}
} // namespace
TEST_F(SummaryFileWriterTest, WriteImageUInt8) {
TF_CHECK_OK(
SummaryTestHelper("image_test_uint8", CreateImage<uint8_t>, CheckImage));
}
TEST_F(SummaryFileWriterTest, WriteImageFloat) {
TF_CHECK_OK(
SummaryTestHelper("image_test_float", CreateImage<float>, CheckImage));
}
TEST_F(SummaryFileWriterTest, WriteImageHalf) {
TF_CHECK_OK(SummaryTestHelper("image_test_half", CreateImage<Eigen::half>,
CheckImage));
}
TEST_F(SummaryFileWriterTest, WriteImageDouble) {
TF_CHECK_OK(
SummaryTestHelper("image_test_double", CreateImage<double>, CheckImage));
}
TEST_F(SummaryFileWriterTest, WriteAudio) {
TF_CHECK_OK(SummaryTestHelper(
"audio_test",
[](SummaryWriterInterface* writer) {
Tensor one(DT_FLOAT, TensorShape({1, 1}));
one.scalar<float>()() = 1.0;
TF_RETURN_IF_ERROR(writer->WriteAudio(2, one, "name", 1, 1));
TF_RETURN_IF_ERROR(writer->Flush());
return absl::OkStatus();
},
[](const Event& e) {
EXPECT_EQ(e.step(), 2);
CHECK_EQ(e.summary().value_size(), 1);
EXPECT_EQ(e.summary().value(0).tag(), "name/audio");
CHECK(e.summary().value(0).has_audio());
}));
}
TEST_F(SummaryFileWriterTest, WriteEvent) {
TF_CHECK_OK(
SummaryTestHelper("event_test",
[](SummaryWriterInterface* writer) {
std::unique_ptr<Event> e{new Event};
e->set_step(7);
e->mutable_summary()->add_value()->set_tag("hi");
TF_RETURN_IF_ERROR(writer->WriteEvent(std::move(e)));
TF_RETURN_IF_ERROR(writer->Flush());
return absl::OkStatus();
},
[](const Event& e) {
EXPECT_EQ(e.step(), 7);
CHECK_EQ(e.summary().value_size(), 1);
EXPECT_EQ(e.summary().value(0).tag(), "hi");
}));
}
TEST_F(SummaryFileWriterTest, WallTime) {
env_.AdvanceByMillis(7023);
TF_CHECK_OK(SummaryTestHelper(
"wall_time_test",
[](SummaryWriterInterface* writer) {
Tensor one(DT_FLOAT, TensorShape({}));
one.scalar<float>()() = 1.0;
TF_RETURN_IF_ERROR(writer->WriteScalar(2, one, "name"));
TF_RETURN_IF_ERROR(writer->Flush());
return absl::OkStatus();
},
[](const Event& e) { EXPECT_EQ(e.wall_time(), 7.023); }));
}
TEST_F(SummaryFileWriterTest, AvoidFilenameCollision) {
// Keep unique with all other test names in this file.
std::string test_name = "avoid_filename_collision_test";
int num_files = 10;
for (int i = 0; i < num_files; i++) {
SummaryWriterInterface* writer;
TF_CHECK_OK(CreateSummaryFileWriter(1, 1, testing::TmpDir(), test_name,
&env_, &writer));
core::ScopedUnref deleter(writer);
}
std::vector<std::string> files;
TF_CHECK_OK(env_.GetChildren(testing::TmpDir(), &files));
// Filter `files` down to just those generated in this test.
files.erase(std::remove_if(files.begin(), files.end(),
[test_name](std::string f) {
return !absl::StrContains(f, test_name);
}),
files.end());
EXPECT_EQ(num_files, files.size())
<< "files = [" << absl::StrJoin(files, ", ") << "]";
}
} // namespace
} // namespace tensorflow
tensorflow/core/summary/vacuum.cc
+139
View File
@@ -0,0 +1,139 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <iostream>
#include <string>
#include "absl/log/log.h"
#include "tensorflow/core/lib/db/sqlite.h"
#include "tensorflow/core/platform/init_main.h"
#include "tensorflow/core/util/command_line_flags.h"
namespace tensorflow {
namespace {
void Vacuum(const char* path) {
LOG(INFO) << "Opening SQLite DB: " << path;
Sqlite* db;
TF_CHECK_OK(Sqlite::Open(path, SQLITE_OPEN_READWRITE, &db));
core::ScopedUnref db_unref(db);
// TODO(jart): Maybe defragment rowids on Tensors.
// TODO(jart): Maybe LIMIT deletes and incremental VACUUM.
// clang-format off
LOG(INFO) << "Deleting orphaned Experiments";
db->PrepareOrDie(R"sql(
DELETE FROM
Experiments
WHERE
user_id IS NOT NULL
AND user_id NOT IN (SELECT user_id FROM Users)
)sql").StepAndResetOrDie();
LOG(INFO) << "Deleting orphaned Runs";
db->PrepareOrDie(R"sql(
DELETE FROM
Runs
WHERE
experiment_id IS NOT NULL
AND experiment_id NOT IN (SELECT experiment_id FROM Experiments)
)sql").StepAndResetOrDie();
LOG(INFO) << "Deleting orphaned Tags";
db->PrepareOrDie(R"sql(
DELETE FROM
Tags
WHERE
run_id IS NOT NULL
AND run_id NOT IN (SELECT run_id FROM Runs)
)sql").StepAndResetOrDie();
// TODO(jart): What should we do if plugins define non-tag tensor series?
LOG(INFO) << "Deleting orphaned Tensors";
db->PrepareOrDie(R"sql(
DELETE FROM
Tensors
WHERE
series IS NOT NULL
AND series NOT IN (SELECT tag_id FROM Tags)
)sql").StepAndResetOrDie();
LOG(INFO) << "Deleting orphaned TensorStrings";
db->PrepareOrDie(R"sql(
DELETE FROM
TensorStrings
WHERE
tensor_rowid NOT IN (SELECT rowid FROM Tensors)
)sql").StepAndResetOrDie();
LOG(INFO) << "Deleting orphaned Graphs";
db->PrepareOrDie(R"sql(
DELETE FROM
Graphs
WHERE
run_id IS NOT NULL
AND run_id NOT IN (SELECT run_id FROM Runs)
)sql").StepAndResetOrDie();
LOG(INFO) << "Deleting orphaned Nodes";
db->PrepareOrDie(R"sql(
DELETE FROM
Nodes
WHERE
graph_id NOT IN (SELECT graph_id FROM Graphs)
)sql").StepAndResetOrDie();
LOG(INFO) << "Deleting orphaned NodeInputs";
db->PrepareOrDie(R"sql(
DELETE FROM
NodeInputs
WHERE
graph_id NOT IN (SELECT graph_id FROM Graphs)
)sql").StepAndResetOrDie();
LOG(INFO) << "Running VACUUM";
db->PrepareOrDie("VACUUM").StepAndResetOrDie();
// clang-format on
}
int main(int argc, char* argv[]) {
std::string usage = Flags::Usage(argv[0], {});
bool parse_result = Flags::Parse(&argc, argv, {});
if (!parse_result) {
std::cerr << "The vacuum tool rebuilds SQLite database files created by\n"
<< "SummaryDbWriter, which makes them smaller.\n\n"
<< "This means deleting orphaned rows and rebuilding b-tree\n"
<< "pages so empty space from deleted rows is cleared. Any\n"
<< "superfluous padding of Tensor BLOBs is also removed.\n\n"
<< usage;
return -1;
}
port::InitMain(argv[0], &argc, &argv);
if (argc < 2 || argv[1][0] == '-') {
std::cerr << "Need at least one SQLite DB path.\n";
return -1;
}
for (int i = 1; i < argc; ++i) {
Vacuum(argv[i]);
}
return 0;
}
} // namespace
} // namespace tensorflow
int main(int argc, char* argv[]) { return tensorflow::main(argc, argv); }