chore: import upstream snapshot with attribution

tensorflow/tools/test/BUILD

@@ -0,0 +1,136 @@
+# Description:
+# Tools for testing
+
+load("@xla//third_party/rules_python/python:py_binary.bzl", "py_binary")
+load("@xla//third_party/rules_python/python:py_library.bzl", "py_library")
+load(
+    "//tensorflow/tools/test:performance.bzl",
+    "tf_cc_logged_benchmark",
+    "tf_py_logged_benchmark",
+)
+
+package(
+    # copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
+    default_visibility = ["//tensorflow:internal"],
+    licenses = ["notice"],
+)
+
+exports_files([
+    "run_and_gather_logs_lib.py",
+    "run_and_gather_logs.py",
+])
+
+py_library(
+    name = "system_info_lib",
+    srcs = ["system_info_lib.py"],
+    strict_deps = True,
+    deps = [
+        ":gpu_info_lib",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/client:device_lib",
+        "//tensorflow/python/framework:errors",
+        "//tensorflow/python/platform:gfile",
+        "@six_archive//:six",
+    ],
+)
+
+py_library(
+    name = "gpu_info_lib",
+    srcs = ["gpu_info_lib.py"],
+    strict_deps = True,
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/framework:errors",
+        "//tensorflow/python/platform:gfile",
+        "@six_archive//:six",
+    ],
+)
+
+py_binary(
+    name = "system_info",
+    srcs = ["system_info.py"],
+    strict_deps = True,
+    deps = [
+        ":system_info_lib",
+        "@absl_py//absl:app",
+    ],
+)
+
+py_library(
+    name = "run_and_gather_logs_lib",
+    srcs = [
+        "run_and_gather_logs_lib.py",
+    ],
+    strict_deps = True,
+    deps = [
+        ":gpu_info_lib",
+        ":system_info_lib",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/platform:gfile",
+        "@absl_py//absl/logging",
+        "@six_archive//:six",
+    ],
+)
+
+py_binary(
+    name = "run_and_gather_logs",
+    srcs = ["run_and_gather_logs.py"],
+    strict_deps = True,
+    visibility = ["//visibility:public"],
+    deps = [
+        ":run_and_gather_logs_main_lib",
+        "@six_archive//:six",
+    ],
+)
+
+py_library(
+    name = "run_and_gather_logs_main_lib",
+    srcs = ["run_and_gather_logs.py"],
+    strict_deps = True,
+    visibility = ["//visibility:public"],
+    deps = [
+        ":run_and_gather_logs_lib",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/platform:client_testlib",
+        "//tensorflow/python/platform:gfile",
+        "//tensorflow/python/platform:tf_logging",
+        "@absl_py//absl:app",
+        "@absl_py//absl/flags",
+        "@six_archive//:six",
+    ],
+)
+
+# Unit test that calls run_and_gather_logs on a benchmark, and
+# prints the result.
+#cuda_py_test(
+#    name = "run_and_gather_logs_test",
+#    srcs = ["run_and_gather_logs.py"],
+#    deps = [
+#        ":run_and_gather_logs",
+#    ],
+#    args = [
+#        "--test_name=" + "//tensorflow/core/kernels:cast_op_test",
+#        "--test_args=" + "'--benchmark_filter=BM_cpu_float'",
+#    ],
+#    data = [
+#        "//tensorflow/core/kernels:cast_op_test",
+#    ],
+#    main = "run_and_gather_logs.py",
+#)
+
+tf_cc_logged_benchmark(
+    name = "cast_op_benchmark",
+    target = "//tensorflow/core/kernels:cast_op_test_gpu",
+)
+
+tf_py_logged_benchmark(
+    name = "rnn_op_benchmark",
+    target = "//tensorflow/python/kernel_tests/nn_ops:rnn_test",
+)
+
+tf_py_logged_benchmark(
+    name = "sparse_csr_matrix_ops_benchmark",
+    target = "//tensorflow/python/kernel_tests/linalg/sparse:csr_sparse_matrix_ops_test",
+)

tensorflow/tools/test/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tools for testing."""
+

tensorflow/tools/test/file_name_test.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Test that checks if we have any issues with case insensitive filesystems.
+
+import os
+
+BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))
+ERROR_MESSAGE = """
+Files with same name but different case detected in directory: {}
+"""
+
+
+def main():
+  # Make sure BASE_DIR ends with tensorflow.  If it doesn't, we probably
+  # computed the wrong directory.
+  if os.path.split(BASE_DIR)[-1] != 'tensorflow':
+    raise AssertionError(
+        "BASE_DIR = '%s' doesn't end with tensorflow" % BASE_DIR)
+
+  for dirpath, dirnames, filenames in os.walk(BASE_DIR, followlinks=True):
+    lowercase_directories = [x.lower() for x in dirnames]
+    lowercase_files = [x.lower() for x in filenames]
+
+    lowercase_dir_contents = lowercase_directories + lowercase_files
+    if len(lowercase_dir_contents) != len(set(lowercase_dir_contents)):
+      raise AssertionError(ERROR_MESSAGE.format(dirpath))
+
+
+if __name__ == '__main__':
+  main()

tensorflow/tools/test/gpu_info_lib.py

@@ -0,0 +1,178 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library for getting system information during TensorFlow tests."""
+
+import ctypes as ct
+import platform
+
+
+from tensorflow.core.util import test_log_pb2
+from tensorflow.python.framework import errors
+from tensorflow.python.platform import gfile
+
+
+def _gather_gpu_devices_proc():
+  """Try to gather NVidia GPU device information via /proc/driver."""
+  dev_info = []
+  for f in gfile.Glob("/proc/driver/nvidia/gpus/*/information"):
+    bus_id = f.split("/")[5]
+    key_values = dict(line.rstrip().replace("\t", "").split(":", 1)
+                      for line in gfile.GFile(f, "r"))
+    key_values = dict(
+        (k.lower(), v.strip(" ").rstrip(" ")) for (k, v) in key_values.items())
+    info = test_log_pb2.GPUInfo()
+    info.model = key_values.get("model", "Unknown")
+    info.uuid = key_values.get("gpu uuid", "Unknown")
+    info.bus_id = bus_id
+    dev_info.append(info)
+  return dev_info
+
+
+class CUDADeviceProperties(ct.Structure):
+  # See $CUDA_HOME/include/cuda_runtime_api.h for the definition of
+  # the cudaDeviceProp struct.
+  _fields_ = [
+      ("name", ct.c_char * 256),
+      ("totalGlobalMem", ct.c_size_t),
+      ("sharedMemPerBlock", ct.c_size_t),
+      ("regsPerBlock", ct.c_int),
+      ("warpSize", ct.c_int),
+      ("memPitch", ct.c_size_t),
+      ("maxThreadsPerBlock", ct.c_int),
+      ("maxThreadsDim", ct.c_int * 3),
+      ("maxGridSize", ct.c_int * 3),
+      ("clockRate", ct.c_int),
+      ("totalConstMem", ct.c_size_t),
+      ("major", ct.c_int),
+      ("minor", ct.c_int),
+      ("textureAlignment", ct.c_size_t),
+      ("texturePitchAlignment", ct.c_size_t),
+      ("deviceOverlap", ct.c_int),
+      ("multiProcessorCount", ct.c_int),
+      ("kernelExecTimeoutEnabled", ct.c_int),
+      ("integrated", ct.c_int),
+      ("canMapHostMemory", ct.c_int),
+      ("computeMode", ct.c_int),
+      ("maxTexture1D", ct.c_int),
+      ("maxTexture1DMipmap", ct.c_int),
+      ("maxTexture1DLinear", ct.c_int),
+      ("maxTexture2D", ct.c_int * 2),
+      ("maxTexture2DMipmap", ct.c_int * 2),
+      ("maxTexture2DLinear", ct.c_int * 3),
+      ("maxTexture2DGather", ct.c_int * 2),
+      ("maxTexture3D", ct.c_int * 3),
+      ("maxTexture3DAlt", ct.c_int * 3),
+      ("maxTextureCubemap", ct.c_int),
+      ("maxTexture1DLayered", ct.c_int * 2),
+      ("maxTexture2DLayered", ct.c_int * 3),
+      ("maxTextureCubemapLayered", ct.c_int * 2),
+      ("maxSurface1D", ct.c_int),
+      ("maxSurface2D", ct.c_int * 2),
+      ("maxSurface3D", ct.c_int * 3),
+      ("maxSurface1DLayered", ct.c_int * 2),
+      ("maxSurface2DLayered", ct.c_int * 3),
+      ("maxSurfaceCubemap", ct.c_int),
+      ("maxSurfaceCubemapLayered", ct.c_int * 2),
+      ("surfaceAlignment", ct.c_size_t),
+      ("concurrentKernels", ct.c_int),
+      ("ECCEnabled", ct.c_int),
+      ("pciBusID", ct.c_int),
+      ("pciDeviceID", ct.c_int),
+      ("pciDomainID", ct.c_int),
+      ("tccDriver", ct.c_int),
+      ("asyncEngineCount", ct.c_int),
+      ("unifiedAddressing", ct.c_int),
+      ("memoryClockRate", ct.c_int),
+      ("memoryBusWidth", ct.c_int),
+      ("l2CacheSize", ct.c_int),
+      ("maxThreadsPerMultiProcessor", ct.c_int),
+      ("streamPrioritiesSupported", ct.c_int),
+      ("globalL1CacheSupported", ct.c_int),
+      ("localL1CacheSupported", ct.c_int),
+      ("sharedMemPerMultiprocessor", ct.c_size_t),
+      ("regsPerMultiprocessor", ct.c_int),
+      ("managedMemSupported", ct.c_int),
+      ("isMultiGpuBoard", ct.c_int),
+      ("multiGpuBoardGroupID", ct.c_int),
+      # Pad with extra space to avoid dereference crashes if future
+      # versions of CUDA extend the size of this struct.
+      ("__future_buffer", ct.c_char * 4096)
+  ]
+
+
+def _gather_gpu_devices_cudart():
+  """Try to gather NVidia GPU device information via libcudart."""
+  dev_info = []
+
+  system = platform.system()
+  if system == "Linux":
+    libcudart = ct.cdll.LoadLibrary("libcudart.so")
+  elif system == "Darwin":
+    libcudart = ct.cdll.LoadLibrary("libcudart.dylib")
+  elif system == "Windows":
+    libcudart = ct.windll.LoadLibrary("libcudart.dll")
+  else:
+    raise NotImplementedError("Cannot identify system.")
+
+  version = ct.c_int()
+  rc = libcudart.cudaRuntimeGetVersion(ct.byref(version))
+  if rc != 0:
+    raise ValueError("Could not get version")
+  if version.value < 6050:
+    raise NotImplementedError("CUDA version must be between >= 6.5")
+
+  device_count = ct.c_int()
+  libcudart.cudaGetDeviceCount(ct.byref(device_count))
+
+  for i in range(device_count.value):
+    properties = CUDADeviceProperties()
+    rc = libcudart.cudaGetDeviceProperties(ct.byref(properties), i)
+    if rc != 0:
+      raise ValueError("Could not get device properties")
+    pci_bus_id = " " * 13
+    rc = libcudart.cudaDeviceGetPCIBusId(ct.c_char_p(pci_bus_id), 13, i)
+    if rc != 0:
+      raise ValueError("Could not get device PCI bus id")
+
+    info = test_log_pb2.GPUInfo()  # No UUID available
+    info.model = properties.name
+    info.bus_id = pci_bus_id
+    dev_info.append(info)
+
+    del properties
+
+  return dev_info
+
+
+def gather_gpu_devices():
+  """Gather gpu device info.
+
+  Returns:
+    A list of test_log_pb2.GPUInfo messages.
+  """
+  try:
+    # Prefer using /proc if possible, it provides the UUID.
+    dev_info = _gather_gpu_devices_proc()
+    if not dev_info:
+      raise ValueError("No devices found")
+    return dev_info
+  except (IOError, ValueError, errors.OpError):
+    pass
+
+  try:
+    # Fall back on using libcudart
+    return _gather_gpu_devices_cudart()
+  except (OSError, ValueError, NotImplementedError, errors.OpError):
+    return []

tensorflow/tools/test/performance.bzl

@@ -0,0 +1,105 @@
+"""
+Benchmark-related macros.
+"""
+
+load(
+    "//tensorflow:tensorflow.default.bzl",
+    "cuda_py_strict_test",
+    "tf_py_strict_test",
+)
+
+# Create a benchmark test target of a TensorFlow C++ test (tf_cc_*_test)
+def tf_cc_logged_benchmark(
+        name = None,
+        target = None,
+        benchmarks = "..",
+        tags = [],
+        benchmark_type = "cpp_microbenchmark",
+        **kwargs):
+    if not name:
+        fail("Must provide a name")
+    if not target:
+        fail("Must provide a target")
+    if (not ":" in target or
+        not target.startswith("//") or
+        target.endswith(":all") or
+        target.endswith(".")):
+        fail(" ".join((
+            "Target must be a single well-defined test, e.g.,",
+            "//path/to:test. Received: %s" % target,
+        )))
+
+    all_tags = tags + ["benchmark-test", "local", "manual", "regression-test"]
+
+    tf_py_strict_test(
+        name = name,
+        tags = all_tags,
+        size = "large",
+        srcs = ["//tensorflow/tools/test:run_and_gather_logs"],
+        args = [
+            "--name=//%s:%s" % (native.package_name(), name),
+            "--test_name=" + target,
+            "--test_args=--benchmarks=%s" % benchmarks,
+            "--benchmark_type=%s" % benchmark_type,
+        ],
+        data = [
+            target,
+        ],
+        main = "//tensorflow/tools/test:run_and_gather_logs.py",
+        deps = [
+            "@absl_py//absl:app",
+            "@absl_py//absl/flags",
+            "@org_tensorflow//tensorflow/core:protos_all_py",
+            "//tensorflow/python/platform:gfile",
+            "//tensorflow/python/platform:test",
+            "//tensorflow/python/platform:tf_logging",
+            "//tensorflow/tools/test:run_and_gather_logs_main_lib",
+        ],
+        **kwargs
+    )
+
+def add_benchmark_tag_to_kwargs(kwargs):
+    """Adds the `benchmark-test` tag to the kwargs, if not already present.
+
+    Notes:
+      For benchmarks which are not technically tests, but whose class methods
+      can still be discovered, and run as such via `bazel run`.
+    Args:
+      kwargs: kwargs to be passed to a test wrapper/rule further down.
+    Returns:
+      kwargs: kwargs with the tags including the `benchmark-test` tags.
+    """
+    benchmark_tag = "benchmark-test"
+    if "tags" in kwargs and kwargs["tags"] != None:
+        if benchmark_tag not in kwargs["tags"]:
+            kwargs["tags"].append(benchmark_tag)
+    else:
+        kwargs["tags"] = [benchmark_tag]
+    return kwargs
+
+def tf_py_benchmark_test(**kwargs):
+    kwargs = add_benchmark_tag_to_kwargs(kwargs)
+    tf_py_strict_test(**kwargs)
+
+def cuda_py_benchmark_test(**kwargs):
+    kwargs = add_benchmark_tag_to_kwargs(kwargs)
+    cuda_py_strict_test(**kwargs)
+
+# Create a benchmark test target of a TensorFlow python test (*py_tests)
+def tf_py_logged_benchmark(
+        name = None,
+        target = None,
+        benchmarks = "..",
+        tags = [],
+        **kwargs):
+    # For now generating a py benchmark is the same as generating a C++
+    # benchmark target. In the future this may change, so we have
+    # two macros just in case
+    tf_cc_logged_benchmark(
+        name = name,
+        target = target,
+        benchmarks = benchmarks,
+        tags = tags,
+        benchmark_type = "python_benchmark",
+        **kwargs
+    )

tensorflow/tools/test/run_and_gather_logs.py

@@ -0,0 +1,125 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test runner for TensorFlow tests."""
+
+import os
+import shlex
+import sys
+import time
+
+from absl import app
+from absl import flags
+
+from google.protobuf import json_format
+from google.protobuf import text_format
+from tensorflow.core.util import test_log_pb2
+from tensorflow.python.platform import gfile
+from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging
+from tensorflow.tools.test import run_and_gather_logs_lib
+
+# pylint: disable=g-import-not-at-top
+# pylint: disable=g-bad-import-order
+# pylint: disable=unused-import
+# Note: cpuinfo and psutil are not installed for you in the TensorFlow
+# OSS tree.  They are installable via pip.
+try:
+  import cpuinfo
+  import psutil
+except ImportError as e:
+  tf_logging.error("\n\n\nERROR: Unable to import necessary library: {}.  "
+                   "Issuing a soft exit.\n\n\n".format(e))
+  sys.exit(0)
+# pylint: enable=g-bad-import-order
+# pylint: enable=unused-import
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string("name", "", """Benchmark target identifier.""")
+flags.DEFINE_string("test_name", "", """Test target to run.""")
+flags.DEFINE_multi_string(
+    "test_args", "", """\
+Test arguments, space separated. May be specified more than once, in which case
+the args are all appended.""")
+flags.DEFINE_boolean("test_log_output_use_tmpdir", False,
+                     "Whether to store the log output into tmpdir.")
+flags.DEFINE_string("benchmark_type", "",
+                    """Benchmark type (BenchmarkType enum string).""")
+flags.DEFINE_string("compilation_mode", "",
+                    """Mode used during this build (e.g. opt, dbg).""")
+flags.DEFINE_string("cc_flags", "", """CC flags used during this build.""")
+flags.DEFINE_string("test_log_output_dir", "",
+                    """Directory for benchmark results output.""")
+flags.DEFINE_string(
+    "test_log_output_filename", "",
+    """Filename to write output benchmark results to. If the filename
+                    is not specified, it will be automatically created.""")
+flags.DEFINE_boolean("skip_export", False,
+                     "Whether to skip exporting test results.")
+
+
+def gather_build_configuration():
+  build_config = test_log_pb2.BuildConfiguration()
+  build_config.mode = FLAGS.compilation_mode
+  # Include all flags except includes
+  cc_flags = [
+      flag for flag in shlex.split(FLAGS.cc_flags) if not flag.startswith("-i")
+  ]
+  build_config.cc_flags.extend(cc_flags)
+  return build_config
+
+
+def main(unused_args):
+  name = FLAGS.name
+  test_name = FLAGS.test_name
+  test_args = " ".join(FLAGS.test_args)
+  benchmark_type = FLAGS.benchmark_type
+  test_results, _ = run_and_gather_logs_lib.run_and_gather_logs(
+      name,
+      test_name=test_name,
+      test_args=test_args,
+      benchmark_type=benchmark_type,
+      skip_processing_logs=FLAGS.skip_export)
+  if FLAGS.skip_export:
+    return
+
+  # Additional bits we receive from bazel
+  test_results.build_configuration.CopyFrom(gather_build_configuration())
+  # Add os.environ data to test_results.
+  test_results.run_configuration.env_vars.update(os.environ)
+
+  if not FLAGS.test_log_output_dir:
+    print(text_format.MessageToString(test_results))
+    return
+
+  if FLAGS.test_log_output_filename:
+    file_name = FLAGS.test_log_output_filename
+  else:
+    file_name = (
+        name.strip("/").translate(str.maketrans("/:", "__")) +
+        time.strftime("%Y%m%d%H%M%S", time.gmtime()))
+  if FLAGS.test_log_output_use_tmpdir:
+    tmpdir = test.get_temp_dir()
+    output_path = os.path.join(tmpdir, FLAGS.test_log_output_dir, file_name)
+  else:
+    output_path = os.path.join(
+        os.path.abspath(FLAGS.test_log_output_dir), file_name)
+  json_test_results = json_format.MessageToJson(test_results)
+  gfile.GFile(output_path + ".json", "w").write(json_test_results)
+  tf_logging.info("Test results written to: %s" % output_path)
+
+
+if __name__ == "__main__":
+  app.run(main)

tensorflow/tools/test/run_and_gather_logs_lib.py

@@ -0,0 +1,189 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library for getting system information during TensorFlow tests."""
+
+import os
+import re
+import shlex
+import subprocess
+import tempfile
+import time
+
+from tensorflow.core.util import test_log_pb2
+from tensorflow.python.platform import gfile
+from tensorflow.tools.test import gpu_info_lib
+from tensorflow.tools.test import system_info_lib
+
+
+class MissingLogsError(Exception):
+  pass
+
+
+def get_git_commit_sha():
+  """Get git commit SHA for this build.
+
+  Attempt to get the SHA from environment variable GIT_COMMIT, which should
+  be available on Jenkins build agents.
+
+  Returns:
+    SHA hash of the git commit used for the build, if available
+  """
+
+  return os.getenv("GIT_COMMIT")
+
+
+def process_test_logs(name, test_name, test_args, benchmark_type,
+                      start_time, run_time, log_files):
+  """Gather test information and put it in a TestResults proto.
+
+  Args:
+    name: Benchmark target identifier.
+    test_name: A unique bazel target, e.g. "//path/to:test"
+    test_args: A string containing all arguments to run the target with.
+    benchmark_type: A string representing the BenchmarkType enum; the
+      benchmark type for this target.
+    start_time: Test starting time (epoch)
+    run_time:   Wall time that the test ran for
+    log_files:  Paths to the log files
+
+  Returns:
+    A TestResults proto
+  """
+
+  results = test_log_pb2.TestResults()
+  results.name = name
+  results.target = test_name
+  results.start_time = start_time
+  results.run_time = run_time
+  results.benchmark_type = test_log_pb2.TestResults.BenchmarkType.Value(
+      benchmark_type.upper())
+
+  # Gather source code information
+  git_sha = get_git_commit_sha()
+  if git_sha:
+    results.commit_id.hash = git_sha
+
+  results.entries.CopyFrom(process_benchmarks(log_files))
+  results.run_configuration.argument.extend(test_args)
+  results.machine_configuration.CopyFrom(
+      system_info_lib.gather_machine_configuration())
+  return results
+
+
+def process_benchmarks(log_files):
+  benchmarks = test_log_pb2.BenchmarkEntries()
+  for f in log_files:
+    content = gfile.GFile(f, "rb").read()
+    if benchmarks.MergeFromString(content) != len(content):
+      raise Exception("Failed parsing benchmark entry from %s" % f)
+  return benchmarks
+
+
+def run_and_gather_logs(name,
+                        test_name,
+                        test_args,
+                        benchmark_type,
+                        skip_processing_logs=False):
+  """Run the bazel test given by test_name.  Gather and return the logs.
+
+  Args:
+    name: Benchmark target identifier.
+    test_name: A unique bazel target, e.g. "//path/to:test"
+    test_args: A string containing all arguments to run the target with.
+    benchmark_type: A string representing the BenchmarkType enum; the
+      benchmark type for this target.
+    skip_processing_logs: Whether to skip processing test results from log
+      files.
+
+  Returns:
+    A tuple (test_results, mangled_test_name), where
+    test_results: A test_log_pb2.TestResults proto, or None if log processing
+      is skipped.
+    test_adjusted_name: Unique benchmark name that consists of
+      benchmark name optionally followed by GPU type.
+
+  Raises:
+    ValueError: If the test_name is not a valid target.
+    subprocess.CalledProcessError: If the target itself fails.
+    IOError: If there are problems gathering test log output from the test.
+    MissingLogsError: If we couldn't find benchmark logs.
+  """
+  if not (test_name and test_name.startswith("//") and ".." not in test_name and
+          not test_name.endswith(":") and not test_name.endswith(":all") and
+          not test_name.endswith("...") and len(test_name.split(":")) == 2):
+    raise ValueError("Expected test_name parameter with a unique test, e.g.: "
+                     "--test_name=//path/to:test")
+  test_executable = test_name.rstrip().strip("/").replace(":", "/")
+
+  if gfile.Exists(os.path.join("bazel-bin", test_executable)):
+    # Running in standalone mode from core of the repository
+    test_executable = os.path.join("bazel-bin", test_executable)
+  else:
+    # Hopefully running in sandboxed mode
+    test_executable = os.path.join(".", test_executable)
+
+  test_adjusted_name = name
+  gpu_config = gpu_info_lib.gather_gpu_devices()
+  if gpu_config:
+    gpu_name = gpu_config[0].model
+    gpu_short_name_match = re.search(
+        r"(Tesla|NVIDIA) (K40|K80|P100|V100|A100)", gpu_name
+    )
+    if gpu_short_name_match:
+      gpu_short_name = gpu_short_name_match.group(0)
+      test_adjusted_name = name + "|" + gpu_short_name.replace(" ", "_")
+
+  temp_directory = tempfile.mkdtemp(prefix="run_and_gather_logs")
+  mangled_test_name = (
+      test_adjusted_name.strip("/").replace("|",
+                                            "_").replace("/",
+                                                         "_").replace(":", "_"))
+  test_file_prefix = os.path.join(temp_directory, mangled_test_name)
+  test_file_prefix = "%s." % test_file_prefix
+
+  try:
+    if not gfile.Exists(test_executable):
+      test_executable_py3 = test_executable + ".python3"
+      if not gfile.Exists(test_executable_py3):
+        raise ValueError("Executable does not exist: %s" % test_executable)
+      test_executable = test_executable_py3
+    test_args = shlex.split(test_args)
+
+    # This key is defined in tf/core/util/reporter.h as
+    # TestReporter::kTestReporterEnv.
+    os.environ["TEST_REPORT_FILE_PREFIX"] = test_file_prefix
+    start_time = time.time()
+    subprocess.check_call([test_executable] + test_args)
+    if skip_processing_logs:
+      return None, test_adjusted_name
+    run_time = time.time() - start_time
+    log_files = gfile.Glob("{}*".format(test_file_prefix))
+    if not log_files:
+      raise MissingLogsError("No log files found at %s." % test_file_prefix)
+
+    return (process_test_logs(
+        test_adjusted_name,
+        test_name=test_name,
+        test_args=test_args,
+        benchmark_type=benchmark_type,
+        start_time=int(start_time),
+        run_time=run_time,
+        log_files=log_files), test_adjusted_name)
+
+  finally:
+    try:
+      gfile.DeleteRecursively(temp_directory)
+    except OSError:
+      pass

tensorflow/tools/test/system_info.py

@@ -0,0 +1,27 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library for getting system information during TensorFlow tests."""
+
+from absl import app
+from tensorflow.tools.test import system_info_lib
+
+
+def main(unused_args):
+  config = system_info_lib.gather_machine_configuration()
+  print(config)
+
+
+if __name__ == "__main__":
+  app.run()  # pylint: disable=no-value-for-parameter

tensorflow/tools/test/system_info_lib.py

@@ -0,0 +1,146 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library for getting system information during TensorFlow tests."""
+
+import glob
+import multiprocessing
+import platform
+import re
+import socket
+
+# pylint: disable=g-bad-import-order
+# Note: cpuinfo and psutil are not installed for you in the TensorFlow
+# OSS tree.  They are installable via pip.
+import cpuinfo
+import psutil
+# pylint: enable=g-bad-import-order
+
+from tensorflow.core.util import test_log_pb2
+from tensorflow.python.client import device_lib
+from tensorflow.python.framework import errors
+from tensorflow.python.platform import gfile
+from tensorflow.tools.test import gpu_info_lib
+
+
+def gather_machine_configuration():
+  """Gather Machine Configuration.  This is the top level fn of this library."""
+  config = test_log_pb2.MachineConfiguration()
+
+  config.cpu_info.CopyFrom(gather_cpu_info())
+  config.platform_info.CopyFrom(gather_platform_info())
+
+  # gather_available_device_info must come before gather_gpu_devices
+  # because the latter may access libcudart directly, which confuses
+  # TensorFlow StreamExecutor.
+  for d in gather_available_device_info():
+    config.available_device_info.add().CopyFrom(d)
+  for gpu in gpu_info_lib.gather_gpu_devices():
+    config.device_info.add().Pack(gpu)
+
+  config.memory_info.CopyFrom(gather_memory_info())
+
+  config.hostname = gather_hostname()
+
+  return config
+
+
+def gather_hostname():
+  return socket.gethostname()
+
+
+def gather_memory_info():
+  """Gather memory info."""
+  mem_info = test_log_pb2.MemoryInfo()
+  vmem = psutil.virtual_memory()
+  mem_info.total = vmem.total
+  mem_info.available = vmem.available
+  return mem_info
+
+
+def gather_cpu_info():
+  """Gather CPU Information.  Assumes all CPUs are the same."""
+  cpu_info = test_log_pb2.CPUInfo()
+  cpu_info.num_cores = multiprocessing.cpu_count()
+
+  # Gather num_cores_allowed
+  try:
+    with gfile.GFile('/proc/self/status', 'rb') as fh:
+      nc = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', fh.read().decode('utf-8'))
+    if nc:  # e.g. 'ff' => 8, 'fff' => 12
+      cpu_info.num_cores_allowed = (
+          bin(int(nc.group(1).replace(',', ''), 16)).count('1'))
+  except errors.OpError:
+    pass
+  finally:
+    if cpu_info.num_cores_allowed == 0:
+      cpu_info.num_cores_allowed = cpu_info.num_cores
+
+  # Gather the rest
+  info = cpuinfo.get_cpu_info()
+  cpu_info.cpu_info = info['brand']
+  cpu_info.num_cores = info['count']
+  cpu_info.mhz_per_cpu = info['hz_advertised_raw'][0] / 1.0e6
+  l2_cache_size = re.match(r'(\d+)', str(info.get('l2_cache_size', '')))
+  if l2_cache_size:
+    # If a value is returned, it's in KB
+    cpu_info.cache_size['L2'] = int(l2_cache_size.group(0)) * 1024
+
+  # Try to get the CPU governor
+  try:
+    cpu_governors = set([
+        gfile.GFile(f, 'r').readline().rstrip()
+        for f in glob.glob(
+            '/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor')
+    ])
+    if cpu_governors:
+      if len(cpu_governors) > 1:
+        cpu_info.cpu_governor = 'mixed'
+      else:
+        cpu_info.cpu_governor = list(cpu_governors)[0]
+  except errors.OpError:
+    pass
+
+  return cpu_info
+
+
+def gather_available_device_info():
+  """Gather list of devices available to TensorFlow.
+
+  Returns:
+    A list of test_log_pb2.AvailableDeviceInfo messages.
+  """
+  device_info_list = []
+  devices = device_lib.list_local_devices()
+
+  for d in devices:
+    device_info = test_log_pb2.AvailableDeviceInfo()
+    device_info.name = d.name
+    device_info.type = d.device_type
+    device_info.memory_limit = d.memory_limit
+    device_info.physical_description = d.physical_device_desc
+    device_info_list.append(device_info)
+
+  return device_info_list
+
+
+def gather_platform_info():
+  """Gather platform info."""
+  platform_info = test_log_pb2.PlatformInfo()
+  (platform_info.bits, platform_info.linkage) = platform.architecture()
+  platform_info.machine = platform.machine()
+  platform_info.release = platform.release()
+  platform_info.system = platform.system()
+  platform_info.version = platform.version()
+  return platform_info

tensorflow/tools/test/upload_test_benchmarks.py

@@ -0,0 +1,244 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Command to upload benchmark test results to a cloud datastore.
+
+This uploader script is typically run periodically as a cron job.  It locates,
+in a specified data directory, files that contain benchmark test results.  The
+results are written by the "run_and_gather_logs.py" script using the JSON-format
+serialization of the "TestResults" protobuf message (core/util/test_log.proto).
+
+For each file, the uploader reads the "TestResults" data, transforms it into
+the schema used in the datastore (see below), and upload it to the datastore.
+After processing a file, the uploader moves it to a specified archive directory
+for safe-keeping.
+
+The uploader uses file-level exclusive locking (non-blocking flock) which allows
+multiple instances of this script to run concurrently if desired, splitting the
+task among them, each one processing and archiving different files.
+
+The "TestResults" object contains test metadata and multiple benchmark entries.
+The datastore schema splits this information into two Kinds (like tables), one
+holding the test metadata in a single "Test" Entity (like rows), and one holding
+each related benchmark entry in a separate "Entry" Entity.  Datastore create a
+unique ID (retrieval key) for each Entity, and this ID is always returned along
+with the data when an Entity is fetched.
+
+* Test:
+  - test:   unique name of this test (string)
+  - start:  start time of this test run (datetime)
+  - info:   JSON-encoded test metadata (string, not indexed)
+
+* Entry:
+  - test:   unique name of this test (string)
+  - entry:  unique name of this benchmark entry within this test (string)
+  - start:  start time of this test run (datetime)
+  - timing: average time (usec) per iteration of this test/entry run (float)
+  - info:   JSON-encoded entry metadata (string, not indexed)
+
+A few composite indexes are created (upload_test_benchmarks_index.yaml) for fast
+retrieval of benchmark data and reduced I/O to the client without adding a lot
+of indexing and storage burden:
+
+* Test: (test, start) is indexed to fetch recent start times for a given test.
+
+* Entry: (test, entry, start, timing) is indexed to use projection and only
+fetch the recent (start, timing) data for a given test/entry benchmark.
+
+Example retrieval GQL statements:
+
+* Get the recent start times for a given test:
+  SELECT start FROM Test WHERE test = <test-name> AND
+    start >= <recent-datetime> LIMIT <count>
+
+* Get the recent timings for a given benchmark:
+  SELECT start, timing FROM Entry WHERE test = <test-name> AND
+    entry = <entry-name> AND start >= <recent-datetime> LIMIT <count>
+
+* Get all test names uniquified (e.g. display a list of available tests):
+  SELECT DISTINCT ON (test) test FROM Test
+
+* For a given test (from the list above), get all its entry names.  The list of
+  entry names can be extracted from the test "info" metadata for a given test
+  name and start time (e.g. pick the latest start time for that test).
+  SELECT * FROM Test WHERE test = <test-name> AND start = <latest-datetime>
+"""
+
+import argparse
+import datetime
+import fcntl
+import json
+import os
+import shutil
+
+from google.cloud import datastore
+
+
+def is_real_file(dirpath, fname):
+  fpath = os.path.join(dirpath, fname)
+  return os.path.isfile(fpath) and not os.path.islink(fpath)
+
+
+def get_mtime(dirpath, fname):
+  fpath = os.path.join(dirpath, fname)
+  return os.stat(fpath).st_mtime
+
+
+def list_files_by_mtime(dirpath):
+  """Return a list of files in the directory, sorted in increasing "mtime".
+
+  Return a list of files in the given directory, sorted from older to newer file
+  according to their modification times.  Only return actual files, skipping
+  directories, symbolic links, pipes, etc.
+
+  Args:
+    dirpath: directory pathname
+
+  Returns:
+    A list of file names relative to the given directory path.
+  """
+  files = [f for f in os.listdir(dirpath) if is_real_file(dirpath, f)]
+  return sorted(files, key=lambda f: get_mtime(dirpath, f))
+
+
+# Note: The file locking code uses flock() instead of lockf() because benchmark
+# files are only opened for reading (not writing) and we still want exclusive
+# locks on them.  This imposes the limitation that the data directory must be
+# local, not NFS-mounted.
+def lock(fd):
+  fcntl.flock(fd, fcntl.LOCK_EX)
+
+
+def unlock(fd):
+  fcntl.flock(fd, fcntl.LOCK_UN)
+
+
+def trylock(fd):
+  try:
+    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    return True
+  except Exception:  # pylint: disable=broad-except
+    return False
+
+
+def upload_benchmark_data(client, data):
+  """Parse benchmark data and use the client to upload it to the datastore.
+
+  Parse the given benchmark data from the serialized JSON-format used to write
+  the test results file.  Create the different datastore Entities from that data
+  and upload them to the datastore in a batch using the client connection.
+
+  Args:
+    client: datastore client connection
+    data: JSON-encoded benchmark data
+  """
+  test_result = json.loads(data)
+
+  test_name = str(test_result["name"])
+  start_time = datetime.datetime.utcfromtimestamp(
+      float(test_result["startTime"]))
+  batch = []
+
+  # Create the Test Entity containing all the test information as a
+  # non-indexed JSON blob.
+  t_key = client.key("Test")
+  t_val = datastore.Entity(t_key, exclude_from_indexes=["info"])
+  t_val.update({"test": test_name, "start": start_time, "info": str(data)})
+  batch.append(t_val)
+
+  # Create one Entry Entity for each benchmark entry.  The wall-clock timing is
+  # the attribute to be fetched and displayed.  The full entry information is
+  # also stored as a non-indexed JSON blob.
+  for ent in test_result["entries"].get("entry", []):
+    ent_name = str(ent["name"])
+    e_key = client.key("Entry")
+    e_val = datastore.Entity(e_key, exclude_from_indexes=["info"])
+    e_val.update({
+        "test": test_name,
+        "start": start_time,
+        "entry": ent_name,
+        "timing": ent["wallTime"],
+        "info": str(json.dumps(ent))
+    })
+    batch.append(e_val)
+
+  # Put the whole batch of Entities in the datastore.
+  client.put_multi(batch)
+
+
+def upload_benchmark_files(opts):
+  """Find benchmark files, process them, and upload their data to the datastore.
+
+  Locate benchmark files in the data directory, process them, and upload their
+  data to the datastore.  After processing each file, move it to the archive
+  directory for safe-keeping.  Each file is locked for processing, which allows
+  multiple uploader instances to run concurrently if needed, each one handling
+  different benchmark files, skipping those already locked by another.
+
+  Args:
+    opts: command line options object
+
+  Note: To use locking, the file is first opened, then its descriptor is used to
+  lock and read it.  The lock is released when the file is closed.  Do not open
+  that same file a 2nd time while the lock is already held, because when that
+  2nd file descriptor is closed, the lock will be released prematurely.
+  """
+  client = datastore.Client()
+
+  for fname in list_files_by_mtime(opts.datadir):
+    fpath = os.path.join(opts.datadir, fname)
+    try:
+      with open(fpath, "r") as fd:
+        if trylock(fd):
+          upload_benchmark_data(client, fd.read())
+          shutil.move(fpath, os.path.join(opts.archivedir, fname))
+          # unlock(fd) -- When "with open()" closes fd, the lock is released.
+    except Exception as e:  # pylint: disable=broad-except
+      print("Cannot process '%s', skipping. Error: %s" % (fpath, e))
+
+
+def parse_cmd_line():
+  """Parse command line options.
+
+  Returns:
+    The parsed arguments object.
+  """
+  desc = "Upload benchmark results to datastore."
+  opts = [
+      ("-a", "--archivedir", str, None, True,
+       "Directory where benchmark files are archived."),
+      ("-d", "--datadir", str, None, True,
+       "Directory of benchmark files to upload."),
+  ]
+
+  parser = argparse.ArgumentParser(description=desc)
+  for opt in opts:
+    parser.add_argument(opt[0], opt[1], type=opt[2], default=opt[3],
+                        required=opt[4], help=opt[5])
+  return parser.parse_args()
+
+
+def main():
+  options = parse_cmd_line()
+
+  # Check that credentials are specified to access the datastore.
+  if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
+    raise ValueError("GOOGLE_APPLICATION_CREDENTIALS env. var. is not set.")
+
+  upload_benchmark_files(options)
+
+
+if __name__ == "__main__":
+  main()

tensorflow/tools/test/upload_test_benchmarks_index.yaml

@@ -0,0 +1,42 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# Note: This file is given a descriptive name in the source tree.  However, for
+# some reason, the command to create/update datastore indexes requires it to be
+# named "index.yaml" even though the filename is given on the command line.  So
+# make a copy of it into a temporary "index.yaml" file and issue this command:
+# % gcloud datastore create-indexes index.yaml
+
+# Index to access a specific (test, start) Entity, and also to be able to only
+# fetch a range of start times of some test without fetching the "info" blob.
+indexes:
+- kind: Test
+  ancestor: no
+  properties:
+  - name: test
+  - name: start
+    direction: desc
+
+# Index to access a specific (test, entry, start) Entity, and also to be able to
+# fetch a range of (start, timing) graph values for a given (test, entry) pair
+# without fetching the "info" blob.
+- kind: Entry
+  ancestor: no
+  properties:
+  - name: test
+  - name: entry
+  - name: start
+    direction: asc
+  - name: timing