chore: import upstream snapshot with attribution

tensorflow/tools/ci_build/builds/android.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/builds_common.sh"
+# To setup Android via `configure` script.
+export TF_SET_ANDROID_WORKSPACE=1
+yes "" | ./configure
+
+# The Bazel builds are intentionally built for x86 and arm64 to maximize build
+# coverage while minimizing compilation time. For full build coverage and
+# exposed binaries, see android_full.sh
+
+echo "========== TensorFlow Basic Build Test =========="
+TARGETS=
+# Building the Eager Runtime ensures compatibility with Android for the
+# benefits of clients like TensorFlow Lite. For now it is enough to build only
+# :execute, which what TF Lite needs. Note that this does *not* build the
+# full set of mobile ops/kernels, as that can be prohibitively expensive.
+TARGETS+=" //tensorflow/core/common_runtime/eager:execute"
+bazel --bazelrc=/dev/null build \
+    --compilation_mode=opt --cxxopt=-std=c++17 \
+    --config=android_arm64 --fat_apk_cpu=x86,arm64-v8a \
+    ${TARGETS}
+
+# TODO(b/122377443): Restore Makefile builds after resolving r18b build issues.

tensorflow/tools/ci_build/builds/android_full.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+copy_lib() {
+  FILE=$1
+  TARGET_DIR=${OUT_DIR}/native/$(basename $FILE)/${CPU}
+  mkdir -p ${TARGET_DIR}
+  echo "Copying ${FILE} to ${TARGET_DIR}"
+  cp ${FILE} ${TARGET_DIR}
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/builds_common.sh"
+# To setup Android via `configure` script.
+export TF_SET_ANDROID_WORKSPACE=1
+yes "" | ./configure
+
+CPUS=armeabi-v7a,arm64-v8a
+
+OUT_DIR="$(pwd)/out/"
+AAR_LIB_TMP="$(pwd)/aar_libs"
+
+rm -rf ${OUT_DIR}
+rm -rf ${AAR_LIB_TMP}
+
+# Build all relevant native libraries for each architecture.
+for CPU in ${CPUS//,/ }
+do
+    echo "========== Building native libs for Android ${CPU} =========="
+    bazel build --config=monolithic --cpu=${CPU} \
+        --compilation_mode=opt --cxxopt=-std=c++17 \
+        --crosstool_top=//external:android/crosstool \
+        --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+        //tensorflow/core:portable_tensorflow_lib \
+        //tensorflow/tools/android/inference_interface:libtensorflow_inference.so \
+        //tensorflow/tools/android/test:libtensorflow_demo.so \
+        //tensorflow/tools/benchmark:benchmark_model
+
+    copy_lib bazel-bin/tensorflow/tools/android/inference_interface/libtensorflow_inference.so
+    copy_lib bazel-bin/tensorflow/tools/android/test/libtensorflow_demo.so
+    copy_lib bazel-bin/tensorflow/tools/benchmark/benchmark_model
+
+    mkdir -p ${AAR_LIB_TMP}/jni/${CPU}
+    cp bazel-bin/tensorflow/tools/android/inference_interface/libtensorflow_inference.so ${AAR_LIB_TMP}/jni/${CPU}
+done
+
+# Build Jar and also demo containing native libs for all architectures.
+# Enable sandboxing so that zip archives don't get incorrectly packaged
+# in assets/ dir (see https://github.com/bazelbuild/bazel/issues/2334)
+# TODO(gunan): remove extra flags once sandboxing is enabled for all builds.
+echo "========== Building TensorFlow Android Jar and Demo =========="
+bazel --bazelrc=/dev/null build --config=monolithic --fat_apk_cpu=${CPUS} \
+    --compilation_mode=opt --cxxopt=-std=c++17 \
+    --define=android_dexmerger_tool=d8_dexmerger \
+    --define=android_incremental_dexing_tool=d8_dexbuilder \
+    --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+    --spawn_strategy=sandboxed --genrule_strategy=sandboxed \
+    //tensorflow/tools/android/inference_interface:android_tensorflow_inference_java \
+    //tensorflow/tools/android/inference_interface:android_tensorflow_inference_java.aar \
+    //tensorflow/tools/android/test:tensorflow_demo
+
+echo "Copying demo, AAR and Jar to ${OUT_DIR}"
+cp bazel-bin/tensorflow/tools/android/test/tensorflow_demo.apk \
+    bazel-bin/tensorflow/tools/android/inference_interface/libandroid_tensorflow_inference_java.jar ${OUT_DIR}
+
+cp bazel-bin/tensorflow/tools/android/inference_interface/android_tensorflow_inference_java.aar \
+   ${OUT_DIR}/tensorflow.aar
+
+# TODO(andrewharp): build native libs into AAR directly once
+# https://github.com/bazelbuild/bazel/issues/348 is resolved.
+echo "Adding native libs to AAR"
+chmod +w ${OUT_DIR}/tensorflow.aar
+pushd ${AAR_LIB_TMP}
+zip -ur ${OUT_DIR}/tensorflow.aar $(find jni -name *.so)
+popd
+rm -rf ${AAR_LIB_TMP}
+
+# TODO(b/122377443): Restore Makefile builds after resolving r18b build issues.

tensorflow/tools/ci_build/builds/benchmark.sh

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Runs benchmark tests.
+# After the completion of each benchmark test, the script calls a hook binary,
+# specified with the environment variable TF_BUILD_BENCHMARK_HOOK, to handle
+# the test log file. This hook binary may perform operations such as entering
+# the test results into a database.
+#
+# Usage: benchmark [-c opt]
+# Option flags
+#    -c opt:  Use optimized C++ build ("-c opt")
+#
+# This script obeys the following environmental variables:
+#   TF_BUILD_BENCHMARK_HOOK:
+#     Path to a binary / script that will handle the test log and other related
+#     info after the completion of each benchmark test.
+
+set -u
+
+echo ""
+echo "====== Benchmark tests start ======"
+
+# Process input arguments
+OPT_FLAG=""
+while getopts c: flag; do
+  case ${flag} in
+    c)
+      if [[ ! -z "{OPTARG}" ]]; then
+        OPT_FLAG="${OPT_FLAG} -c ${OPTARG}"
+      fi
+      ;;
+  esac
+done
+
+BENCHMARK_HOOK=${TF_BUILD_BENCHMARK_HOOK:-""}
+
+
+BENCHMARK_TAG="benchmark-test"
+BENCHMARK_TESTS=$(bazel query \
+    'attr("tags", "'"${BENCHMARK_TAG}"'", //tensorflow/...)')
+
+if [[ -z "${BENCHMARK_TESTS}" ]]; then
+  echo "ERROR: Cannot find any benchmark tests with the tag "\
+"\"${BENCHMARK_TAG}\""
+  exit 1
+fi
+
+N_TESTS=$(echo ${BENCHMARK_TESTS} | wc -w)
+
+echo "Discovered ${N_TESTS} benchmark test(s) with the tag \"${BENCHMARK_TAG}\":"
+echo ${BENCHMARK_TESTS}
+echo ""
+
+PASS_COUNTER=0
+FAIL_COUNTER=0
+FAILED_TESTS=""
+COUNTER=0
+
+# Iterate through the benchmark tests
+for BENCHMARK_TEST in ${BENCHMARK_TESTS}; do
+  ((COUNTER++))
+
+  echo ""
+  echo "Running benchmark test (${COUNTER} / ${N_TESTS}): ${BENCHMARK_TEST}"
+
+  bazel test ${OPT_FLAG} --cache_test_results=no "${BENCHMARK_TEST}"
+  TEST_RESULT=$?
+
+  # Hook for database
+  # Verify that test log exists
+  TEST_LOG=$(echo ${BENCHMARK_TEST} |  sed -e 's/:/\//g')
+  TEST_LOG="bazel-testlogs/${TEST_LOG}/test.log"
+  if [[ -f "${TEST_LOG}" ]]; then
+    echo "Benchmark ${BENCHMARK_TEST} done: log @ ${TEST_LOG}"
+
+    # Call database hook if exists
+    if [[ ! -z "${BENCHMARK_HOOK}" ]]; then
+      # Assume that the hook binary/script takes two arguments:
+      #   Argument 1: Compilation flags such as "-c opt" as a whole
+      #   Argument 2: Test log containing the serialized TestResults proto
+
+      echo "Calling database hook: ${TF_BUILD_BENCHMARK_LOG_HOOK} "\
+"${OPT_FLAG} ${TEST_LOG}"
+
+      ${TF_BUILD_BENCHMARK_LOG_HOOK} "${OPT_FLAG}" "${TEST_LOG}"
+    else
+      echo "WARNING: No hook binary is specified to handle test log ${TEST_LOG}"
+    fi
+  else
+    # Mark as failure if the test log file cannot be found
+    TEST_RESULT=2
+
+    echo "ERROR: Cannot find log file from benchmark ${BENCHMARK_TEST} @ "\
+"${TEST_LOG}"
+  fi
+
+  echo ""
+  if [[ ${TEST_RESULT} -eq 0 ]]; then
+    ((PASS_COUNTER++))
+
+    echo "Benchmark test PASSED: ${BENCHMARK_TEST}"
+  else
+    ((FAIL_COUNTER++))
+
+    FAILED_TESTS="${FAILED_TESTS} ${BENCHMARK_TEST}"
+
+    echo "Benchmark test FAILED: ${BENCHMARK_TEST}"
+
+    if [[ -f "${TEST_LOG}" ]]; then
+      echo "============== BEGINS failure log content =============="
+      cat ${TEST_LOG} >&2
+      echo "============== ENDS failure log content =============="
+      echo ""
+    fi
+  fi
+
+done
+
+# Summarize test results
+echo ""
+echo "${N_TESTS} Benchmark test(s):" \
+     "${PASS_COUNTER} passed;" \
+     "${FAIL_COUNTER} failed"
+
+if [[ ${FAIL_COUNTER} -eq 0  ]]; then
+  echo ""
+  echo "Benchmark tests SUCCEEDED"
+
+  exit 0
+else
+  echo "FAILED benchmark test(s):"
+  FAIL_COUNTER=0
+  for TEST_NAME in ${FAILED_TESTS}; do
+    echo "  ${TEST_NAME}"
+    ((FAIL_COUNTER++))
+  done
+
+  echo ""
+  echo "Benchmark tests FAILED"
+  exit 1
+fi

tensorflow/tools/ci_build/builds/builds_common.sh

@@ -0,0 +1,242 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Common Bash functions used by build scripts
+
+COLOR_NC='\033[0m'
+COLOR_BOLD='\033[1m'
+COLOR_LIGHT_GRAY='\033[0;37m'
+COLOR_GREEN='\033[0;32m'
+COLOR_RED='\033[0;31m'
+
+die() {
+  # Print a message and exit with code 1.
+  #
+  # Usage: die <error_message>
+  #   e.g., die "Something bad happened."
+
+  echo $@
+  exit 1
+}
+
+realpath() {
+  # Get the real path of a file
+  # Usage: realpath <file_path>
+
+  if [[ $# != "1" ]]; then
+    die "realpath: incorrect usage"
+  fi
+
+  [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
+}
+
+to_lower () {
+  # Convert string to lower case.
+  # Usage: to_lower <string>
+
+  echo "$1" | tr '[:upper:]' '[:lower:]'
+}
+
+calc_elapsed_time() {
+  # Calculate elapsed time. Takes nanosecond format input of the kind output
+  # by date +'%s%N'
+  #
+  # Usage: calc_elapsed_time <START_TIME> <END_TIME>
+
+  if [[ $# != "2" ]]; then
+    die "calc_elapsed_time: incorrect usage"
+  fi
+
+  START_TIME=$1
+  END_TIME=$2
+
+  if [[ ${START_TIME} == *"N" ]]; then
+    # Nanosecond precision not available
+    START_TIME=$(echo ${START_TIME} | sed -e 's/N//g')
+    END_TIME=$(echo ${END_TIME} | sed -e 's/N//g')
+    ELAPSED="$(expr ${END_TIME} - ${START_TIME}) s"
+  else
+    ELAPSED="$(expr $(expr ${END_TIME} - ${START_TIME}) / 1000000) ms"
+  fi
+
+  echo ${ELAPSED}
+}
+
+run_in_directory() {
+  # Copy the test script to a destination directory and run the test there.
+  # Write test log to a log file.
+  #
+  # Usage: run_in_directory <DEST_DIR> <LOG_FILE> <TEST_SCRIPT>
+  #                         [ARGS_FOR_TEST_SCRIPT]
+
+  if [[ $# -lt "3" ]]; then
+    die "run_in_directory: incorrect usage"
+  fi
+
+  DEST_DIR="$1"
+  LOG_FILE="$2"
+  TEST_SCRIPT="$3"
+  shift 3
+  SCRIPT_ARGS=("$@")
+
+  # Get the absolute path of the log file
+  LOG_FILE_ABS=$(realpath "${LOG_FILE}")
+
+  cp "${TEST_SCRIPT}" "${DEST_DIR}"/
+  SCRIPT_BASENAME=$(basename "${TEST_SCRIPT}")
+
+  if [[ ! -f "${DEST_DIR}/${SCRIPT_BASENAME}" ]]; then
+    echo "FAILED to copy script ${TEST_SCRIPT} to temporary directory "\
+"${DEST_DIR}"
+    return 1
+  fi
+
+  pushd "${DEST_DIR}" > /dev/null
+
+  "${TIMEOUT_BIN}" --preserve-status ${TIMEOUT} \
+    "${PYTHON_BIN_PATH}" "${SCRIPT_BASENAME}" ${SCRIPT_ARGS[@]} 2>&1 \
+    > "${LOG_FILE_ABS}"
+
+  rm -f "${SCRIPT_BASENAME}"
+  popd > /dev/null
+
+  if [[ $? != 0 ]]; then
+    echo "Test \"${SCRIPT_BASENAME}\" FAILED"
+    return 1
+  fi
+
+  return 0
+}
+
+
+test_runner() {
+  # Run a suite of tests, print failure logs (if any), wall-time each test,
+  # and show the summary at the end.
+  #
+  # Usage: test_runner <TEST_DESC> <ALL_TESTS> <TEST_DENYLIST> <LOGS_DIR>
+  # e.g.,  test_runner "Tutorial test-on-install" \
+  #                    "test1 test2 test3" "test2 test3" "/tmp/log_dir"
+
+  if [[ $# != "4" ]]; then
+    die "test_runner: incorrect usage"
+  fi
+
+  TEST_DESC=$1
+  ALL_TESTS_STR=$2
+  TEST_DENYLIST_SR=$3
+  LOGS_DIR=$4
+
+  NUM_TESTS=$(echo "${ALL_TESTS_STR}" | wc -w)
+  ALL_TESTS=(${ALL_TESTS_STR})
+
+  COUNTER=0
+  PASSED_COUNTER=0
+  FAILED_COUNTER=0
+  FAILED_TESTS=""
+  FAILED_TEST_LOGS=""
+  SKIPPED_COUNTER=0
+  for CURR_TEST in ${ALL_TESTS[@]}; do
+    ((COUNTER++))
+    STAT_STR="(${COUNTER} / ${NUM_TESTS})"
+
+    if [[ "${TEST_DENYLIST_STR}" == *"${CURR_TEST}"* ]]; then
+      ((SKIPPED_COUNTER++))
+      echo "${STAT_STR} Denylisted ${TEST_DESC} SKIPPED: ${CURR_TEST}"
+      continue
+    fi
+
+    START_TIME=$(date +'%s%N')
+
+    LOG_FILE="${LOGS_DIR}/${CURR_TEST}.log"
+    rm -rf ${LOG_FILE} ||
+    die "Unable to remove existing log file: ${LOG_FILE}"
+
+    "test_${CURR_TEST}" "${LOG_FILE}"
+    TEST_RESULT=$?
+
+    END_TIME=$(date +'%s%N')
+    ELAPSED_TIME=$(calc_elapsed_time "${START_TIME}" "${END_TIME}")
+
+    if [[ ${TEST_RESULT} == 0 ]]; then
+      ((PASSED_COUNTER++))
+      echo "${STAT_STR} ${TEST_DESC} PASSED: ${CURR_TEST} "\
+  "(Elapsed time: ${ELAPSED_TIME})"
+    else
+      ((FAILED_COUNTER++))
+      FAILED_TESTS="${FAILED_TESTS} ${CURR_TEST}"
+      FAILED_TEST_LOGS="${FAILED_TEST_LOGS} ${LOG_FILE}"
+
+      echo "${STAT_STR} ${TEST_DESC} FAILED: ${CURR_TEST} "\
+  "(Elapsed time: ${ELAPSED_TIME})"
+
+      echo "============== BEGINS failure log content =============="
+      cat ${LOG_FILE}
+      echo "============== ENDS failure log content =============="
+      echo ""
+    fi
+  done
+
+  echo "${NUM_TUT_TESTS} ${TEST_DESC} test(s): "\
+  "${PASSED_COUNTER} passed; ${FAILED_COUNTER} failed; ${SKIPPED_COUNTER} skipped"
+
+  if [[ ${FAILED_COUNTER} -eq 0  ]]; then
+    echo ""
+    echo "${TEST_DESC} SUCCEEDED"
+
+    exit 0
+  else
+    echo "FAILED test(s):"
+    FAILED_TEST_LOGS=($FAILED_TEST_LOGS)
+    FAIL_COUNTER=0
+    for TEST_NAME in ${FAILED_TESTS}; do
+      echo "  ${TEST_DESC} (Log @: ${FAILED_TEST_LOGS[${FAIL_COUNTER}]})"
+      ((FAIL_COUNTER++))
+    done
+
+    echo ""
+    die "${TEST_DESC} FAILED"
+  fi
+}
+
+configure_android_workspace() {
+  # Modify the WORKSPACE file.
+  # Note: This is workaround. This should be done by bazel.
+  if grep -q '^android_sdk_repository' WORKSPACE && grep -q '^android_ndk_repository' WORKSPACE; then
+    echo "You probably have your WORKSPACE file setup for Android."
+  else
+    if [ -z "${ANDROID_API_LEVEL}" -o -z "${ANDROID_BUILD_TOOLS_VERSION}" ] || \
+        [ -z "${ANDROID_SDK_HOME}" -o -z "${ANDROID_NDK_HOME}" ]; then
+      echo "ERROR: Your WORKSPACE file does not seems to have proper android"
+      echo "       configuration and not all the environment variables expected"
+      echo "       inside ci_build android docker container are set."
+      echo "       Please configure it manually. See: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/android/test/README.md"
+    else
+      cat << EOF >> WORKSPACE
+android_sdk_repository(
+    name = "androidsdk",
+    api_level = ${ANDROID_API_LEVEL},
+    build_tools_version = "${ANDROID_BUILD_TOOLS_VERSION}",
+    path = "${ANDROID_SDK_HOME}",
+)
+
+android_ndk_repository(
+    name="androidndk",
+    path="${ANDROID_NDK_HOME}",
+    api_level=21)
+EOF
+    fi
+  fi
+}

tensorflow/tools/ci_build/builds/check_system_libs.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Checks that the options mentioned in syslibs_configure.bzl are consistent with
+# the valid options in workspace.bzl
+# Expects the tensorflow source folder as the first argument
+
+import glob
+import os
+import sys
+
+tf_source_path = sys.argv[1]
+
+syslibs_configure_path = os.path.join(tf_source_path, 'third_party',
+                                      'systemlibs', 'syslibs_configure.bzl')
+workspace0_path = os.path.join(tf_source_path, 'tensorflow', 'workspace0.bzl')
+workspace_glob = os.path.join(tf_source_path, 'tensorflow', 'workspace*.bzl')
+third_party_path = os.path.join(tf_source_path, 'third_party')
+third_party_glob = os.path.join(third_party_path, '*', 'workspace.bzl')
+
+if not os.path.isdir(tf_source_path):
+  raise ValueError('The path to the TensorFlow source must be passed as'
+                   ' the first argument')
+
+if not os.path.isfile(syslibs_configure_path):
+  raise ValueError('Could not find syslibs_configure.bzl at %s' %
+                   syslibs_configure_path)
+
+if not os.path.isfile(workspace0_path):
+  raise ValueError('Could not find workspace0.bzl at %s' % workspace0_path)
+
+def extract_valid_libs(filepath):
+  """Evaluate syslibs_configure.bzl, return the VALID_LIBS set from that file."""
+
+  # Stub only
+  def repository_rule(**kwargs):  # pylint: disable=unused-variable
+    del kwargs
+
+  # Populates VALID_LIBS
+  with open(filepath, 'r') as f:
+    f_globals = {'repository_rule': repository_rule}
+    f_locals = {}
+    exec(f.read(), f_globals, f_locals)  # pylint: disable=exec-used
+
+  return set(f_locals['VALID_LIBS'])
+
+
+def extract_system_builds(filepath):
+  """Extract the 'name' argument of all rules with a system_build_file argument."""
+  lib_names = []
+  system_build_files = []
+  current_name = None
+  with open(filepath, 'r') as f:
+    for line in f:
+      line = line.strip()
+      if line.startswith('name = '):
+        current_name = line[7:-1].strip('"')
+      elif line.startswith('system_build_file = '):
+        lib_names.append(current_name)
+        # Split at '=' to extract rhs, then extract value between quotes
+        system_build_spec = line.split('=')[-1].split('"')[1]
+        assert system_build_spec.startswith('//')
+        system_build_files.append(system_build_spec[2:].replace(':', os.sep))
+  return lib_names, system_build_files
+
+
+syslibs = extract_valid_libs(syslibs_configure_path)
+
+syslibs_from_workspace = set()
+system_build_files_from_workspace = []
+for current_path in glob.glob(workspace_glob) + glob.glob(third_party_glob):
+  cur_lib_names, build_files = extract_system_builds(current_path)
+  syslibs_from_workspace.update(cur_lib_names)
+  system_build_files_from_workspace.extend(build_files)
+
+missing_build_files = [
+    file for file in system_build_files_from_workspace
+    if not os.path.isfile(os.path.join(tf_source_path, file))
+]
+
+has_error = False
+
+if missing_build_files:
+  has_error = True
+  print('Missing system build files: ' + ', '.join(missing_build_files))
+
+if syslibs != syslibs_from_workspace:
+  has_error = True
+  # Libs present in workspace files but not in the allowlist
+  missing_syslibs = syslibs_from_workspace - syslibs
+  if missing_syslibs:
+    libs = ', '.join(sorted(missing_syslibs))
+    print('Libs missing from syslibs_configure: ' + libs)
+  # Libs present in the allow list but not in workspace files
+  additional_syslibs = syslibs - syslibs_from_workspace
+  if additional_syslibs:
+    libs = ', '.join(sorted(additional_syslibs))
+    print('Libs missing in workspace (or superfluous in syslibs_configure): ' +
+          libs)
+
+sys.exit(1 if has_error else 0)

tensorflow/tools/ci_build/builds/cmake.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+
+# Determine the number of cores, for parallel make.
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+if [[ -z ${N_JOBS} ]]; then
+  # The Linux way didn't work. Try the Mac way.
+  N_JOBS=$(sysctl -n hw.ncpu)
+fi
+if [[ -z ${N_JOBS} ]]; then
+  N_JOBS=1
+  echo ""
+  echo "WARNING: Failed to determine the number of CPU cores. "\
+"Will use --jobs=1 for make."
+fi
+
+echo ""
+echo "make will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+
+# Run TensorFlow cmake build.
+# Clean up, because certain modules, e.g., highwayhash, seem to be sensitive
+# to state.
+rm -rf build
+
+mkdir -p build
+pushd build
+
+cmake -DCMAKE_BUILD_TYPE=Release ../tensorflow/contrib/cmake
+make --jobs=${N_JOBS} all
+
+popd

tensorflow/tools/ci_build/builds/configured

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# This script is a wrapper to run any build inside the docker container
+# when running ci_build.sh. It's purpose is to automate the call of ./configure.
+# Yes, this script is a workaround of a workaround.
+#
+# Usage: configured <CONTAINER_TYPE> [--disable-gcp] <COMMAND>
+#
+# The optional flag --disable-gcp disabled support for Google Cloud Platform
+# (GCP) in the builds.
+
+set -e
+
+CONTAINER_TYPE=$( echo "$1" | tr '[:upper:]' '[:lower:]' )
+shift 1
+
+COMMAND=("$@")
+
+export CI_BUILD_PYTHON="${CI_BUILD_PYTHON:-python}"
+export PYTHON_BIN_PATH="${PYTHON_BIN_PATH:-$(which ${CI_BUILD_PYTHON})}"
+# XLA currently does not build under Android, so disable it for now.
+if [[ "${CONTAINER_TYPE}" == 'android' ]]; then
+  export TF_ENABLE_XLA=0
+fi
+
+if [[ "${CONTAINER_TYPE}" == 'rocm' ]]; then
+  export TF_NEED_CLANG=0
+fi
+
+pushd "${CI_TENSORFLOW_SUBMODULE_PATH:-.}"
+yes "" | $PYTHON_BIN_PATH configure.py
+popd
+
+# Gather and print build information
+SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
+${SCRIPT_DIR}/print_build_info.sh ${CONTAINER_TYPE} ${COMMAND[@]}
+
+${COMMAND[@]}

tensorflow/tools/ci_build/builds/docker_cpu_pip.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+set -x
+
+cd bazel_pip
+virtualenv --system-site-packages --python=python .env
+source .env/bin/activate
+pip --version
+pip install portpicker
+pip install *.whl
+
+# Install bazelisk
+rm -rf ~/bin/bazel
+mkdir ~/bin/bazel
+wget --no-verbose -O "~/bin/bazel" \
+    "https://github.com/bazelbuild/bazelisk/releases/download/v1.3.0/bazelisk-linux-amd64"
+chmod u+x "~/bin/bazel"
+if [[ ! ":$PATH:" =~ :"~"/bin/?: ]]; then
+  PATH="~/bin:$PATH"
+fi
+which bazel
+bazel version
+
+# Use default configuration
+yes "" | python configure.py
+
+PIP_TEST_ROOT=pip_test_root
+mkdir -p ${PIP_TEST_ROOT}
+ln -s $(pwd)/tensorflow ${PIP_TEST_ROOT}/tensorflow
+bazel --output_base=/tmp test --define=no_tensorflow_py_deps=true \
+      --test_lang_filters=py \
+      --build_tests_only \
+      -k \
+      --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-no_pip,-nopip,-gpu,-tpu \
+      --test_size_filters=small,medium \
+      --test_timeout 300,450,1200,3600 \
+      --test_output=errors \
+      -- //${PIP_TEST_ROOT}/tensorflow/python/... \
+      -//${PIP_TEST_ROOT}/tensorflow/python/compiler/xla:xla_test \
+      -//${PIP_TEST_ROOT}/tensorflow/python/distribute:parameter_server_strategy_test \
+      -//${PIP_TEST_ROOT}/tensorflow/python/client:virtual_gpu_test \
+      -//${PIP_TEST_ROOT}/tensorflow/python:collective_ops_gpu_test
+# The above tests are excluded because they seem to require a GPU.
+# TODO(yifeif): Investigate and potentially add an unconditional 'gpu' tag.

tensorflow/tools/ci_build/builds/docker_test.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Build and test TensorFlow docker images.
+# The tests include Python unit tests-on-install and tutorial tests.
+#
+# Usage: docker_test.sh <IMAGE_TYPE> <TAG> <WHL_PATH>
+# Arguments:
+#   IMAGE_TYPE : Type of the image: (CPU|GPU|ROCM)
+#   TAG        : Docker image tag
+#   WHL_PATH   : Path to the whl file to be installed inside the docker image
+#
+#   e.g.: docker_test.sh CPU someone/tensorflow:0.8.0 pip_test/whl/tensorflow-0.8.0-cp27-none-linux_x86_64.whl
+#
+
+# Helper functions
+# Exit after a failure
+die() {
+  echo $@
+  exit 1
+}
+
+# Convert to lower case
+to_lower () {
+  echo "$1" | tr '[:upper:]' '[:lower:]'
+}
+
+
+# Helper function to traverse directories up until given file is found.
+function upsearch () {
+  test / == "$PWD" && return || \
+      test -e "$1" && echo "$PWD" && return || \
+      cd .. && upsearch "$1"
+}
+
+
+# Verify command line argument
+if [[ $# != "3" ]]; then
+  die "Usage: $(basename $0) <IMAGE_TYPE> <TAG> <WHL_PATH>"
+fi
+IMAGE_TYPE=$(to_lower "$1")
+DOCKER_IMG_TAG=$2
+WHL_PATH=$3
+
+# Verify image type
+if [[ "${IMAGE_TYPE}" == "cpu" ]]; then
+  DOCKERFILE="tensorflow/tools/docker/Dockerfile"
+elif [[ "${IMAGE_TYPE}" == "gpu" ]]; then
+  DOCKERFILE="tensorflow/tools/docker/Dockerfile.gpu"
+elif [[ "${IMAGE_TYPE}" == "rocm" ]]; then
+  DOCKERFILE="tensorflow/tools/docker/Dockerfile.rocm"
+else
+  die "Unrecognized image type: $1"
+fi
+
+# Verify docker binary existence
+if [[ -z $(which docker) ]]; then
+  die "FAILED: docker binary unavailable"
+fi
+
+# Locate the base directory
+BASE_DIR=$(upsearch "${DOCKERFILE}")
+if [[ -z "${BASE_DIR}" ]]; then
+  die "FAILED: Unable to find the base directory where the dockerfile "\
+"${DOCKERFILE} resides"
+fi
+echo "Base directory: ${BASE_DIR}"
+
+pushd ${BASE_DIR} > /dev/null
+
+# Build docker image
+DOCKERFILE_PATH="${BASE_DIR}/${DOCKERFILE}"
+DOCKERFILE_DIR="$(dirname ${DOCKERFILE_PATH})"
+
+# Check to make sure that the whl file exists
+test -f ${WHL_PATH} || \
+    die "whl file does not exist: ${WHL_PATH}"
+
+TMP_WHL_DIR="${DOCKERFILE_DIR}/whl"
+mkdir -p "${TMP_WHL_DIR}"
+cp "${WHL_PATH}" "${TMP_WHL_DIR}/" || \
+    die "FAILED to copy whl file from ${WHL_PATH} to ${TMP_WHL_DIR}/"
+
+docker build -t "${DOCKER_IMG_TAG}" -f "${DOCKERFILE_PATH}" \
+"${DOCKERFILE_DIR}" || \
+    die "FAILED to build docker image from Dockerfile ${DOCKERFILE_PATH}"
+
+# Clean up
+rm -rf "${TMP_WHL_DIR}" || \
+    die "Failed to remove temporary directory ${TMP_WHL_DIR}"
+
+
+# Add extra params for cuda devices and libraries for GPU container.
+if [ "${IMAGE_TYPE}" == "gpu" ]; then
+  devices=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}')
+  libs=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}')
+  GPU_EXTRA_PARAMS="${devices} ${libs}"
+elif [ "${IMAGE_TYPE}" == "rocm" ]; then
+  ROCM_EXTRA_PARAMS="--device=/dev/kfd --device=/dev/dri --group-add video \
+  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size 16G"
+else
+  GPU_EXTRA_PARAMS=""
+  ROCM_EXTRA_PARAMS=""
+fi
+
+# Run docker image with source directory mapped
+docker run -v ${BASE_DIR}:/tensorflow-src -w /tensorflow-src \
+${GPU_EXTRA_PARAMS} ${ROCM_EXTRA_PARAMS} \
+"${DOCKER_IMG_TAG}" \
+/bin/bash -c "tensorflow/tools/ci_build/builds/run_pip_tests.sh && "\
+"tensorflow/tools/ci_build/builds/test_tutorials.sh && "\
+"tensorflow/tools/ci_build/builds/integration_tests.sh"
+
+RESULT=$?
+
+popd > /dev/null
+if [[ ${RESULT} == 0 ]]; then
+  echo "SUCCESS: Built and tested docker image: ${DOCKER_IMG_TAG}"
+else
+  die "FAILED to build and test docker image: ${DOCKER_IMG_TAG}"
+fi

tensorflow/tools/ci_build/builds/integration_tests.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# This script runs integration tests on the TensorFlow source code
+# using a pip installation.
+#
+# Usage: integration_tests.sh [--virtualenv]
+#
+# If the flag --virtualenv is set, the script will use "python" as the Python
+# binary path. Otherwise, it will use tools/python_bin_path.sh to determine
+# the Python binary path.
+#
+# This script obeys the following environment variables (if exists):
+#   TF_BUILD_INTEG_TEST_DENYLIST: Force skipping of specified integration tests
+#       listed in INTEG_TESTS below.
+#
+
+# List of all integration tests to run, separated by spaces
+INTEG_TESTS="ffmpeg_lib"
+
+if [[ -z "${TF_BUILD_INTEG_TEST_DENYLIST}" ]]; then
+  TF_BUILD_INTEG_TEST_DENYLIST=""
+fi
+echo ""
+echo "=== Integration Tests ==="
+echo "TF_BUILD_INTEG_TEST_DENYLIST = \"${TF_BUILD_INTEG_TEST_DENYLIST}\""
+
+# Timeout (in seconds) for each integration test
+TIMEOUT=1800
+
+INTEG_TEST_ROOT="$(mktemp -d)"
+LOGS_DIR=pip_test/integration_tests/logs
+
+# Current script directory
+SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
+source "${SCRIPT_DIR}/builds_common.sh"
+
+# Helper functions
+cleanup() {
+  rm -rf $INTEG_TEST_ROOT
+}
+
+
+die() {
+  echo $@
+  cleanup
+  exit 1
+}
+
+
+# Determine the binary path for "timeout"
+TIMEOUT_BIN="timeout"
+if [[ -z "$(which ${TIMEOUT_BIN})" ]]; then
+  TIMEOUT_BIN="gtimeout"
+  if [[ -z "$(which ${TIMEOUT_BIN})" ]]; then
+    die "Unable to locate binary path for timeout command"
+  fi
+fi
+echo "Binary path for timeout: \"$(which ${TIMEOUT_BIN})\""
+
+# Avoid permission issues outside Docker containers
+umask 000
+
+mkdir -p "${LOGS_DIR}" || die "Failed to create logs directory"
+mkdir -p "${INTEG_TEST_ROOT}" || die "Failed to create test directory"
+
+if [[ "$1" == "--virtualenv" ]]; then
+  PYTHON_BIN_PATH="$(which python)"
+else
+  source tools/python_bin_path.sh
+fi
+
+if [[ -z "${PYTHON_BIN_PATH}" ]]; then
+  die "PYTHON_BIN_PATH was not provided. If this is not virtualenv, "\
+"did you run configure?"
+else
+  echo "Binary path for python: \"$PYTHON_BIN_PATH\""
+fi
+
+# Determine the TensorFlow installation path
+# pushd/popd avoids importing TensorFlow from the source directory.
+pushd /tmp > /dev/null
+TF_INSTALL_PATH=$(dirname \
+    $("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; print(tf.__file__)"))
+popd > /dev/null
+
+echo "Detected TensorFlow installation path: ${TF_INSTALL_PATH}"
+
+TEST_DIR="pip_test/integration"
+mkdir -p "${TEST_DIR}" || \
+    die "Failed to create test directory: ${TEST_DIR}"
+
+# -----------------------------------------------------------
+# ffmpeg_lib_test
+test_ffmpeg_lib() {
+  # If FFmpeg is not installed then run a test that assumes it is not installed.
+  if [[ -z "$(which ffmpeg)" ]]; then
+    bazel test tensorflow/contrib/ffmpeg/default:ffmpeg_lib_uninstalled_test
+    return $?
+  else
+    bazel test tensorflow/contrib/ffmpeg/default:ffmpeg_lib_installed_test \
+        tensorflow/contrib/ffmpeg:decode_audio_op_test \
+        tensorflow/contrib/ffmpeg:encode_audio_op_test
+    return $?
+  fi
+}
+
+
+# Run the integration tests
+test_runner "integration test-on-install" \
+    "${INTEG_TESTS}" "${TF_BUILD_INTEG_TEST_DENYLIST}" "${LOGS_DIR}"

tensorflow/tools/ci_build/builds/nightly_release_smoke_test.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Pip install TensorFlow and run basic test on the pip package.
+
+set -e
+set -x
+
+# Get size check function
+source tensorflow/tools/ci_build/release/common.sh
+
+function run_smoke_test() {
+
+  if [[ -z "${WHL_NAME}" ]]; then
+    echo "TF WHL path not given, unable to install and test."
+    exit 1
+  fi
+
+  # Upload the PIP package if whl test passes.
+  if [ ${IN_VENV} -eq 0 ]; then
+    VENV_TMP_DIR=$(mktemp -d)
+
+    ${PYTHON_BIN_PATH} -m pip install virtualenv
+
+    ${PYTHON_BIN_PATH} -m virtualenv -p ${PYTHON_BIN_PATH} "${VENV_TMP_DIR}" || \
+        die "FAILED: Unable to create virtualenv"
+
+    source "${VENV_TMP_DIR}/bin/activate" || \
+        die "FAILED: Unable to activate virtualenv "
+  fi
+
+  # install tensorflow
+  python -m pip install ${WHL_NAME} || \
+      die "pip install (forcing to reinstall tensorflow) FAILED"
+      echo "Successfully installed pip package ${WHL_NAME}"
+
+  # Test TensorflowFlow imports
+  test_tf_imports
+
+  # Test TensorFlow whl file size
+  test_tf_whl_size ${WHL_NAME}
+
+  RESULT=$?
+
+  # Upload the PIP package if whl test passes.
+  if [ ${IN_VENV} -eq 0 ]; then
+    # Deactivate from virtualenv.
+    deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
+    sudo rm -rf "${KOKORO_GFILE_DIR}/venv"
+  fi
+
+  return $RESULT
+}
+
+function test_tf_imports() {
+  TMP_DIR=$(mktemp -d)
+  pushd "${TMP_DIR}"
+
+  # test for basic import and perform tf.add operation.
+  RET_VAL=$(python -c "import tensorflow as tf; t1=tf.constant([1,2,3,4]); t2=tf.constant([5,6,7,8]); print(tf.add(t1,t2).shape)")
+  if ! [[ ${RET_VAL} == *'(4,)'* ]]; then
+    echo "Unexpected return value: ${RET_VALUE}"
+    echo "PIP test on virtualenv FAILED, will not upload ${WHL_NAME} package."
+     return 1
+  fi
+
+  # test basic keras is available
+  RET_VAL=$(python -c "import tensorflow as tf; print(tf.keras.__name__)")
+  if ! [[ ${RET_VAL} == *'keras.api'* ]]; then
+    echo "Unexpected return value: ${RET_VALUE}"
+    echo "PIP test on virtualenv FAILED, will not upload ${WHL_NAME} package."
+    return 1
+  fi
+
+  # similar test for estimator
+  RET_VAL=$(python -c "import tensorflow as tf; print(tf.estimator.__name__)")
+  if ! [[ ${RET_VAL} == *'tensorflow_estimator.python.estimator.api._v2.estimator'* ]]; then
+    echo "Unexpected return value: ${RET_VALUE}"
+    echo "PIP test on virtualenv FAILED, will not upload ${WHL_NAME} package."
+    return 1
+  fi
+
+  RESULT=$?
+
+  # TODO(b/210940071): Debug import order. Ignore returned errors.
+  set +e
+  python -m pip install "google-cloud-bigquery<=3.3.1"
+  $(python -c "from google.cloud.bigquery_v2 import types; import tensorflow")
+  $(python -c "import tensorflow; from google.cloud.bigquery_v2 import types")
+  set -e
+
+  popd
+  return $RESULT
+}
+
+###########################################################################
+# Main
+###########################################################################
+
+IN_VENV=$(python -c 'import sys; print("1" if sys.version_info.major == 3 and sys.prefix != sys.base_prefix else "0")')
+WHL_NAME=${1}
+run_smoke_test

tensorflow/tools/ci_build/builds/pip_new.sh

@@ -0,0 +1,784 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Build the Python PIP installation package for TensorFlow and install
+# the package.
+#
+# Usage:
+#   pip_new.sh
+#
+# Required step(s):
+#   Run configure.py prior to running this script.
+#
+# Required environment variable(s):
+#   CONTAINER_TYPE:      (CPU | GPU)
+#   OS_TYPE:             (UBUNTU | MACOS)
+#   TF_PYTHON_VERSION:   ( python3.6 | python3.7 | python3.8 )
+#   TF_BUILD_FLAGS:      Bazel build flags.
+#                          e.g. TF_BUILD_FLAGS="--config=opt"
+#   TF_TEST_FLAGS:       Bazel test flags.
+#                          e.g. TF_TEST_FLAGS="--verbose_failures=true \
+#                               --build_tests_only --test_output=errors"
+#   TF_TEST_FILTER_TAGS: Filtering tags for bazel tests. More specifically,
+#                        input tags for `--test_filter_tags` flag.
+#                          e.g. TF_TEST_FILTER_TAGS="no_pip,-nomac,no_oss"
+#   TF_TEST_TARGETS:     Bazel test targets.
+#                          e.g. TF_TEST_TARGETS="//tensorflow/... \
+#                               -//tensorflow/contrib/... \
+#                               -//tensorflow/python/..."
+#   IS_NIGHTLY:          Nightly run flag.
+#                          e.g. IS_NIGHTLY=1  # nightly runs
+#                          e.g. IS_NIGHTLY=0  # non-nightly runs
+#
+# Optional environment variables. If provided, overwrites any default values.
+#   TF_PIP_TESTS:        PIP tests to run. If NOT specified, skips all tests.
+#                          e.g. TF_PIP_TESTS="test_pip_virtualenv_clean \
+#                               test_pip_virtualenv_clean \
+#                               test_pip_virtualenv_oss_serial"
+#   TF_PROJECT_NAME:     Name of the project. This string will be pass onto
+#                        the wheel file name. For nightly builds, it will be
+#                        overwritten to 'tf_nightly'. For gpu builds, '_gpu'
+#                        will be appended.
+#                          e.g. TF_PROJECT_NAME="tensorflow"
+#                          e.g. TF_PROJECT_NAME="tf_nightly_gpu"
+#   TF_PIP_TEST_ROOT:    Root directory for building and testing pip pkgs.
+#                          e.g. TF_PIP_TEST_ROOT="pip_test"
+#   TF_BUILD_BOTH_GPU_PACKAGES:    (1 | 0)
+#                                  1 will build both tensorflow (w/ gpu support)
+#                                  and tensorflow-gpu pip package. Will
+#                                  automatically handle adding/removing of _gpu
+#                                  suffix depending on what project name was
+#                                  passed. Only work for Ubuntu.
+#   TF_BUILD_BOTH_CPU_PACKAGES:    (1 | 0)
+#                                  1 will build both tensorflow (no gpu support)
+#                                  and tensorflow-cpu pip package. Will
+#                                  automatically handle adding/removing of _cpu
+#                                  suffix depending on what project name was
+#                                  passed. Only work for MacOS
+#   AUDITWHEEL_TARGET_PLAT:    Manylinux platform tag that is to be used for
+#                              tagging the linux wheel files. By default, it is
+#                              set to `manylinux2010` . For manylinux2014
+#                              builds, change to `manylinux2014`.
+#
+# To-be-deprecated variable(s).
+#   GIT_TAG_OVERRIDE:    Values for `--git_tag_override`. This flag gets passed
+#                        in as `--action_env` for bazel build and tests.
+#   TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES:
+#                        Additional pip packages to be installed.
+#                        Caveat: pip version needs to be checked prior.
+#
+# ==============================================================================
+
+# set bash options
+set -e
+set -x
+
+###########################################################################
+# General helper function(s)
+###########################################################################
+
+# Strip leading and trailing whitespaces
+str_strip () {
+  echo -e "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'
+}
+
+# Convert string to all lower case
+lowercase() {
+  if [[ -z "${1}" ]]; then
+    die "Nothing to convert to lowercase. No argument given."
+  fi
+  echo "${1}" | tr '[:upper:]' '[:lower:]'
+}
+
+check_global_vars() {
+  # Check container type
+  if ! [[ ${CONTAINER_TYPE} == "cpu" ]] && \
+     ! [[ ${CONTAINER_TYPE} == "rocm" ]] && \
+     ! [[ ${CONTAINER_TYPE} == "gpu" ]]; then
+    die "Error: Provided CONTAINER_TYPE \"${CONTAINER_TYPE}\" "\
+        "is not supported."
+  fi
+  # Check OS type
+  if ! [[ ${OS_TYPE} == "ubuntu" ]] && \
+     ! [[ ${OS_TYPE} == "macos" ]]; then
+    die "Error: Provided OS_TYPE \"${OS_TYPE}\" is not supported."
+  fi
+  # Check build flags
+  if [[ -z ${TF_BUILD_FLAGS} ]]; then
+    die "Error: TF_BUILD_FLAGS is not specified."
+  fi
+  # Check test flags
+  if [[ -z ${TF_TEST_FLAGS} ]]; then
+    die "Error: TF_TEST_FLAGS is not specified."
+  fi
+  # Check test filter tags
+  if [[ -z ${TF_TEST_FILTER_TAGS} ]]; then
+    die "Error: TF_TEST_FILTER_TAGS is not specified."
+  fi
+  # Check test targets
+  if [[ -z ${TF_TEST_TARGETS} ]]; then
+    die "Error: TF_TEST_TARGETS is not specified."
+  fi
+  # Check nightly status
+  if [[ -z ${IS_NIGHTLY} ]]; then
+    die "Error: IS_NIGHTLY is not specified."
+  fi
+}
+
+add_test_filter_tag() {
+  EMPTY=""
+  while true; do
+    FILTER="${1:$EMPTY}"
+    if ! [[ $TF_TEST_FILTER_TAGS == *"${FILTER}"* ]]; then
+      TF_TEST_FILTER_TAGS="${FILTER},${TF_TEST_FILTER_TAGS}"
+    fi
+    shift
+    if [[ -z "${1}" ]]; then
+      break
+    fi
+  done
+}
+
+remove_test_filter_tag() {
+  EMPTY=""
+  while true; do
+    FILTER="${1:$EMPTY}"
+    TF_TEST_FILTER_TAGS="$(echo ${TF_TEST_FILTER_TAGS} | sed -e 's/^'${FILTER}',//g' -e 's/,'${FILTER}'//g')"
+    shift
+    if [[ -z "${1}" ]]; then
+      break
+    fi
+  done
+}
+
+# Clean up bazel build & test flags with proper configuration.
+update_bazel_flags() {
+  # Add git tag override flag if necessary.
+  GIT_TAG_STR=" --action_env=GIT_TAG_OVERRIDE"
+  if [[ -z "${GIT_TAG_OVERRIDE}" ]] && \
+    ! [[ ${TF_BUILD_FLAGS} = *${GIT_TAG_STR}* ]]; then
+    TF_BUILD_FLAGS+="${GIT_TAG_STR}"
+  fi
+  # Clean up whitespaces
+  TF_BUILD_FLAGS=$(str_strip "${TF_BUILD_FLAGS}")
+  TF_TEST_FLAGS=$(str_strip "${TF_TEST_FLAGS}")
+  # Cleaned bazel flags
+  echo "Bazel build flags (cleaned):\n" "${TF_BUILD_FLAGS}"
+  echo "Bazel test flags (cleaned):\n" "${TF_TEST_FLAGS}"
+}
+
+update_test_filter_tags() {
+  # Add test filter tags
+  # This script is for validating built PIP packages. Add pip tags.
+  add_test_filter_tag -no_pip -nopip
+  # MacOS filter tags
+  if [[ ${OS_TYPE} == "macos" ]]; then
+    remove_test_filter_tag nomac no_mac mac_excluded
+    add_test_filter_tag -nomac -no_mac -mac_excluded
+  fi
+  echo "Final test filter tags: ${TF_TEST_FILTER_TAGS}"
+}
+
+# Check currently running python and pip version
+check_python_pip_version() {
+  # Check if only the major version of python is provided by the user.
+  MAJOR_VER_ONLY=0
+  if [[ ${#PYTHON_VER} -lt 9 ]]; then
+    # User only provided major version (e.g. 'python3' instead of 'python3.7')
+    MAJOR_VER_ONLY=1
+  fi
+
+  # Retrieve only the version number of the user requested python.
+  PYTHON_VER_REQUESTED=${PYTHON_VER:6:3}
+  echo "PYTHON_VER_REQUESTED: ${PYTHON_VER_REQUESTED}"
+
+  # Retrieve only the version numbers of the python & pip in use currently.
+  PYTHON_VER_IN_USE=$(python --version 2>&1)
+  PYTHON_VER_IN_USE=${PYTHON_VER_IN_USE:7:3}
+  PIP_VER_IN_USE=$(${PIP_BIN_PATH} --version)
+  PIP_VER_IN_USE=${PIP_VER_IN_USE:${#PIP_VER_IN_USE}-4:3}
+
+  # If only major versions are applied, drop minor versions.
+  if [[ $MAJOR_VER_ONLY == 1 ]]; then
+    PYTHON_VER_IN_USE=${PYTHON_VER_IN_USE:0:1}
+    PIP_VER_IN_USE=${PIP_VER_IN_USE:0:1}
+  fi
+
+  # Check if all versions match.
+  echo -e "User requested python version: '${PYTHON_VER_REQUESTED}'\n" \
+    "Detected python version in use: '${PYTHON_VER_IN_USE}'\n"\
+    "Detected pip version in use: '${PIP_VER_IN_USE}'"
+  if ! [[ $PYTHON_VER_REQUESTED == $PYTHON_VER_IN_USE ]]; then
+    die "Error: Mismatch in python versions detected."
+  else:
+    echo "Python and PIP versions in use match the requested."
+  fi
+}
+
+# Write an entry to the sponge key-value store for this job.
+write_to_sponge() {
+  # The location of the key-value CSV file sponge imports.
+  TF_SPONGE_CSV="${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv"
+  echo "$1","$2" >> "${TF_SPONGE_CSV}"
+}
+
+
+###########################################################################
+# Setup: directories, local/global variables
+###########################################################################
+
+# Script directory and source necessary files.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/builds_common.sh"
+
+# Required global variables
+# Checks on values for these vars are done in "Build TF PIP Package" section.
+CONTAINER_TYPE=$(lowercase "${CONTAINER_TYPE}")
+OS_TYPE=$(lowercase "${OS_TYPE}")
+PYTHON_VER=$(lowercase "${TF_PYTHON_VERSION}")
+
+# Python bin path
+if [[ -z "$PYTHON_BIN_PATH" ]]; then
+  die "Error: PYTHON_BIN_PATH was not provided. Did you run configure?"
+fi
+
+# Set optional environment variables; set to default in case not user defined.
+DEFAULT_PIP_TESTS="" # Do not run any tests by default
+DEFAULT_PROJECT_NAME="tensorflow"
+DEFAULT_PIP_TEST_ROOT="pip_test"
+DEFAULT_BUILD_BOTH_GPU_PACKAGES=0
+DEFAULT_BUILD_BOTH_CPU_PACKAGES=0
+DEFAULT_AUDITWHEEL_TARGET_PLAT="manylinux2010"
+# Take in optional global variables
+PIP_TESTS=${TF_PIP_TESTS:-$DEFAULT_PIP_TESTS}
+PROJECT_NAME=${TF_PROJECT_NAME:-$DEFAULT_PROJECT_NAME}
+PIP_TEST_ROOT=${TF_PIP_TEST_ROOT:-$DEFAULT_PIP_TEST_ROOT}
+BUILD_BOTH_GPU_PACKAGES=${TF_BUILD_BOTH_GPU_PACKAGES:-$DEFAULT_BUILD_BOTH_GPU_PACKAGES}
+BUILD_BOTH_CPU_PACKAGES=${TF_BUILD_BOTH_CPU_PACKAGES:-$DEFAULT_BUILD_BOTH_CPU_PACKAGES}
+AUDITWHEEL_TARGET_PLAT=${TF_AUDITWHEEL_TARGET_PLAT:-$DEFAULT_AUDITWHEEL_TARGET_PLAT}
+
+# Override breaking change in setuptools v60 (https://github.com/pypa/setuptools/pull/2896)
+export SETUPTOOLS_USE_DISTUTILS=stdlib
+
+# Local variables
+PIP_WHL_DIR="${KOKORO_ARTIFACTS_DIR}/tensorflow/${PIP_TEST_ROOT}/whl"
+mkdir -p "${PIP_WHL_DIR}"
+PIP_WHL_DIR=$(realpath "${PIP_WHL_DIR}") # Get absolute path
+WHL_PATH=""
+# Determine the major.minor versions of python being used (e.g., 3.7).
+# Useful for determining the directory of the local pip installation.
+PY_MAJOR_MINOR_VER=$(${PYTHON_BIN_PATH} -c "print(__import__('sys').version)" 2>&1 | awk '{ print $1 }' | head -n 1 | cut -d. -f1-2)
+
+if [[ -z "${PY_MAJOR_MINOR_VER}" ]]; then
+  die "ERROR: Unable to determine the major.minor version of Python."
+fi
+echo "Python binary path to be used in PIP install: ${PYTHON_BIN_PATH} "\
+"(Major.Minor version: ${PY_MAJOR_MINOR_VER})"
+PYTHON_BIN_PATH_INIT=${PYTHON_BIN_PATH}
+PIP_BIN_PATH="$(which pip${PY_MAJOR_MINOR_VER})"
+
+# PIP packages
+INSTALL_EXTRA_PIP_PACKAGES="h5py portpicker scipy scikit-learn ${TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES}"
+
+###########################################################################
+# Build TF PIP Package
+###########################################################################
+
+# First remove any already existing binaries for a clean start and test.
+if [[ -d ${PIP_TEST_ROOT} ]]; then
+  echo "Test root directory ${PIP_TEST_ROOT} already exists. Deleting it."
+  sudo rm -rf ${PIP_TEST_ROOT}
+fi
+
+# Check that global variables are properly set.
+check_global_vars
+
+# Check if in a virtualenv and exit if yes.
+# TODO(rameshsampath): Python 3.10 has pip conflicts when using global env, so build in virtualenv
+# Once confirmed to work, run builds for all python env in a virtualenv
+if [[ "x${PY_MAJOR_MINOR_VER}x" != "x3.10x" ]]; then
+  IN_VENV=$(python -c 'import sys; print("1" if sys.version_info.major == 3 and sys.prefix != sys.base_prefix else "0")')
+  if [[ "$IN_VENV" == "1" ]]; then
+    echo "It appears that we are already in a virtualenv. Deactivating..."
+    deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
+  fi
+fi
+
+# Obtain the path to python binary as written by ./configure if it was run.
+if [[ -e tools/python_bin_path.sh ]]; then
+  source tools/python_bin_path.sh
+fi
+# Assume PYTHON_BIN_PATH is exported by the script above or the caller.
+if [[ -z "$PYTHON_BIN_PATH" ]]; then
+  die "PYTHON_BIN_PATH was not provided. Did you run configure?"
+fi
+
+if [[ "$IS_NIGHTLY" == 1 ]]; then
+  ${PYTHON_BIN_PATH} -m pip install tb-nightly
+else
+  ${PYTHON_BIN_PATH} -m pip install tensorboard
+fi
+
+if [[ "x${PY_MAJOR_MINOR_VER}x" == "x3.8x" ]]; then
+  ${PYTHON_BIN_PATH} -m pip uninstall -y protobuf
+  ${PYTHON_BIN_PATH} -m pip install "protobuf < 4"
+fi
+
+# Bazel build the file.
+PIP_BUILD_TARGET="//tensorflow/tools/pip_package:build_pip_package"
+# Clean bazel cache.
+bazel clean
+# Clean up and update bazel flags
+update_bazel_flags
+# Build. This outputs the file `build_pip_package`.
+bazel build \
+  --action_env=PYTHON_BIN_PATH=${PYTHON_BIN_PATH} \
+  ${TF_BUILD_FLAGS} \
+  ${PIP_BUILD_TARGET} \
+  || die "Error: Bazel build failed for target: '${PIP_BUILD_TARGET}'"
+
+###########################################################################
+# Test function(s)
+###########################################################################
+
+test_pip_virtualenv() {
+  # Get args
+  WHL_PATH=$1
+  shift
+  VENV_DIR_NAME=$1
+  shift
+  TEST_TYPE_FLAG=$1
+
+  # Check test type args
+  if ! [[ ${TEST_TYPE_FLAG} == "--oss_serial" ]] && \
+     ! [[ ${TEST_TYPE_FLAG} == "--clean" ]] && \
+     ! [[ ${TEST_TYPE_FLAG} == "" ]]; then
+     die "Error: Wrong test type given. TEST_TYPE_FLAG=${TEST_TYPE_FLAG}"
+  fi
+
+  # Create virtualenv directory for test
+  VENV_DIR="${PIP_TEST_ROOT}/${VENV_DIR_NAME}"
+
+  # Activate virtualenv
+  create_activate_virtualenv ${TEST_TYPE_FLAG} ${VENV_DIR}
+  # Install TF with pip
+  TIME_START=$SECONDS
+  install_tensorflow_pip "${WHL_PATH}"
+  TIME_ELAPSED=$(($SECONDS - $TIME_START))
+  echo "Time elapsed installing tensorflow = ${TIME_ELAPSED} seconds"
+
+  # cd to a temporary directory to avoid picking up Python files in the source
+  # tree.
+  TMP_DIR=$(mktemp -d)
+  pushd "${TMP_DIR}"
+
+  # Run a quick check on tensorflow installation.
+  RET_VAL=$(python -c "import tensorflow as tf; t1=tf.constant([1,2,3,4]); t2=tf.constant([5,6,7,8]); print(tf.add(t1,t2).shape)")
+
+  # Return to original directory. Remove temp dirs.
+  popd
+  sudo rm -rf "${TMP_DIR}"
+
+  # Check result to see if tensorflow is properly installed.
+  if ! [[ ${RET_VAL} == *'(4,)'* ]]; then
+    echo "PIP test on virtualenv (non-clean) FAILED"
+    return 1
+  fi
+
+  # Install extra pip packages, if specified.
+  for PACKAGE in ${INSTALL_EXTRA_PIP_PACKAGES}; do
+    echo "Installing extra pip package required by test-on-install: ${PACKAGE}"
+
+    ${PIP_BIN_PATH} install ${PACKAGE}
+    if [[ $? != 0 ]]; then
+      echo "${PIP_BIN_PATH} install ${PACKAGE} FAILED."
+      deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
+      return 1
+    fi
+  done
+
+  # Run bazel test.
+  run_test_with_bazel ${TEST_TYPE_FLAG}
+  RESULT=$?
+
+  # Deactivate from virtualenv.
+  deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
+  sudo rm -rf "${VENV_DIR}"
+
+  return $RESULT
+}
+
+###########################################################################
+# Test helper function(s)
+###########################################################################
+
+create_activate_virtualenv() {
+  VIRTUALENV_FLAGS="--system-site-packages"
+  if [[ "${1}" == "--clean" ]]; then
+    VIRTUALENV_FLAGS=""
+    shift
+  elif [[ "${1}" == "--oss_serial" ]]; then
+    shift
+  fi
+
+  VIRTUALENV_DIR="${1}"
+  if [[ -d "${VIRTUALENV_DIR}" ]]; then
+    if sudo rm -rf "${VIRTUALENV_DIR}"
+    then
+      echo "Removed existing virtualenv directory: ${VIRTUALENV_DIR}"
+    else
+      die "Failed to remove existing virtualenv directory: ${VIRTUALENV_DIR}"
+    fi
+  fi
+
+  if mkdir -p "${VIRTUALENV_DIR}"
+  then
+    echo "Created virtualenv directory: ${VIRTUALENV_DIR}"
+  else
+    die "FAILED to create virtualenv directory: ${VIRTUALENV_DIR}"
+  fi
+
+  # Use the virtualenv from the default python version (i.e., python-virtualenv)
+  # to create the virtualenv directory for testing. Use the -p flag to specify
+  # the python version inside the to-be-created virtualenv directory.
+  ${PYTHON_BIN_PATH_INIT} -m virtualenv -p ${PYTHON_BIN_PATH_INIT} ${VIRTUALENV_FLAGS} ${VIRTUALENV_DIR} || \
+    ${PYTHON_BIN_PATH_INIT} -m venv ${VIRTUALENV_DIR} || \
+    die "FAILED: Unable to create virtualenv"
+
+  source "${VIRTUALENV_DIR}/bin/activate" || \
+    die "FAILED: Unable to activate virtualenv in ${VIRTUALENV_DIR}"
+
+  # Update .tf_configure.bazelrc with venv python path for bazel test.
+  PYTHON_BIN_PATH="$(which python)"
+  yes "" | ./configure
+}
+
+install_tensorflow_pip() {
+  if [[ -z "${1}" ]]; then
+    die "Please provide a proper wheel file path."
+  fi
+
+  # Set path to pip.
+  PIP_BIN_PATH="${PYTHON_BIN_PATH} -m pip"
+
+  # Print python and pip bin paths
+  echo "PYTHON_BIN_PATH to be used to install the .whl: ${PYTHON_BIN_PATH}"
+  echo "PIP_BIN_PATH to be used to install the .whl: ${PIP_BIN_PATH}"
+
+  # Upgrade pip so it supports tags such as cp27mu, manylinux2010 etc.
+  echo "Upgrade pip in virtualenv"
+
+  # NOTE: pip install --upgrade pip leads to a documented TLS issue for
+  # some versions in python
+  curl https://bootstrap.pypa.io/get-pip.py | ${PYTHON_BIN_PATH} || \
+    die "Error: pip install (get-pip.py) FAILED"
+
+  # Check that requested python version matches configured one.
+  check_python_pip_version
+
+  # setuptools v60.0.0 introduced a breaking change on how distutils is linked
+  # https://github.com/pypa/setuptools/blob/main/CHANGES.rst#v6000
+  ${PIP_BIN_PATH} install --upgrade "setuptools" || \
+    die "Error: setuptools install, upgrade FAILED"
+
+  # Force tensorflow reinstallation. Otherwise it may not get installed from
+  # last build if it had the same version number as previous build.
+  PIP_FLAGS="--upgrade --force-reinstall"
+  ${PIP_BIN_PATH} install ${PIP_FLAGS} ${WHL_PATH} || \
+    die "pip install (forcing to reinstall tensorflow) FAILED"
+  echo "Successfully installed pip package ${WHL_PATH}"
+
+  # Install the future package in the virtualenv. Installing it in user system
+  # packages does not appear to port it over when creating a virtualenv.
+  #   ImportError: No module named builtins
+  ${PIP_BIN_PATH} install --upgrade "future>=0.17.1" || \
+    die "Error: future install, upgrade FAILED"
+
+  # Install the gast package in the virtualenv. Installing it in user system
+  # packages does not appear to port it over when creating a virtualenv.
+  ${PIP_BIN_PATH} install --upgrade "gast==0.4.0" || \
+    die "Error: gast install, upgrade FAILED"
+
+}
+
+run_test_with_bazel() {
+  IS_OSS_SERIAL=0
+  if [[ "${1}" == "--oss_serial" ]]; then
+    IS_OSS_SERIAL=1
+  fi
+  TF_GPU_COUNT=${TF_GPU_COUNT:-4}
+
+  # PIP tests should have a "different" path. Different than the one we place
+  # virtualenv, because we are deleting and recreating it here.
+  PIP_TEST_PREFIX=bazel_pip
+  TEST_ROOT=$(pwd)/${PIP_TEST_PREFIX}
+  sudo rm -rf $TEST_ROOT
+  mkdir -p $TEST_ROOT
+  ln -s $(pwd)/tensorflow $TEST_ROOT/tensorflow
+
+  if [[ "${IS_OSS_SERIAL}" == "1" ]]; then
+    remove_test_filter_tag -no_oss
+    remove_test_filter_tag -oss_excluded
+    remove_test_filter_tag -oss_serial
+    add_test_filter_tag oss_serial
+  else
+    add_test_filter_tag -oss_serial
+  fi
+
+  # Clean the bazel cache
+  bazel clean
+  # Clean up flags before running bazel commands
+  update_bazel_flags
+  # Clean up and update test filter tags
+  update_test_filter_tags
+
+  # Figure out how many concurrent tests we can run and do run the tests.
+  BAZEL_PARALLEL_TEST_FLAGS=""
+  if [[ $CONTAINER_TYPE == "gpu" ]] || [[ $CONTAINER_TYPE == "rocm" ]]; then
+    # Number of test threads is the number of GPU cards available.
+    if [[ $OS_TYPE == "macos" ]]; then
+      BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=1"
+    else
+      BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=${TF_GPU_COUNT} \
+        --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute"
+    fi
+  else
+    # Number of test threads is the number of physical CPUs.
+    if [[ $OS_TYPE == "macos" ]]; then
+      BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=$(sysctl -n hw.ncpu)"
+    else
+      BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=$(grep -c ^processor /proc/cpuinfo)"
+    fi
+  fi
+
+  if [[ ${IS_OSS_SERIAL} == 1 ]]; then
+    BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=1"
+  fi
+
+  TEST_TARGETS_SYMLINK=""
+  for TARGET in ${TF_TEST_TARGETS[@]}; do
+    TARGET_NEW=$(echo ${TARGET} | sed -e "s/\/\//\/\/${PIP_TEST_PREFIX}\//g")
+    TEST_TARGETS_SYMLINK+="${TARGET_NEW} "
+  done
+  echo "Test targets (symlink): ${TEST_TARGETS_SYMLINK}"
+
+  # Run the test.
+  bazel test --build_tests_only ${TF_TEST_FLAGS} ${BAZEL_PARALLEL_TEST_FLAGS} --test_tag_filters=${TF_TEST_FILTER_TAGS} -k -- ${TEST_TARGETS_SYMLINK}
+
+  unlink ${TEST_ROOT}/tensorflow
+}
+
+run_all_tests() {
+  WHL_PATH=$1
+
+  if [[ -z "${PIP_TESTS}" ]]; then
+    echo "No test was specified to run. Skipping all tests."
+    return 0
+  fi
+  FAIL_COUNTER=0
+  PASS_COUNTER=0
+  for TEST in ${PIP_TESTS[@]}; do
+
+    # Run tests.
+    case "${TEST}" in
+    "test_pip_virtualenv_clean")
+      test_pip_virtualenv ${WHL_PATH} venv_clean --clean
+      ;;
+    "test_pip_virtualenv_non_clean")
+      test_pip_virtualenv ${WHL_PATH} venv
+      ;;
+    "test_pip_virtualenv_oss_serial")
+      test_pip_virtualenv ${WHL_PATH} venv_oss --oss_serial
+      ;;
+    *)
+      die "No matching test ${TEST} was found. Stopping test."
+      ;;
+    esac
+
+    # Check and update the results.
+    RETVAL=$?
+
+    # Update results counter
+    if [ ${RETVAL} -eq 0 ]; then
+      echo "Test (${TEST}) PASSED. (PASS COUNTER: ${PASS_COUNTER})"
+      PASS_COUNTER=$(($PASS_COUNTER+1))
+    else
+      echo "Test (${TEST}) FAILED. (FAIL COUNTER: ${FAIL_COUNTER})"
+      FAIL_COUNTER=$(($FAIL_COUNTER+1))
+    fi
+  done
+  printf "${PASS_COUNTER} PASSED | ${FAIL_COUNTER} FAILED"
+  if [[ "${FAIL_COUNTER}" == "0" ]]; then
+    printf "PIP tests ${COLOR_GREEN}PASSED${COLOR_NC}\n"
+    return 0
+  else:
+    printf "PIP tests ${COLOR_RED}FAILED${COLOR_NC}\n"
+    return 1
+  fi
+}
+
+###########################################################################
+# Build TF PIP Wheel file
+###########################################################################
+
+# Update the build flags for building whl.
+# Flags: GPU, OS, tf_nightly, project name
+GPU_FLAG=""
+NIGHTLY_FLAG=""
+
+# TF Nightly flag
+if [[ "$IS_NIGHTLY" == 1 ]]; then
+  # If 'nightly' is not specified in the project name already, then add.
+  if ! [[ $PROJECT_NAME == *"nightly"* ]]; then
+    echo "WARNING: IS_NIGHTLY=${IS_NIGHTLY} but requested project name \
+    (PROJECT_NAME=${PROJECT_NAME}) does not include 'nightly' string. \
+    Renaming it to 'tf_nightly'."
+    PROJECT_NAME="tf_nightly"
+  fi
+  NIGHTLY_FLAG="--nightly_flag"
+fi
+
+# CPU / GPU flag
+if [[ ${CONTAINER_TYPE} == "gpu" ]]; then
+  GPU_FLAG="--gpu"
+  if ! [[ $PROJECT_NAME == *"gpu"* ]]; then
+    # Only update PROJECT_NAME if TF_PROJECT_NAME is not set
+    if [[ -z "${TF_PROJECT_NAME}" ]]; then
+      echo "WARNING: GPU is specified but requested project name (PROJECT_NAME=${PROJECT_NAME}) \
+      does not include 'gpu'. Appending '_gpu' to the project name."
+      PROJECT_NAME="${PROJECT_NAME}_gpu"
+    fi
+  fi
+fi
+
+if [[ ${CONTAINER_TYPE} == "rocm" ]]; then
+  GPU_FLAG="--rocm"
+  if ! [[ $PROJECT_NAME == *"rocm"* ]]; then
+    # Only update PROJECT_NAME if TF_PROJECT_NAME is not set
+    if [[ -z "${TF_PROJECT_NAME}" ]]; then
+      echo "WARNING: ROCM is specified but requested project name (PROJECT_NAME=${PROJECT_NAME}) \
+      does not include 'rocm'. Appending '_rocm' to the project name."
+      PROJECT_NAME="${PROJECT_NAME}_rocm"
+    fi
+  fi
+fi
+
+./bazel-bin/tensorflow/tools/pip_package/build_pip_package ${PIP_WHL_DIR} ${GPU_FLAG} ${NIGHTLY_FLAG} "--project_name" ${PROJECT_NAME} || die "build_pip_package FAILED"
+
+PY_DOTLESS_MAJOR_MINOR_VER=$(echo $PY_MAJOR_MINOR_VER | tr -d '.')
+if [[ $PY_DOTLESS_MAJOR_MINOR_VER == "2" ]]; then
+  PY_DOTLESS_MAJOR_MINOR_VER="27"
+fi
+
+# Set wheel path and verify that there is only one .whl file in the path.
+WHL_PATH=$(ls "${PIP_WHL_DIR}"/"${PROJECT_NAME}"-*"${PY_DOTLESS_MAJOR_MINOR_VER}"*"${PY_DOTLESS_MAJOR_MINOR_VER}"*.whl)
+if [[ $(echo "${WHL_PATH}" | wc -w) -ne 1 ]]; then
+  echo "ERROR: Failed to find exactly one built TensorFlow .whl file in "\
+  "directory: ${PIP_WHL_DIR}"
+fi
+
+WHL_DIR=$(dirname "${WHL_PATH}")
+
+# Print the size of the wheel file and log to sponge.
+WHL_SIZE=$(ls -l ${WHL_PATH} | awk '{print $5}')
+echo "Size of the PIP wheel file built: ${WHL_SIZE}"
+write_to_sponge TF_INFO_WHL_SIZE ${WHL_SIZE}
+
+# Build the other GPU package.
+if [[ "$BUILD_BOTH_GPU_PACKAGES" -eq "1" ]] || [[ "$BUILD_BOTH_CPU_PACKAGES" -eq "1" ]]; then
+
+  if [[ "$BUILD_BOTH_GPU_PACKAGES" -eq "1" ]] && [[ "$BUILD_BOTH_CPU_PACKAGES" -eq "1" ]]; then
+    die "ERROR: TF_BUILD_BOTH_GPU_PACKAGES and TF_BUILD_BOTH_GPU_PACKAGES cannot both be set. No additional package will be built."
+  fi
+
+  echo "====================================="
+  if [[ "$BUILD_BOTH_GPU_PACKAGES" -eq "1" ]]; then
+    if ! [[ ${OS_TYPE} == "ubuntu" ]]; then
+      die "ERROR: pip_new.sh only support building both GPU wheels on ubuntu."
+    fi
+    echo "Building the other GPU pip package."
+    PROJECT_SUFFIX="gpu"
+  else
+    if ! [[ ${OS_TYPE} == "macos" ]]; then
+      die "ERROR: pip_new.sh only support building both CPU wheels on macos."
+    fi
+    echo "Building the other CPU pip package."
+    PROJECT_SUFFIX="cpu"
+  fi
+
+  # Check container type
+  if ! [[ ${CONTAINER_TYPE} == ${PROJECT_SUFFIX} ]]; then
+    die "Error: CONTAINER_TYPE needs to be \"${PROJECT_SUFFIX}\" to build ${PROJECT_SUFFIX} packages. Got"\
+        "\"${CONTAINER_TYPE}\" instead."
+  fi
+  if [[ "$PROJECT_NAME" == *_${PROJECT_SUFFIX} ]]; then
+    NEW_PROJECT_NAME=${PROJECT_NAME}"_${PROJECT_SUFFIX}"
+  else
+    NEW_PROJECT_NAME="${PROJECT_NAME}_${PROJECT_SUFFIX}"
+  fi
+  echo "The given ${PROJECT_SUFFIX} \$PROJECT_NAME is ${PROJECT_NAME}. The additional ${PROJECT_SUFFIX}"\
+  "pip package will have project name ${NEW_PROJECT_NAME}."
+
+  ./bazel-bin/tensorflow/tools/pip_package/build_pip_package ${PIP_WHL_DIR} ${GPU_FLAG} ${NIGHTLY_FLAG} "--project_name" ${NEW_PROJECT_NAME} || die "build_pip_package FAILED"
+fi
+
+# On MacOS we not have to rename the wheel because it is generated with the
+# wrong tag.
+if [[ ${OS_TYPE} == "macos" ]] ; then
+  for WHL_PATH in $(ls ${PIP_WHL_DIR}/*macosx_10_15_x86_64.whl); do
+    # change 10_15 to 10_14
+    NEW_WHL_PATH=${WHL_PATH/macosx_10_15/macosx_10_14}
+    mv ${WHL_PATH} ${NEW_WHL_PATH}
+  done
+
+  # Also change global WHL_PATH. Ignore above shadow and everywhere else
+  NEW_WHL_PATH=${WHL_PATH/macosx_10_15/macosx_10_14}
+  WHL_PATH=${NEW_WHL_PATH}
+fi
+
+# Run tests (if any is specified).
+run_all_tests ${WHL_PATH}
+
+
+if [[ ${OS_TYPE} == "ubuntu" ]] && \
+   ! [[ ${CONTAINER_TYPE} == "rocm" ]] ; then
+  # Avoid Python3.6 abnormality by installing auditwheel here.
+  # TODO(rameshsampath) - Cleanup and remove the need for auditwheel install
+  # Python 3.10 requires auditwheel > 2 and its already installed in common.sh
+  if [[ $PY_MAJOR_MINOR_VER -ne "3.10" ]]; then
+    set +e
+    pip3 show auditwheel || "pip${PY_MAJOR_MINOR_VER}" show auditwheel
+    # For tagging wheels as manylinux2014, auditwheel needs to >= 3.0.0
+    pip3 install auditwheel==3.3.1 || "pip${PY_MAJOR_MINOR_VER}" install auditwheel==3.3.1
+    sudo pip3 install auditwheel==3.3.1 || \
+      sudo "pip${PY_MAJOR_MINOR_VER}" install auditwheel==3.3.1
+    set -e
+  fi
+  auditwheel --version
+
+  for WHL_PATH in $(ls ${PIP_WHL_DIR}/*.whl); do
+    # Repair the wheels for cpu manylinux2010/manylinux2014
+    echo "auditwheel repairing ${WHL_PATH}"
+    auditwheel repair --plat ${AUDITWHEEL_TARGET_PLAT}_$(uname -m) -w "${WHL_DIR}" "${WHL_PATH}"
+
+    if [[ $(ls ${WHL_DIR} | grep ${AUDITWHEEL_TARGET_PLAT} | wc -l) == 1 ]] ; then
+      WHL_PATH=${WHL_DIR}/$(ls ${WHL_DIR} | grep ${AUDITWHEEL_TARGET_PLAT})
+      echo "Repaired ${AUDITWHEEL_TARGET_PLAT} wheel file at: ${WHL_PATH}"
+    else
+      die "WARNING: Cannot find repaired wheel."
+    fi
+  done
+fi
+echo "EOF: Successfully ran pip_new.sh"

tensorflow/tools/ci_build/builds/print_build_info.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+# Print build info, including info related to the machine, OS, build tools
+# and TensorFlow source code. This can be used by build tools such as Jenkins.
+# All info is printed on a single line, in JSON format, to workaround the
+# limitation of Jenkins Description Setter Plugin that multi-line regex is
+# not supported.
+#
+# Usage:
+#   print_build_info.sh (CONTAINER_TYPE) (COMMAND)
+#     e.g.,
+#       print_build_info.sh GPU bazel test -c opt --config=cuda //tensorflow/...
+
+# Information about the command
+CONTAINER_TYPE=$1
+shift 1
+COMMAND=("$@")
+
+# Information about machine and OS
+OS=$(uname)
+KERNEL=$(uname -r)
+
+ARCH=$(uname -p)
+PROCESSOR=$(grep "model name" /proc/cpuinfo | head -1 | awk '{print substr($0, index($0, $4))}')
+PROCESSOR_COUNT=$(grep "model name" /proc/cpuinfo | wc -l)
+
+MEM_TOTAL=$(grep MemTotal /proc/meminfo | awk '{print $2, $3}')
+SWAP_TOTAL=$(grep SwapTotal /proc/meminfo | awk '{print $2, $3}')
+
+# Information about build tools
+if [[ ! -z $(which bazel) ]]; then
+  BAZEL_VER=$(bazel version | head -1)
+fi
+
+if [[ ! -z $(which javac) ]]; then
+  JAVA_VER=$(javac -version 2>&1 | awk '{print $2}')
+fi
+
+if [[ ! -z $(which python) ]]; then
+  PYTHON_VER=$(python -V 2>&1 | awk '{print $2}')
+fi
+
+if [[ ! -z $(which g++) ]]; then
+  GPP_VER=$(g++ --version | head -1)
+fi
+
+if [[ ! -z $(which swig) ]]; then
+  SWIG_VER=$(swig -version > /dev/null | grep -m 1 . | awk '{print $3}')
+fi
+
+# Information about TensorFlow source
+TF_FETCH_URL=$(git config --get remote.origin.url)
+TF_HEAD=$(git rev-parse HEAD)
+
+# NVIDIA & CUDA info
+NVIDIA_DRIVER_VER=""
+if [[ -f /proc/driver/nvidia/version ]]; then
+  NVIDIA_DRIVER_VER=$(head -1 /proc/driver/nvidia/version | awk '{print $(NF-6)}')
+fi
+
+CUDA_DEVICE_COUNT="0"
+CUDA_DEVICE_NAMES=""
+if [[ ! -z $(which nvidia-debugdump) ]]; then
+  CUDA_DEVICE_COUNT=$(nvidia-debugdump -l | grep "^Found [0-9]*.*device.*" | awk '{print $2}')
+  CUDA_DEVICE_NAMES=$(nvidia-debugdump -l | grep "Device name:.*" | awk '{print substr($0, index($0,\
+ $3)) ","}')
+fi
+
+CUDA_TOOLKIT_VER=""
+if [[ ! -z $(which nvcc) ]]; then
+  CUDA_TOOLKIT_VER=$(nvcc -V | grep release | awk '{print $(NF)}')
+fi
+
+# Print info
+echo "TF_BUILD_INFO = {"\
+"container_type: \"${CONTAINER_TYPE}\", "\
+"command: \"${COMMAND[*]}\", "\
+"source_HEAD: \"${TF_HEAD}\", "\
+"source_remote_origin: \"${TF_FETCH_URL}\", "\
+"OS: \"${OS}\", "\
+"kernel: \"${KERNEL}\", "\
+"architecture: \"${ARCH}\", "\
+"processor: \"${PROCESSOR}\", "\
+"processor_count: \"${PROCESSOR_COUNT}\", "\
+"memory_total: \"${MEM_TOTAL}\", "\
+"swap_total: \"${SWAP_TOTAL}\", "\
+"Bazel_version: \"${BAZEL_VER}\", "\
+"Java_version: \"${JAVA_VER}\", "\
+"Python_version: \"${PYTHON_VER}\", "\
+"gpp_version: \"${GPP_VER}\", "\
+"swig_version: \"${SWIG_VER}\", "\
+"NVIDIA_driver_version: \"${NVIDIA_DRIVER_VER}\", "\
+"CUDA_device_count: \"${CUDA_DEVICE_COUNT}\", "\
+"CUDA_device_names: \"${CUDA_DEVICE_NAMES}\", "\
+"CUDA_toolkit_version: \"${CUDA_TOOLKIT_VER}\""\
+"}"

tensorflow/tools/ci_build/builds/py_test_delegate.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# This file runs a Python test file using specified Python binary, and store
+# the test log, along with exit code and elapsed time in a log file.
+#
+# Usage: test_delegate.sh <PYTHON_BIN_PATH> <TEST_PATH> <TEST_LOG>
+
+PYTHON_BIN_PATH=$1
+TEST_PATH=$2
+TEST_LOG=$3
+
+# Current script directory
+SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
+source "${SCRIPT_DIR}/builds_common.sh"
+
+rm -f ${TEST_LOG}
+
+TEST_DIR=$(dirname ${TEST_PATH})
+cd ${TEST_DIR}
+
+START_TIME=$(date +'%s%N')
+
+${PYTHON_BIN_PATH} ${TEST_PATH} >${TEST_LOG} 2>&1
+TEST_EXIT_CODE=$?
+
+END_TIME=$(date +'%s%N')
+
+ELAPSED=$(calc_elapsed_time "${START_TIME}" "${END_TIME}")
+
+echo "${TEST_EXIT_CODE} ${ELAPSED}" >> ${TEST_LOG}

tensorflow/tools/ci_build/builds/run_pip_tests.sh

@@ -0,0 +1,179 @@
+#!/bin/bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+#
+# Run the python unit tests from the source code on the pip installation.
+#
+# Usage:
+#   run_pip_tests.sh [--virtualenv] [--gpu] [--mac] [--oss_serial]
+#
+# If the flag --virtualenv is set, the script will use "python" as the Python
+# binary path. Otherwise, it will use tools/python_bin_path.sh to determine
+# the Python binary path.
+#
+# The --gpu flag informs the script that this is a GPU build, so that the
+# appropriate test denylists can be applied accordingly.
+#
+# The --mac flag informs the script that this is running on mac. Mac does not
+# have flock, so we should skip using parallel_gpu_execute on mac.
+#
+# The --oss_serial flag lets the script run only the py tests with the
+# oss_serial tag, in a serial fashion, i.e., using the bazel flag
+# --local_test_jobs=1
+#
+#   TF_BUILD_APPEND_ARGUMENTS:
+#                      Additional command line arguments for the bazel,
+#                      pip.sh or android.sh command
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/builds_common.sh"
+
+# Process input arguments
+IS_VIRTUALENV=0
+IS_GPU=0
+IS_ROCM=0
+IS_MAC=0
+IS_OSS_SERIAL=0
+while true; do
+  if [[ "$1" == "--virtualenv" ]]; then
+    IS_VIRTUALENV=1
+  elif [[ "$1" == "--gpu" ]]; then
+    IS_GPU=1
+  elif [[ "$1" == "--rocm" ]]; then
+    IS_ROCM=1
+  elif [[ "$1" == "--mac" ]]; then
+    IS_MAC=1
+  elif [[ "$1" == "--oss_serial" ]]; then
+    IS_OSS_SERIAL=1
+  fi
+  shift
+
+  if [[ -z "$1" ]]; then
+    break
+  fi
+done
+
+TF_GPU_COUNT=${TF_GPU_COUNT:-4}
+if [[ ${IS_ROCM} == "1" ]]; then
+  TF_GPU_COUNT=$(lspci|grep 'controller'|grep 'AMD/ATI'|wc -l)
+  TF_TESTS_PER_GPU=1
+  N_TEST_JOBS=$(expr ${TF_GPU_COUNT} \* ${TF_TESTS_PER_GPU})
+fi
+
+# PIP tests should have a "different" path. Different than the one we place
+# virtualenv, because we are deleting and recreating it here.
+PIP_TEST_PREFIX=bazel_pip
+PIP_TEST_ROOT=$(pwd)/${PIP_TEST_PREFIX}
+rm -rf $PIP_TEST_ROOT
+mkdir -p $PIP_TEST_ROOT
+ln -s $(pwd)/tensorflow ${PIP_TEST_ROOT}/tensorflow
+
+# Do not run tests with "no_pip" tag. If running GPU tests, also do not run
+# tests with no_pip_gpu tag.
+PIP_TEST_FILTER_TAG="-no_pip,-no_oss,-oss_excluded,-benchmark-test"
+if [[ ${IS_OSS_SERIAL} == "1" ]]; then
+  PIP_TEST_FILTER_TAG="$(echo "${PIP_TEST_FILTER_TAG}" | sed s/-no_oss//)"
+   PIP_TEST_FILTER_TAG="$(echo "${PIP_TEST_FILTER_TAG}" | sed s/-oss_excluded//)"
+  PIP_TEST_FILTER_TAG="${PIP_TEST_FILTER_TAG},oss_serial"
+else
+  PIP_TEST_FILTER_TAG="${PIP_TEST_FILTER_TAG},-oss_serial"
+fi
+
+if [[ ${IS_GPU} == "1" ]] || [[ ${IS_ROCM} == "1" ]]; then
+  PIP_TEST_FILTER_TAG="-no_gpu,-no_pip_gpu,${PIP_TEST_FILTER_TAG}"
+fi
+if [[ ${IS_ROCM} == "1" ]]; then
+  PIP_TEST_FILTER_TAG="-no_rocm,-no_pip_rocm,${PIP_TEST_FILTER_TAG}"
+fi
+if [[ ${IS_MAC} == "1" ]]; then
+  # TODO(b/122370901): Fix nomac, no_mac inconsistency.
+  PIP_TEST_FILTER_TAG="-nomac,-no_mac,-mac_excluded,${PIP_TEST_FILTER_TAG}"
+fi
+
+# Bazel flags we need for all tests:
+#     define=no_tensorflow_py_deps=true, to skip all test dependencies.
+#     test_lang_filters=py only py tests for pip package testing
+#     TF_BUILD_APPEND_ARGUMENTS any user supplied args.
+BAZEL_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py \
+  --build_tests_only -k --test_tag_filters=${PIP_TEST_FILTER_TAG} \
+  ${TF_BUILD_APPEND_ARGUMENTS} \
+  --test_output=errors"
+
+if [[ ${IS_ROCM} == "1" ]]; then
+  BAZEL_FLAGS="${BAZEL_FLAGS} \
+  --test_timeout 600,900,2400,7200"
+else
+  BAZEL_FLAGS="${BAZEL_FLAGS} \
+  --test_timeout 300,450,1200,3600"
+fi
+
+
+BAZEL_TEST_TARGETS="//${PIP_TEST_PREFIX}/tensorflow/python/..."
+
+# Clean the bazel cache
+bazel clean
+
+# Run configure again, we might be using a different python path, due to
+# virtualenv.
+export TF_NEED_GCP=0
+export TF_NEED_HDFS=0
+
+# Obtain the path to Python binary
+if [[ ${IS_VIRTUALENV} == "1" ]]; then
+  PYTHON_BIN_PATH="$(which python)"
+else
+  source tools/python_bin_path.sh
+  # Assume: PYTHON_BIN_PATH is exported by the script above
+fi
+
+export TF_NEED_CUDA=$IS_GPU
+export TF_NEED_ROCM=$IS_ROCM
+yes "" | ${PYTHON_BIN_PATH} configure.py
+
+# Figure out how many concurrent tests we can run and do run the tests.
+BAZEL_PARALLEL_TEST_FLAGS=""
+if [[ $IS_GPU == 1 ]] || [[ $IS_ROCM == 1 ]]; then
+  # Number of test threads is the number of GPU cards available.
+  if [[ $IS_MAC == 1 ]]; then
+    BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=1"
+  elif [[ $IS_ROCM == 1 ]]; then
+    BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=${N_TEST_JOBS} \
+        --test_env=TF_GPU_COUNT=$TF_GPU_COUNT \
+        --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
+        --test_sharding_strategy=disabled \
+        --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute"
+  else
+    PAR_TEST_JOBS=$TF_GPU_COUNT
+    BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=${TF_GPU_COUNT} \
+        --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute"
+  fi
+else
+  # Number of test threads is the number of physical CPUs.
+  if [[ $IS_MAC == 1 ]]; then
+    BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=$(sysctl -n hw.ncpu)"
+  else
+    BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=$(grep -c ^processor /proc/cpuinfo)"
+  fi
+fi
+
+if [[ ${IS_OSS_SERIAL} == 1 ]]; then
+  BAZEL_PARALLEL_TEST_FLAGS="--local_test_jobs=1"
+fi
+
+# Actually run the tests.
+bazel test ${BAZEL_FLAGS} ${BAZEL_PARALLEL_TEST_FLAGS} -- \
+    ${BAZEL_TEST_TARGETS}

tensorflow/tools/ci_build/builds/test_tutorials.sh

@@ -0,0 +1,272 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# This script tests the tutorials and examples in TensorFlow source code
+# on pip installation. The tutorial scripts are copied from source to a
+# separate directory for testing. The script performs some very basic quality
+# checks on the results, such as thresholding final accuracy, verifying
+# decrement of loss with training, and verifying the existence of saved
+# checkpoints and summaries files.
+#
+# Usage: test_tutorials.sh [--virtualenv]
+#
+# If the flag --virtualenv is set, the script will use "python" as the Python
+# binary path. Otherwise, it will use tools/python_bin_path.sh to determine
+# the Python binary path.
+#
+# This script obeys the following environment variables (if exists):
+#   TUT_TESTS_DENYLIST: Force skipping of specified tutorial tests listed
+#                        in TUT_TESTS below.
+#
+
+# List of all tutorial tests to run, separated by spaces
+TUT_TESTS="mnist_with_summaries word2vec"
+
+if [[ -z "${TUT_TESTS_DENYLIST}" ]]; then
+  TF_BUILD_TUT_TEST_DENYLIST=""
+fi
+echo ""
+echo "=== Testing tutorials ==="
+echo "TF_BUILD_TUT_TEST_DENYLIST = \"${TF_BUILD_TUT_TEST_DENYLIST}\""
+
+# Timeout (in seconds) for each tutorial test
+TIMEOUT=1800
+
+TUT_TEST_ROOT=/tmp/tf_tutorial_test
+TUT_TEST_DATA_DIR=/tmp/tf_tutorial_test_data
+LOGS_DIR=pip_test/tutorial_tests/logs
+
+# Current script directory
+SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
+source "${SCRIPT_DIR}/builds_common.sh"
+
+
+# Determine the binary path for "timeout"
+TIMEOUT_BIN="timeout"
+if [[ -z "$(which ${TIMEOUT_BIN})" ]]; then
+  TIMEOUT_BIN="gtimeout"
+  if [[ -z "$(which ${TIMEOUT_BIN})" ]]; then
+    die "Unable to locate binary path for timeout command"
+  fi
+fi
+echo "Binary path for timeout: \"$(which ${TIMEOUT_BIN})\""
+
+# Avoid permission issues outside Docker containers
+umask 000
+
+mkdir -p "${LOGS_DIR}" || die "Failed to create logs directory"
+mkdir -p "${TUT_TEST_ROOT}" || die "Failed to create test directory"
+
+if [[ "$1" == "--virtualenv" ]]; then
+  PYTHON_BIN_PATH="$(which python)"
+else
+  source tools/python_bin_path.sh
+fi
+
+if [[ -z "${PYTHON_BIN_PATH}" ]]; then
+  die "PYTHON_BIN_PATH was not provided. If this is not virtualenv, "\
+"did you run configure?"
+else
+  echo "Binary path for python: \"$PYTHON_BIN_PATH\""
+fi
+
+# Determine the TensorFlow installation path
+# pushd/popd avoids importing TensorFlow from the source directory.
+pushd /tmp > /dev/null
+TF_INSTALL_PATH=$(dirname \
+    $("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; print(tf.__file__)"))
+popd > /dev/null
+
+echo "Detected TensorFlow installation path: ${TF_INSTALL_PATH}"
+
+TEST_DIR="pip_test/tutorials"
+mkdir -p "${TEST_DIR}" || \
+    die "Failed to create test directory: ${TEST_DIR}"
+
+# Copy folders required by mnist tutorials
+mkdir -p "${TF_INSTALL_PATH}/examples/tutorials"
+cp tensorflow/examples/tutorials/__init__.py \
+    "${TF_INSTALL_PATH}/examples/tutorials/"
+cp -r tensorflow/examples/tutorials/mnist \
+    "${TF_INSTALL_PATH}/examples/tutorials/"
+
+if [[ ! -d "${TF_INSTALL_PATH}/examples/tutorials/mnist" ]]; then
+  die "FAILED: Unable to copy directory required by MNIST tutorials: "\
+"${TF_INSTALL_PATH}/examples/tutorials/mnist"
+fi
+
+
+# -----------------------------------------------------------
+# mnist_softmax
+test_mnist_softmax() {
+  LOG_FILE=$1
+
+  run_in_directory "${TEST_DIR}" "${LOG_FILE}" \
+    tensorflow/examples/tutorials/mnist/mnist_softmax.py \
+    --data_dir="${TUT_TEST_DATA_DIR}/mnist"
+
+  # Check final accuracy
+  FINAL_ACCURACY=$(tail -1 "${LOG_FILE}")
+  if [[ $(python -c "print(${FINAL_ACCURACY}>0.85)") != "True" ]] ||
+     [[ $(python -c "print(${FINAL_ACCURACY}<=1.00)") != "True" ]]; then
+    echo "mnist_softmax accuracy check FAILED: "\
+"FINAL_ACCURACY = ${FINAL_ACCURACY}"
+    return 1
+  fi
+}
+
+
+# -----------------------------------------------------------
+# mnist_with_summaries
+test_mnist_with_summaries() {
+  LOG_FILE=$1
+
+  SUMMARIES_DIR="${TUT_TEST_ROOT}/summaries/mnist"
+  rm -rf "${SUMMARIES_DIR}"
+  # rm -rf "${TEST_DIR}/tensorflow"
+
+  if [[ $? != 0 ]]; then
+    echo "FAILED: unable to remove existing summaries directory: "\
+"${SUMMARIES_DIR}"
+    return 1
+  fi
+
+  run_in_directory "${TEST_DIR}" "${LOG_FILE}" \
+    tensorflow/examples/tutorials/mnist/mnist_with_summaries.py \
+    --data_dir="${TUT_TEST_DATA_DIR}/mnist" --log_dir="${SUMMARIES_DIR}"
+
+  # Verify final accuracy
+  FINAL_ACCURACY=$(grep "Accuracy at step" "${LOG_FILE}" \
+                   | tail -1 | awk '{print $NF}')
+  if [[ $(python -c "print(${FINAL_ACCURACY}>0.85)") != "True" ]] ||
+     [[ $(python -c "print(${FINAL_ACCURACY}<=1.00)") != "True" ]]; then
+    echo "mnist_with_summaries accuracy check FAILED: final accuracy = "\
+"${FINAL_ACCURACY}"
+    return 1
+  fi
+
+  # Verify that the summaries file have been generated
+  if [[ ! -d "${SUMMARIES_DIR}" ]] ||
+     [[ -z $(ls "${SUMMARIES_DIR}") ]]; then
+    echo "FAILED: It appears that no summaries files were generated in "\
+"${SUMMARIES_DIR}"
+    return 1
+  fi
+}
+
+
+# -----------------------------------------------------------
+# cifar10_train
+test_cifar10_train() {
+  LOG_FILE=$1
+
+  rm -rf "${TUT_TEST_ROOT}/cifar10_train"
+  if [[ $? != 0 ]]; then
+    echo "Unable to remove old cifar10_train directory"
+    return 1
+  fi
+
+  run_in_directory "${TEST_DIR}" "${LOG_FILE}" \
+    ${TF_MODELS_DIR}/tutorials/image/cifar10/cifar10_train.py \
+    --data_dir="${TUT_TEST_DATA_DIR}/cifar10" --max_steps=50 \
+    --train_dir="${TUT_TEST_ROOT}/cifar10_train"
+
+  # Verify that final loss is less than initial loss
+  INIT_LOSS=$(grep -o "loss = [0-9\.]*" "${LOG_FILE}" | head -1 | \
+    awk '{print $NF}')
+  FINAL_LOSS=$(grep -o "loss = [0-9\.]*" "${LOG_FILE}" | tail -1 | \
+    awk '{print $NF}')
+
+  if [[ $(python -c "print(${FINAL_LOSS}<${INIT_LOSS})") != "True" ]] ||
+     [[ $(python -c "print(${INIT_LOSS}>=0)") != "True" ]] ||
+     [[ $(python -c "print(${FINAL_LOSS}>=0)") != "True" ]]; then
+    echo "cifar10_train loss check FAILED: "\
+"FINAL_LOSS = ${FINAL_LOSS}; INIT_LOSS = ${INIT_LOSS}"
+    return 1
+  fi
+
+  return 0
+}
+
+
+# -----------------------------------------------------------
+# word2vec_test
+test_word2vec() {
+  LOG_FILE=$1
+
+  run_in_directory "${TEST_DIR}" "${LOG_FILE}" \
+    tensorflow/examples/tutorials/word2vec/word2vec_basic.py
+}
+
+
+# -----------------------------------------------------------
+# ptb_word_lm
+test_ptb_word_lm() {
+  LOG_FILE=$1
+
+  PTB_DATA_URL="http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz"
+
+  DATA_DIR="${TUT_TEST_DATA_DIR}/ptb"
+  if [[ ! -f "${DATA_DIR}/simple-examples/data/ptb.train.txt" ]] || \
+     [[ ! -f "${DATA_DIR}/simple-examples/data/ptb.valid.txt" ]] || \
+     [[ ! -f "${DATA_DIR}/simple-examples/data/ptb.test.txt" ]]; then
+    # Download and extract data
+    echo "Downloading and extracting PTB data from \"${PTB_DATA_URL}\" to "\
+"${DATA_DIR}"
+
+    mkdir -p ${DATA_DIR}
+    pushd ${DATA_DIR} > /dev/null
+    curl --retry 5 --retry-delay 10 -O "${PTB_DATA_URL}" || \
+        die "Failed to download data file for ptb_world_lm tutorial from "\
+"${PTB_DATA_URL}"
+    tar -xzf $(basename "${PTB_DATA_URL}")
+    rm -f $(basename "${PTB_DATA_URL}")
+    popd > /dev/null
+
+    if [[ ! -d "${DATA_DIR}/simple-examples/data" ]]; then
+      echo "FAILED to download and extract data for \"ptb_word_lm\""
+      return 1
+    fi
+  fi
+
+  run_in_directory "${TEST_DIR}" "${LOG_FILE}" \
+    "${TF_MODELS_DIR}/tutorials/rnn/ptb/ptb_word_lm.py" \
+    --data_path="${DATA_DIR}/simple-examples/data" --model test
+
+  if [[ $? != 0 ]]; then
+    echo "Tutorial test \"ptb_word_lm\" FAILED"
+    return 1
+  fi
+
+  # Extract epoch initial and final training perplexities
+  INIT_PERPL=$(grep -o "[0-9\.]* perplexity: [0-9\.]*" "${LOG_FILE}" | head -1 | awk '{print $NF}')
+  FINAL_PERPL=$(grep -o "[0-9\.]* perplexity: [0-9\.]*" "${LOG_FILE}" | tail -1 | awk '{print $NF}')
+
+  echo "INIT_PERPL=${INIT_PERPL}"
+  echo "FINAL_PERPL=${FINAL_PERPL}"
+
+  if [[ $(python -c "print(${FINAL_PERPL}<${INIT_PERPL})") != "True" ]] ||
+     [[ $(python -c "print(${INIT_PERPL}>=0)") != "True" ]] ||
+     [[ $(python -c "print(${FINAL_PERPL}>=0)") != "True" ]]; then
+    echo "ptb_word_lm perplexity check FAILED: "\
+"FINAL_PERPL = ${FINAL_PERPL}; INIT_PERPL = ${INIT_PERPL}"
+    return 1
+  fi
+}
+
+# Run the tutorial tests
+test_runner "tutorial test-on-install" \
+    "${TUT_TESTS}" "${TF_BUILD_TUT_TEST_DENYLIST}" "${LOGS_DIR}"

tensorflow/tools/ci_build/builds/test_user_ops.sh

@@ -0,0 +1,258 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Test user-defined ops against installation of TensorFlow.
+#
+# Usage: test_user_ops.sh [--virtualenv] [--gpu]
+#
+# If the flag --virtualenv is set, the script will use "python" as the Python
+# binary path. Otherwise, it will use tools/python_bin_path.sh to determine
+# the Python binary path.
+#
+# The --gpu flag informs the script that this is a GPU build, so that the
+# appropriate test denylists can be applied accordingly.
+#
+
+echo ""
+echo "=== Testing user ops ==="
+
+# Script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/builds_common.sh"
+
+
+# Process input arguments
+IS_VIRTUALENV=0
+IS_GPU=0
+while true; do
+  if [[ "$1" == "--virtualenv" ]]; then
+    IS_VIRTUALENV=1
+  elif [[ "$1" == "--gpu" ]]; then
+    IS_GPU=1
+  fi
+  shift
+
+  if [[ -z "$1" ]]; then
+    break
+  fi
+done
+
+TMP_DIR=$(mktemp -d)
+mkdir -p "${TMP_DIR}"
+
+cleanup() {
+  rm -rf "${TMP_DIR}"
+}
+
+die() {
+  echo $@
+  cleanup
+  exit 1
+}
+
+
+# Obtain the path to Python binary
+if [[ ${IS_VIRTUALENV} == "1" ]]; then
+  PYTHON_BIN_PATH="$(which python)"
+else
+  source tools/python_bin_path.sh
+  # Assume: PYTHON_BIN_PATH is exported by the script above
+fi
+echo "PYTHON_BIN_PATH: ${PYTHON_BIN_PATH}"
+
+
+pushd "${TMP_DIR}"
+
+# Obtain compilation and linking flags
+TF_CFLAGS=( $("${PYTHON_BIN_PATH}" \
+	      -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )
+TF_LFLAGS=( $("${PYTHON_BIN_PATH}" \
+	      -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') )
+
+if [[ -z "${TF_CFLAGS[*]}" || -z "${TF_LFLAGS[*]}" ]]; then
+  die "FAILED to determine TensorFlow compilation or linking flags"
+else
+  echo "TensorFlow compile flags: ${TF_CFLAGS[*]}"
+  echo "TensorFlow link flags: ${TF_LFLAGS[*]}"
+fi
+
+# Check g++ availability
+GPP_BIN="g++"
+if [[ -z $(which "${GPP_BIN}") ]]; then
+  die "ERROR: ${GPP_BIN} not on path"
+fi
+
+echo ""
+echo "g++ version:"
+"${GPP_BIN}" -v
+echo ""
+
+
+IS_MAC=0
+if [[ $(uname) == "Darwin" ]]; then
+  echo "Detected Mac OS X environment"
+  IS_MAC=1
+fi
+
+EXTRA_GPP_FLAGS=""
+if [[ ${IS_MAC} == "1" ]]; then
+  # Extra flags required on Mac OS X, where dynamic_lookup is not the default
+  # behavior.
+  EXTRA_GPP_FLAGS="${EXTRA_GPP_FLAGS} -undefined dynamic_lookup"
+fi
+
+echo "Extra GPP flag: ${EXTRA_GPP_FLAGS}"
+
+# Input to the user op
+OP_INPUT="[42, 43, 44]"
+
+if [[ ${IS_GPU} == "0" ]]; then
+  echo "Testing user ops in CPU environment"
+
+  # Expected output from user op
+  EXPECTED_OUTPUT="[42, 0, 0]"
+
+  # Locate the op kernel C++ file
+  OP_KERNEL_CC="${SCRIPT_DIR}/user_ops/zero_out_op_kernel_1.cc"
+  OP_KERNEL_CC=$(realpath "${OP_KERNEL_CC}")
+
+  if [[ ! -f "${OP_KERNEL_CC}" ]]; then
+    die "ERROR: Unable to find user-op kernel C++ file at: ${OP_KERNEL_CC}"
+  fi
+
+  # Copy the file to a non-TensorFlow source directory
+  cp "${OP_KERNEL_CC}" ./
+
+  # Compile the op kernel into an .so file
+  SRC_FILE=$(basename "${OP_KERNEL_CC}")
+
+  echo "Compiling user op C++ source file ${SRC_FILE}"
+
+  USER_OP_SO="zero_out.so"
+
+  "${GPP_BIN}" -std=c++11 ${EXTRA_GPP_FLAGS} \
+    -shared "${SRC_FILE}" -o "${USER_OP_SO}" \
+    -fPIC ${TF_CFLAGS[@]} ${TF_LFLAGS[@]}  || \
+    die "g++ compilation of ${SRC_FILE} FAILED"
+
+else
+  echo "Testing user ops in GPU environment"
+
+  # Expected output from user op
+  EXPECTED_OUTPUT="[43, 44, 45]"
+
+  # Check nvcc availability
+  NVCC_BIN="/usr/local/cuda/bin/nvcc"
+  if [[ -z $(which "${NVCC_BIN}") ]]; then
+    die "ERROR: ${NVCC_BIN} not on path"
+  fi
+
+  echo ""
+  echo "nvcc version:"
+  "${NVCC_BIN}" --version
+  echo ""
+
+  OP_KERNEL_CU="${SCRIPT_DIR}/user_ops/cuda_op_kernel.cu.cc"
+  OP_KERNEL_CU=$(realpath "${OP_KERNEL_CU}")
+  if [[ ! -f "${OP_KERNEL_CU}" ]]; then
+    die "ERROR: Unable to find user-op kernel CUDA file at: ${OP_KERNEL_CU}"
+  fi
+
+  OP_KERNEL_CC="${SCRIPT_DIR}/user_ops/cuda_op_kernel.cc"
+  OP_KERNEL_CC=$(realpath "${OP_KERNEL_CC}")
+  if [[ ! -f "${OP_KERNEL_CC}" ]]; then
+    die "ERROR: Unable to find user-op kernel C++ file at: ${OP_KERNEL_CC}"
+  fi
+
+  # Copy the file to a non-TensorFlow source directory
+  cp "${OP_KERNEL_CU}" ./
+  cp "${OP_KERNEL_CC}" ./
+
+  OP_KERNEL_O=$(echo "${OP_KERNEL_CC}" | sed -e 's/\.cc/\.o/')
+  "${NVCC_BIN}" -std=c++11 \
+      -c -o "${OP_KERNEL_O}" "${OP_KERNEL_CU}" \
+      ${TF_CFLAGS[@]} -D GOOGLE_CUDA=1 -x cu -Xcompiler -fPIC || \
+      die "nvcc compilation of ${OP_KERNEL_CC} FAILED"
+
+  CUDA_LIB_DIR="/usr/local/cuda/lib64"
+  if [[ ! -d "${CUDA_LIB_DIR}" ]]; then
+    CUDA_LIB_DIR="/usr/local/cuda/lib"
+  fi
+  if [[ ! -d "${CUDA_LIB_DIR}" ]]; then
+    die "ERROR: Failed to find CUDA library directory at either of "
+"/usr/local/cuda/lib and /usr/local/cuda/lib64"
+  fi
+
+  echo "Found CUDA library directory at: ${CUDA_LIB_DIR}"
+  echo ""
+
+  # USER_OP_SO=$(basename $(echo "${OP_KERNEL_CC}" | sed -e 's/\.cc/\.so/'))
+  USER_OP_SO="add_one.so"
+  "${GPP_BIN}" -std=c++11 ${EXTRA_GPP_FLAGS} \
+      -shared -o "${USER_OP_SO}" "${OP_KERNEL_CC}" \
+      "${OP_KERNEL_O}" ${TF_CFLAGS[@]} -L "${CUDA_LIB_DIR}" ${TF_LFLAGS[@]} \
+      -fPIC -lcudart || \
+      die "g++ compilation of ${OP_KERNEL_CC}" FAILED
+fi
+
+# Try running the op
+USER_OP=$(echo "${USER_OP_SO}" | sed -e 's/\.so//')
+echo "Invoking user op ${USER_OP} defined in file ${USER_OP_SO} "\
+"via pip installation"
+
+function run_op() {
+  local ORIG_OUTPUT=$1
+  local ADDITIONAL_LOG=$2
+
+  # Format OUTPUT for analysis
+  if [[ -z $(echo "${ORIG_OUTPUT}" | grep -o ',') ]]; then
+    if [[ ${IS_MAC} == "1" ]]; then
+      local OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -E -e 's/[ \t]+/,/g')
+    else
+      local OUTPUT=$(echo "${ORIG_OUTPUT}" | sed -r -e 's/[ \t]+/,/g')
+    fi
+  else
+    local OUTPUT="${ORIG_OUTPUT}"
+  fi
+
+  local EQUALS_EXPECTED=$("${PYTHON_BIN_PATH}" -c "print(${OUTPUT} == ${EXPECTED_OUTPUT})")
+
+  if [[ "${EQUALS_EXPECTED}" != "True" ]]; then
+    local ERROR="FAILED: Output from user op (${OUTPUT}) does not match expected "\
+  "output ${EXPECTED_OUTPUT}"${ADDITIONAL_LOG}
+    die ${ERROR}
+  else
+    echo "Output from user op (${OUTPUT}) matches expected output"
+  fi
+}
+
+printf "\nTesting execution of user-defined op under graph mode:\n\n"
+run_op "$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; print(tf.Session('').run(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT})))")"
+
+if [[ "${IS_GPU}" == "0" ]]; then
+  printf "\nTesting execution of user-defined op under eager mode:\n\n"
+  run_op "$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; tf.enable_eager_execution(); print(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT}).numpy())")" " in eager mode"
+else
+  printf "\nSKIPPING the testing of execution of user-defined GPU kernel under eager mode. See b/122972785.\n\n"
+fi
+
+
+popd
+
+cleanup
+
+echo ""
+echo "SUCCESS: Testing of user ops PASSED"

tensorflow/tools/ci_build/builds/user_ops/cuda_op_kernel.cc

@@ -0,0 +1,55 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+
+using namespace tensorflow;
+
+REGISTER_OP("AddOne")
+    .Input("input: int32")
+    .Output("output: int32")
+    .Doc(R"doc(
+Adds 1 to all elements of the tensor.
+
+output: A Tensor.
+  output = input + 1
+)doc");
+
+void AddOneKernelLauncher(const int* in, const int N, int* out);
+
+class AddOneOp : public OpKernel {
+ public:
+  explicit AddOneOp(OpKernelConstruction* context) : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    // Grab the input tensor
+    const Tensor& input_tensor = context->input(0);
+    auto input = input_tensor.flat<int32>();
+
+    // Create an output tensor
+    Tensor* output_tensor = NULL;
+    OP_REQUIRES_OK(context, context->allocate_output(0, input_tensor.shape(),
+                                                     &output_tensor));
+    auto output = output_tensor->template flat<int32>();
+
+    // Set all but the first element of the output tensor to 0.
+    const int N = input.size();
+    // Call the cuda kernel launcher
+    AddOneKernelLauncher(input.data(), N, output.data());
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("AddOne").Device(DEVICE_GPU), AddOneOp);

tensorflow/tools/ci_build/builds/user_ops/cuda_op_kernel.cu.cc

@@ -0,0 +1,33 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+#include "unsupported/Eigen/CXX11/Tensor"  // from @eigen_archive
+#include "tensorflow/core/util/gpu_launch_config.h"
+
+__global__ void AddOneKernel(const int* in, const int N, int* out) {
+  for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N;
+       i += blockDim.x * gridDim.x) {
+    out[i] = in[i] + 1;
+  }
+}
+
+void AddOneKernelLauncher(const int* in, const int N, int* out) {
+  TF_CHECK_OK(::tensorflow::GpuLaunchKernel(AddOneKernel, 32, 256, 0, nullptr,
+                                            in, N, out));
+}
+
+#endif

tensorflow/tools/ci_build/builds/user_ops/zero_out_op_kernel_1.cc

@@ -0,0 +1,62 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/shape_inference.h"
+
+using namespace tensorflow;
+
+REGISTER_OP("ZeroOut")
+    .Input("to_zero: int32")
+    .Output("zeroed: int32")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      c->set_output(0, c->input(0));
+      return OkStatus();
+    })
+    .Doc(R"doc(
+Zeros out all but the first value of a Tensor.
+
+zeroed: A Tensor whose first value is identical to `to_zero`, and 0
+  otherwise.
+)doc");
+
+class ZeroOutOp : public OpKernel {
+ public:
+  explicit ZeroOutOp(OpKernelConstruction* context) : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    // Grab the input tensor
+    const Tensor& input_tensor = context->input(0);
+    auto input = input_tensor.flat<int32>();
+
+    // Create an output tensor
+    Tensor* output_tensor = NULL;
+    OP_REQUIRES_OK(context, context->allocate_output(0, input_tensor.shape(),
+                                                     &output_tensor));
+    auto output = output_tensor->template flat<int32>();
+
+    // Set all but the first element of the output tensor to 0.
+    const int N = input.size();
+    for (int i = 1; i < N; i++) {
+      output(i) = 0;
+    }
+
+    // Preserve the first input value.
+    if (N > 0) output(0) = input(0);
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp);

tensorflow/tools/ci_build/builds/with_the_same_user

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# This script is a wrapper creating the same user inside container as the one
+# running the ci_build.sh outside the container. It also set the home directory
+# for the user inside container to match the same absolute path as the workspace
+# outside of container.
+# We do this so that the bazel running inside container generate symbolic links
+# and user permissions which makes sense outside of container.
+# Do not run this manually. It does not make sense. It is intended to be called
+# by ci_build.sh only.
+
+set -e
+
+COMMAND=("$@")
+
+if [[ $(awk -F= '/^NAME/{print $2}' /etc/os-release) == *"CentOS"* ]]; then
+  ${COMMAND[@]}
+else
+
+  if ! touch /this_is_writable_file_system; then
+    echo "You can't write to your filesystem!"
+    echo "If you are in Docker you should check you do not have too many images" \
+        "with too many files in them. Docker has some issue with it."
+    exit 1
+  else
+    rm /this_is_writable_file_system
+  fi
+
+  if [ -n "${CI_BUILD_USER_FORCE_BADNAME}" ]; then
+    ADDUSER_OPTS="--force-badname"
+  fi
+
+  apt-get install sudo
+
+  getent group "${CI_BUILD_GID}" || addgroup ${ADDUSER_OPTS} --gid "${CI_BUILD_GID}" "${CI_BUILD_GROUP}"
+  getent passwd "${CI_BUILD_UID}" || adduser ${ADDUSER_OPTS} \
+      --gid "${CI_BUILD_GID}" --uid "${CI_BUILD_UID}" \
+      --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \
+      --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}"
+  usermod -a -G sudo "${CI_BUILD_USER}"
+  echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo
+
+  if [[ "${TF_NEED_ROCM}" -eq 1 ]]; then
+    # ROCm requires the video group in order to use the GPU for compute. If it
+    # exists on the host, add it to the container.
+    getent group video || addgroup video && adduser "${CI_BUILD_USER}" video
+  fi
+
+  if [ -e /root/.bazelrc ]; then
+    cp /root/.bazelrc "${CI_BUILD_HOME}/.bazelrc"
+    chown "${CI_BUILD_UID}:${CI_BUILD_GID}" "${CI_BUILD_HOME}/.bazelrc"
+  fi
+
+  sudo -u "#${CI_BUILD_UID}" --preserve-env "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" \
+  "HOME=${CI_BUILD_HOME}" ${COMMAND[@]}
+
+fi