chore: import upstream snapshot with attribution

tensorflow/tools/ci_build/linux/cmake/run.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+# Run TensorFlow cmake build.
+# Clean up, because certain modules, e.g., highwayhash, seem to be sensitive
+# to state.
+rm -rf build
+
+mkdir -p build
+pushd build
+
+cmake -DCMAKE_BUILD_TYPE=Release ../tensorflow/contrib/cmake
+# When building do not use all CPUs due to jobs running out of memory.
+# TODO(gunan): Figure out why we run out of memory in large GCE instances.
+make --jobs 20 tf_python_build_pip_package
+
+virtualenv cmake_test --system-site-packages
+source cmake_test/bin/activate
+
+# For older versions of PIP, remove the ABI tag.
+# TODO(gunan) get rid of this part once pip is upgraded on all test machines.
+WHEEL_FILE_PATH=`ls tf_python/dist/*tensorflow*.whl`
+FIXED_WHEEL_PATH=`echo $WHEEL_FILE_PATH | sed -e s/cp27mu/none/`
+mv $WHEEL_FILE_PATH $FIXED_WHEEL_PATH
+
+# Install the pip package we just built.
+pip install --upgrade $FIXED_WHEEL_PATH
+
+# Run all tests.
+ctest -C Release --output-on-failure -j
+
+# Finalize and go back to the initial directory.
+deactivate
+popd

tensorflow/tools/ci_build/linux/cpu/run_cc_core.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export TF_NEED_ROCM=0
+export CC_OPT_FLAGS='-mavx'
+# Only running cc tests, python version does not matter.
+export PYTHON_BIN_PATH=`which python`
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --test_tag_filters=-no_oss,-oss_excluded,-gpu,-tpu,-benchmark-test --test_lang_filters=cc,java -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --config=opt \
+    --test_output=errors --test_size_filters=small,medium -- \
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...

tensorflow/tools/ci_build/linux/cpu/run_mkl.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+# This script accepts only one parameter: either the word "eigen", or an
+# integer value greater than 0 that is passed to the bazel test command
+# via the OMP_NUM_THREADS action environment variable. If an integer is
+# passed, the script assumes it is running in DNNL mode; the
+# OMP_NUM_THREADS variable is irrelevant in eigen mode.
+
+set -e
+set -x
+
+#using default targets from tensorflow project
+source "./tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh"
+if [[ -z "$DEFAULT_BAZEL_TARGETS" ]]; then
+   DEFAULT_BAZEL_TARGETS="//tensorflow/...  -//tensorflow/compiler/...  -//tensorflow/lite/..."
+else
+   DEFAULT_BAZEL_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/..."
+fi
+echo "DEFAULT_BAZEL_TARGETS: $DEFAULT_BAZEL_TARGETS "
+
+DEFAULT_OMP_NUM_THREADS="10"
+DEFAULT_CONFIG="--config=mkl"
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export PYTHON_BIN_PATH=`which python3`
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Get parameters from command-line rather than from env.
+# Setting OMP_THREADS for low performing benchmarks.
+#   Default value(=core count) degrades performance of some benchmark cases.
+#   Optimal thread count is case specific.
+RE_DIGITS_ONLY="^[0-9]+$"
+MIN_OMP_THREADS=1
+CONFIG=""
+OMPTHREADS=""
+BLOCK_FORMAT=""
+ENABLE_ONEDNN=""
+KMP_BLOCKTIME=""
+if [[ $# -ge 1 ]]; then
+  if [[ "$1" == "eigen" ]]; then
+     echo "uses all default values for eigen"
+  elif [[ "$1" == "threadpool" ]]; then
+    ENABLE_ONEDNN="--action_env=TF_ENABLE_ONEDNN_OPTS=1"
+  elif [[ "$1" =~ ${RE_DIGITS_ONLY} && $1 -ge ${MIN_OMP_THREADS} ]]; then
+    CONFIG="${DEFAULT_CONFIG}"
+    OMPTHREADS="--action_env=OMP_NUM_THREADS=${1}"
+    BLOCK_FORMAT="--action_env=TF_ENABLE_MKL_NATIVE_FORMAT=0"
+    KMP_BLOCKTIME="--test_env=KMP_BLOCKTIME=0"
+  else
+    echo "${1} isn't a valid configuration or"
+    echo "number of OM_NUM_THREADS. Exiting..."
+    exit 1
+  fi
+else  # No parameters were passed in so set default values.
+  CONFIG="${DEFAULT_CONFIG}"
+  OMPTHREADS="--action_env=OMP_NUM_THREADS=${DEFAULT_OMP_NUM_THREADS}"
+  BLOCK_FORMAT="--action_env=TF_ENABLE_MKL_NATIVE_FORMAT=0"
+  KMP_BLOCKTIME="--test_env=KMP_BLOCKTIME=0"
+fi
+
+echo ""
+echo "Bazel will test with CONFIG=${CONFIG}, BLOCK_FORMAT=${BLOCK_FORMAT}, ENABLE_ONEDNN=${ENABLE_ONEDNN}, KMP_BLOCKTIME=${KMP_BLOCKTIME}  and OMPTHREADS=${OMPTHREADS}"
+echo ""
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+# Setting KMP_BLOCKTIME to 0 lets OpenMP threads to sleep right after parallel
+# execution in an MKL primitive. This reduces the effects of an oversubscription
+# of OpenMP threads caused by executing multiple tests concurrently.
+bazel test \
+    --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only \
+    --test_lang_filters=cc,py \
+    -k \
+    --jobs=${N_JOBS} \
+    --test_timeout 300,450,1200,3600 \
+    --build_tests_only \
+    ${CONFIG} \
+    ${KMP_BLOCKTIME} \
+    ${OMPTHREADS} \
+    ${ENABLE_ONEDNN} \
+    ${BLOCK_FORMAT} \
+    --config=opt \
+    --test_output=errors \
+    -- ${DEFAULT_BAZEL_TARGETS}

tensorflow/tools/ci_build/linux/cpu/run_py2_core.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export TF_NEED_ROCM=0
+export CC_OPT_FLAGS='-mavx'
+export PYTHON_BIN_PATH=`which python2`
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --test_tag_filters=-no_oss,-oss_excluded,-no_oss_py2,-oss_serial,-gpu,-tpu,-benchmark-test --test_lang_filters=py -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only --config=opt \
+    --test_output=errors --test_size_filters=small,medium -- \
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...

tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export TF_NEED_ROCM=0
+export CC_OPT_FLAGS='-mavx'
+export PYTHON_BIN_PATH=`which python3`
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --config=opt \
+    --test_size_filters=small,medium --test_output=errors -- \
+    //tensorflow/contrib/...

tensorflow/tools/ci_build/linux/cpu/run_py3_core.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export TF_NEED_ROCM=0
+export CC_OPT_FLAGS='-mavx'
+export PYTHON_BIN_PATH=`which python3`
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test --test_lang_filters=py -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only --config=opt \
+    --test_output=errors --test_size_filters=small,medium -- \
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...

tensorflow/tools/ci_build/linux/gpu/run_cc_core.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export PYTHON_BIN_PATH=`which python3`
+export CC_OPT_FLAGS='-mavx'
+
+export TF_NEED_ROCM=0
+export TF_NEED_CUDA=1
+export TF_CUDA_COMPUTE_CAPABILITIES=3.7
+
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --config=cuda --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-no_gpu,-benchmark-test -k \
+    --test_lang_filters=cc --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 \
+    --build_tests_only --test_output=errors --local_test_jobs=8 --config=opt \
+    --test_size_filters=small,medium \
+    --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute -- \
+    //tensorflow/... -//tensorflow/compiler/...

tensorflow/tools/ci_build/linux/gpu/run_mkl.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+#default config
+DEFAULT_CONFIG="--config=cuda"
+OMPTHREADS="--action_env=OMP_NUM_THREADS=$N_JOBS"
+KMP_BLOCKTIME="--test_env=KMP_BLOCKTIME=0"
+
+#install packages needed
+
+pip install tensorboard
+pip install keras-nightly
+
+pip list
+
+# Run configure.
+export TF_NEED_CUDA=1
+export TF_CUDA_COMPUTE_CAPABILITIES=6.0
+export PYTHON_BIN_PATH=`which python`
+yes "" | TF_NEED_CUDA=1 TF_CUDA_COMPUTE_CAPABILITIES=6.0 $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+# Setting KMP_BLOCKTIME to 0 lets OpenMP threads to sleep right after parallel execution
+# in an MKL primitive. This reduces the effects of an oversubscription of OpenMP threads
+# caused by executing multiple tests concurrently.
+
+ENABLE_ONEDNN=""
+CONFIG=${DEFAULT_CONFIG}
+if [[ $# -ge 1 ]]; then
+  if [[ "$1" == "eigencuda" ]]; then
+     echo "uses all default values for eigen"
+     ENABLE_ONEDNN=""
+  else
+     ENABLE_ONEDNN="--action_env=TF_ENABLE_ONEDNN_OPTS=1"
+  fi
+fi
+
+#using default targets from tensorflow project
+source "./tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh"
+if [[ -z "$DEFAULT_BAZEL_TARGETS" ]]; then
+   DEFAULT_BAZEL_TARGETS="//tensorflow/...  -//tensorflow/compiler/...  -//tensorflow/lite/..."
+else
+   DEFAULT_BAZEL_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/..."
+fi
+echo "DEFAULT_BAZEL_TARGETS: $DEFAULT_BAZEL_TARGETS "
+
+echo ""
+echo "Bazel will test with CONFIG=${CONFIG}, ENABLE_ONEDNN=${ENABLE_ONEDNN}"
+echo ""
+#Bazel test command with two option eigencuda or dnllcuda
+
+bazel test \
+    --test_tag_filters=gpu,-no_gpu,-benchmark-test,-no_gpu_presubmit,-no_cuda11,-v1only,-no_oss,-oss_excluded,-oss_serial \
+    --build_tag_filters=gpu,-no_gpu,-benchark-test,-no_oss,-oss_excluded,-oss_serial,-no_gpu_presubmit,-no_cuda11,-v1only \
+    --test_lang_filters=cc,py \
+    -c opt -k \
+    --test_timeout 300,450,1200,3600 \
+    --build_tests_only \
+    --local_test_jobs=1 \
+    --cache_test_results \
+    ${CONFIG} \
+    ${KMP_BLOCKTIME} \
+    ${OMPTHREADS} \
+    ${ENABLE_ONEDNN} \
+    --test_output=errors \
+    -- ${DEFAULT_BAZEL_TARGETS}

tensorflow/tools/ci_build/linux/gpu/run_py3_core.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export PYTHON_BIN_PATH=`which python3`
+export CC_OPT_FLAGS='-mavx'
+
+export TF_NEED_ROCM=0
+export TF_NEED_CUDA=1
+export TF_CUDA_COMPUTE_CAPABILITIES=3.7
+
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --config=cuda --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-no_gpu,-benchmark-test -k \
+    --test_lang_filters=py --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 \
+    --build_tests_only --test_output=errors --local_test_jobs=8 --config=opt \
+    --test_size_filters=small,medium \
+    --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute -- \
+    //tensorflow/... -//tensorflow/compiler/...

tensorflow/tools/ci_build/linux/mkl/Dockerfile.devel-mkl

@@ -0,0 +1,113 @@
+# Copyright 2026 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+ARG ROOT_CONTAINER_TAG=devel
+ARG ROOT_CONTAINER=tensorflow/tensorflow
+
+FROM ${ROOT_CONTAINER}:${ROOT_CONTAINER_TAG}
+
+LABEL maintainer="Clayne Robison <clayne.b.robison@intel.com>"
+
+# These parameters can be overridden
+ARG PYTHON="python"
+ARG WHL_DIR="/tmp/pip"
+ARG PIP="pip"
+ARG TARGET_PLATFORM="haswell"
+ARG CONFIG_V2_DISABLE=""
+ARG CONFIG_BFLOAT16_BUILD=""
+ARG ENABLE_SECURE_BUILD
+ARG BAZEL_VERSION=""
+ARG ENABLE_DNNL1=""
+ARG ENABLE_HOROVOD=""
+ARG ENABLE_GCC8=""
+ARG OPENMPI_VERSION=""
+ARG OPENMPI_DOWNLOAD_URL=""
+ARG HOROVOD_VERSION=""
+ARG INSTALL_HOROVOD_FROM_COMMIT=""
+ARG BUILD_SSH=""
+ARG TF_NIGHTLY_FLAG=""
+ARG RELEASE_CONTAINER=""
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Upgrade Bazel version if argument is passed
+RUN if [ "${BAZEL_VERSION}" != "" ]; then \
+        curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
+        chmod +x bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
+        ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
+        rm -rf bazel-$BAZEL_VERSION-installer-linux-x86_64.sh; \
+    fi
+
+# Upgrade gcc-8 if argument is passed
+RUN if [ "${ENABLE_GCC8}" = "yes" ]; then \
+        add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
+        apt-get update && \
+        apt-get install gcc-8 g++-8  -y && \
+        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8 --slave /usr/bin/gcov gcov /usr/bin/gcov-8 && \
+        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 70 --slave /usr/bin/g++ g++ /usr/bin/g++-7 --slave /usr/bin/gcov gcov /usr/bin/gcov-7 ;\
+    fi
+
+# Download and build TensorFlow from the latest sources found in the root container
+# make sure that if they pass in a tag, that it is loaded or we'll get an error
+WORKDIR /
+
+COPY tensorflow/ /tensorflow/
+
+WORKDIR /tensorflow
+
+RUN yes "" | ${PYTHON} configure.py
+
+ENV CI_BUILD_PYTHON ${PYTHON}
+
+# This script detects the version of gcc in the container, sets the appropriate
+# compiler flags based on parameters
+ADD set-build-env.py .
+RUN ${PYTHON} set-build-env.py -p ${TARGET_PLATFORM} -f /root/.mkl.bazelrc \
+    ${CONFIG_V2_DISABLE} ${ENABLE_SECURE_BUILD} ${CONFIG_BFLOAT16_BUILD} ${ENABLE_DNNL1}
+
+# Pull the compiler flags we just wrote into root user's .bazelrc file
+RUN echo "import /root/.mkl.bazelrc" >>/root/.bazelrc
+
+# Install futures>=0.17.1 for Python2.7 compatibility mode
+RUN ${PIP} install future>=0.17.1
+
+RUN bazel --bazelrc=/root/.bazelrc build -c opt \
+    tensorflow/tools/pip_package:build_pip_package && \
+    bazel-bin/tensorflow/tools/pip_package/build_pip_package "${TF_NIGHTLY_FLAG}" "${WHL_DIR}" && \
+    ${PIP} --no-cache-dir install --upgrade "${WHL_DIR}"/*.whl && \
+    rm -rf /root/.cache
+    # Clean up Bazel cache when done.
+
+# Install OpenMPI/Horovod
+COPY install_openmpi_horovod.sh .
+RUN if [ "${ENABLE_HOROVOD}" = "yes" ]; then \
+        chmod +x install_openmpi_horovod.sh && \
+        OPENMPI_VERSION=${OPENMPI_VERSION} OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL} BUILD_SSH=${BUILD_SSH} \
+        INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT} HOROVOD_VERSION=${HOROVOD_VERSION} ./install_openmpi_horovod.sh && \
+        rm -rf install_openmpi_horovod.sh; \
+    fi
+
+# Remove crypto python packages for software compliance check.
+RUN if [ "${RELEASE_CONTAINER}" = "yes" ]; then \
+    ${PIP} uninstall --yes cryptography && \
+    rm -rf /usr/lib/python3/dist-packages/pycrypto-2.6.1.egg-info; \
+    fi
+
+# TensorBoard
+EXPOSE 6006
+# IPython
+EXPOSE 8888
+
+WORKDIR /root

tensorflow/tools/ci_build/linux/mkl/basic-mkl-gpu-test.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Usage: basic_mkl_test.sh
+
+# Helper function to traverse directories up until given file is found.
+function upsearch () {
+  test / == "$PWD" && return || \
+      test -e "$1" && echo "$PWD" && return || \
+      cd .. && upsearch "$1"
+}
+
+# Set up WORKSPACE.
+WORKSPACE="${WORKSPACE:-$(upsearch WORKSPACE)}"
+
+BUILD_TAG=mkl-gpu-ci-test CI_BUILD_USER_FORCE_BADNAME=yes ${WORKSPACE}/tensorflow/tools/ci_build/ci_build.sh gpu tensorflow/tools/ci_build/linux/gpu/run_mkl.sh

tensorflow/tools/ci_build/linux/mkl/basic-mkl-test.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# Usage: basic_mkl_test.sh
+
+# Helper function to traverse directories up until given file is found.
+function upsearch () {
+  test / == "$PWD" && return || \
+      test -e "$1" && echo "$PWD" && return || \
+      cd .. && upsearch "$1"
+}
+
+# Set up WORKSPACE.
+WORKSPACE="${WORKSPACE:-$(upsearch WORKSPACE)}"
+
+OMP_NUM_THREADS=${OMP_NUM_THREADS:-8}
+if [[ ! -z ${OMP_NUM_THREADS} ]]; then
+    if [[ ${OMP_NUM_THREADS} -gt 112 ]] || [[ ${OMP_NUM_THREADS} < 0 ]]; then
+        >&2 echo "Usage: OMP_NUM_THREADS value should be between 0 and 112"
+        exit 1
+    fi
+fi
+
+BUILD_TAG=mkl-ci-test CI_BUILD_USER_FORCE_BADNAME=yes ${WORKSPACE}/tensorflow/tools/ci_build/ci_build.sh cpu OMP_NUM_THREADS=${OMP_NUM_THREADS} tensorflow/tools/ci_build/linux/cpu/run_mkl.sh

tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh

@@ -0,0 +1,384 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Build a whl and container with Intel(R) MKL support
+# Usage: build-dev-container.sh
+
+DEBUG=1
+DOCKER_BINARY="docker"
+TMP_DIR=$(pwd)
+
+# Helper function to traverse directories up until given file is found.
+function upsearch () {
+  test / == "$PWD" && return || \
+      test -e "$1" && echo "$PWD" && return || \
+      cd .. && upsearch "$1"
+}
+
+function debug()
+{
+  if [[ ${DEBUG} == 1 ]] ; then
+    echo $1
+  fi
+}
+
+function die()
+{
+  echo $1
+  exit 1
+}
+
+# Set up WORKSPACE.
+WORKSPACE="${WORKSPACE:-$(upsearch WORKSPACE)}"
+
+ROOT_CONTAINER=${ROOT_CONTAINER:-tensorflow/tensorflow}
+TF_ROOT_CONTAINER_TAG=${ROOT_CONTAINER_TAG:-devel}
+
+# TF_BUILD_VERSION can be either a tag, branch, commit ID or PR number.
+# For a PR, set TF_BUILD_VERSION_IS_PR="yes"
+TF_BUILD_VERSION=${TF_DOCKER_BUILD_DEVEL_BRANCH:-master}
+TF_BUILD_VERSION_IS_PR=${TF_DOCKER_BUILD_DEVEL_BRANCH_IS_PR:-no}
+TF_REPO=${TF_REPO:-https://github.com/tensorflow/tensorflow}
+FINAL_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME:-intel-mkl/tensorflow}
+TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION:-nightly}
+BUILD_AVX_CONTAINERS=${BUILD_AVX_CONTAINERS:-no}
+BUILD_AVX2_CONTAINERS=${BUILD_AVX2_CONTAINERS:-no}
+BUILD_SKX_CONTAINERS=${BUILD_SKX_CONTAINERS:-no}
+BUILD_CLX_CONTAINERS=${BUILD_CLX_CONTAINERS:-no}
+BUILD_ICX_CLIENT_CONTAINERS=${BUILD_ICX_CLIENT_CONTAINERS:-no}
+BUILD_ICX_SERVER_CONTAINERS=${BUILD_ICX_SERVER_CONTAINERS:-no}
+CONTAINER_PORT=${TF_DOCKER_BUILD_PORT:-8888}
+BUILD_TF_V2_CONTAINERS=${BUILD_TF_V2_CONTAINERS:-yes}
+BUILD_TF_BFLOAT16_CONTAINERS=${BUILD_TF_BFLOAT16_CONTAINERS:-no}
+ENABLE_SECURE_BUILD=${ENABLE_SECURE_BUILD:-no}
+BAZEL_VERSION=${BAZEL_VERSION}
+BUILD_PY2_CONTAINERS=${BUILD_PY2_CONTAINERS:-no}
+ENABLE_DNNL1=${ENABLE_DNNL1:-no}
+ENABLE_HOROVOD=${ENABLE_HOROVOD:-no}
+INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT:-no}
+ENABLE_GCC8=${ENABLE_GCC8:-no}
+OPENMPI_VERSION=${OPENMPI_VERSION}
+OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL}
+HOROVOD_VERSION=${HOROVOD_VERSION}
+BUILD_SSH=${BUILD_SSH:-no}
+IS_NIGHTLY=${IS_NIGHTLY:-no}
+RELEASE_CONTAINER=${RELEASE_CONTAINER:-no}
+
+debug "ROOT_CONTAINER=${ROOT_CONTAINER}"
+debug "TF_ROOT_CONTAINER_TAG=${TF_ROOT_CONTAINER_TAG}"
+debug "TF_BUILD_VERSION=${TF_BUILD_VERSION}"
+debug "TF_BUILD_VERSION_IS_PR=${TF_BUILD_VERSION_IS_PR}"
+debug "FINAL_IMAGE_NAME=${FINAL_IMAGE_NAME}"
+debug "TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION}"
+debug "BUILD_AVX_CONTAINERS=${BUILD_AVX_CONTAINERS}"
+debug "BUILD_AVX2_CONTAINERS=${BUILD_AVX2_CONTAINERS}"
+debug "BUILD_SKX_CONTAINERS=${BUILD_SKX_CONTAINERS}"
+debug "BUILD_CLX_CONTAINERS=${BUILD_CLX_CONTAINERS}"
+debug "BUILD_ICX_CLIENT_CONTAINERS=${BUILD_ICX_CLIENT_CONTAINERS}"
+debug "BUILD_ICX_SERVER_CONTAINERS=${BUILD_ICX_SERVER_CONTAINERS}"
+debug "BUILD_TF_V2_CONTAINERS=${BUILD_TF_V2_CONTAINERS}"
+debug "BUILD_TF_BFLOAT16_CONTAINERS=${BUILD_TF_BFLOAT16_CONTAINERS}"
+debug "ENABLE_SECURE_BUILD=${ENABLE_SECURE_BUILD}"
+debug "TMP_DIR=${TMP_DIR}"
+debug "BAZEL_VERSION=${BAZEL_VERSION}"
+debug "ENABLE_GCC8=${ENABLE_GCC8}"
+debug "BUILD_PY2_CONTAINERS=${BUILD_PY2_CONTAINERS}"
+debug "ENABLE_DNNL1=${ENABLE_DNNL1}"
+debug "ENABLE_HOROVOD=${ENABLE_HOROVOD}"
+debug "INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT}"
+debug "OPENMPI_VERSION=${OPENMPI_VERSION}"
+debug "OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL}"
+debug "HOROVOD_VERSION=${HOROVOD_VERSION}"
+debug "BUILD_SSH=${BUILD_SSH}"
+debug "IS_NIGHTLY=${IS_NIGHTLY}"
+debug "RELEASE_CONTAINER=${RELEASE_CONTAINER}"
+
+function build_container()
+{
+  if [[ $# -lt 2 ]]; then
+    die "Usage: build_container <TEMP_IMAGE_NAME> <TF_DOCKER_BUILD_ARGS>."
+  fi
+  TEMP_IMAGE_NAME=${1}
+  debug "TEMP_IMAGE_NAME=${TEMP_IMAGE_NAME}"
+  shift
+  TF_DOCKER_BUILD_ARGS=("${@}")
+
+  # Add the proxy info build args. This will be later on passed to docker as
+  # --build-arg so that users behind corporate proxy can build the images
+  TF_DOCKER_BUILD_ARGS+=("--build-arg http_proxy=${http_proxy}")
+  TF_DOCKER_BUILD_ARGS+=("--build-arg https_proxy=${https_proxy}")
+  TF_DOCKER_BUILD_ARGS+=("--build-arg socks_proxy=${socks_proxy}")
+  TF_DOCKER_BUILD_ARGS+=("--build-arg no_proxy=${no_proxy}")
+  # In general having uppercase proxies is a good idea because different
+  # applications running inside Docker may only honor uppercase proxies
+  TF_DOCKER_BUILD_ARGS+=("--build-arg HTTP_PROXY=${HTTP_PROXY}")
+  TF_DOCKER_BUILD_ARGS+=("--build-arg HTTPS_PROXY=${HTTPS_PROXY}")
+  TF_DOCKER_BUILD_ARGS+=("--build-arg SOCKS_PROXY=${SOCKS_PROXY}")
+  TF_DOCKER_BUILD_ARGS+=("--build-arg NO_PROXY=${NO_PROXY}")
+
+  #Add --config=v2 build arg for TF v2
+  if [[ ${BUILD_TF_V2_CONTAINERS} == "no" ]]; then
+    TF_DOCKER_BUILD_ARGS+=("--build-arg CONFIG_V2_DISABLE=--disable-v2")
+  fi
+
+  #Add build arg for bfloat16 build
+  if [[ ${BUILD_TF_BFLOAT16_CONTAINERS} == "yes" ]]; then
+    TF_DOCKER_BUILD_ARGS+=("--build-arg CONFIG_BFLOAT16_BUILD=--enable-bfloat16")
+  fi
+
+  #Add build arg for Secure Build
+  if [[ ${ENABLE_SECURE_BUILD} == "yes" ]]; then
+    TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_SECURE_BUILD=--secure-build")
+  fi
+
+  # Add build arg for DNNL1
+  if [[ ${ENABLE_DNNL1} == "yes" ]]; then
+    TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_DNNL1=--enable-dnnl1")
+  fi
+
+  # BAZEL Version
+  if [[ ${BAZEL_VERSION} != "" ]]; then
+    TF_DOCKER_BUILD_ARGS+=("--build-arg BAZEL_VERSION=${BAZEL_VERSION}")
+  fi
+
+  # Add build arg for installing OpenMPI/Horovod
+  if [[ ${ENABLE_HOROVOD} == "yes" ]]; then
+    TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_HOROVOD=${ENABLE_HOROVOD}")
+    TF_DOCKER_BUILD_ARGS+=("--build-arg OPENMPI_VERSION=${OPENMPI_VERSION}")
+    TF_DOCKER_BUILD_ARGS+=("--build-arg OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL}")
+    TF_DOCKER_BUILD_ARGS+=("--build-arg HOROVOD_VERSION=${HOROVOD_VERSION}")
+    TF_DOCKER_BUILD_ARGS+=("--build-arg INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT}")
+    TF_DOCKER_BUILD_ARGS+=("--build-arg BUILD_SSH=${BUILD_SSH}")
+  fi
+
+  # Add build arg --nightly_flag for the nightly build
+  if [[ ${IS_NIGHTLY} == "yes" ]]; then
+    TF_DOCKER_BUILD_ARGS+=("--build-arg TF_NIGHTLY_FLAG=--nightly_flag")
+  fi
+
+  # Add build arg GCC8 install
+  TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_GCC8=${ENABLE_GCC8}")
+
+  TF_DOCKER_BUILD_ARGS+=("--build-arg RELEASE_CONTAINER=${RELEASE_CONTAINER}")
+
+  # Perform docker build
+  debug "Building docker image with image name and tag: ${TEMP_IMAGE_NAME}"
+  CMD="${DOCKER_BINARY} build ${TF_DOCKER_BUILD_ARGS[@]} --no-cache --pull -t ${TEMP_IMAGE_NAME} -f Dockerfile.devel-mkl ."
+  debug "CMD=${CMD}"
+  ${CMD}
+
+  if [[ $? == "0" ]]; then
+    debug "${DOCKER_BINARY} build of ${TEMP_IMAGE_NAME} succeeded"
+  else
+    die "FAIL: ${DOCKER_BINARY} build of ${TEMP_IMAGE_NAME} failed"
+  fi
+}
+
+function test_container()
+{
+  if [[ "$#" != "1" ]]; then
+    die "Usage: ${FUNCNAME} <TEMP_IMAGE_NAME>"
+  fi
+
+  TEMP_IMAGE_NAME=${1}
+
+  # Make sure that there is no other containers of the same image running
+  if "${DOCKER_BINARY}" ps | grep -q "${TEMP_IMAGE_NAME}"; then
+    die "ERROR: It appears that there are docker containers of the image "\
+  "${TEMP_IMAGE_NAME} running. Please stop them before proceeding"
+  fi
+
+  # Start a docker container from the newly-built docker image
+  DOCKER_RUN_LOG="${TMP_DIR}/docker_run.log"
+  debug "  Log file is at: ${DOCKER_RUN_LOG}"
+
+  debug "Running docker container from image ${TEMP_IMAGE_NAME}..."
+  RUN_CMD="${DOCKER_BINARY} run --rm -d -p ${CONTAINER_PORT}:${CONTAINER_PORT} ${TEMP_IMAGE_NAME} tail -f /dev/null 2>&1 > ${DOCKER_RUN_LOG}"
+  debug "RUN_CMD=${RUN_CMD}"
+  ${RUN_CMD}
+
+  # Get the container ID
+  CONTAINER_ID=""
+  while [[ -z ${CONTAINER_ID} ]]; do
+    sleep 1
+    debug "Polling for container ID..."
+    CONTAINER_ID=$("${DOCKER_BINARY}" ps | grep "${TEMP_IMAGE_NAME}" | awk '{print $1}')
+  done
+
+  debug "ID of the running docker container: ${CONTAINER_ID}"
+
+  debug "Performing basic sanity checks on the running container..."
+  {
+    ${DOCKER_BINARY} exec ${CONTAINER_ID} bash -c "${PYTHON} -c 'from tensorflow.python import _pywrap_util_port; print(_pywrap_util_port.IsMklEnabled())'"
+    echo "PASS: MKL enabled test in ${TEMP_IMAGE_NAME}"
+  } || {
+    ${DOCKER_BINARY} exec ${CONTAINER_ID} bash -c "${PYTHON} -c 'from tensorflow.python import pywrap_tensorflow; print(pywrap_tensorflow.IsMklEnabled())'"
+    echo "PASS: Old MKL enabled in ${TEMP_IMAGE_NAME}"
+  } || {
+    die "FAIL: MKL enabled test in ${TEMP_IMAGE_NAME}"
+  }
+
+  # Test to check if horovod is installed successfully
+  if [[ ${ENABLE_HOROVOD} == "yes" ]]; then
+      debug "Test horovod in the container..."
+      ${DOCKER_BINARY} exec ${CONTAINER_ID} bash -c "${PYTHON} -c 'import horovod.tensorflow as hvd;'"
+      if [[ $? == "0" ]]; then
+          echo "PASS: HOROVOD installation test in ${TEMP_IMAGE_NAME}"
+      else
+          die "FAIL: HOROVOD installation test in ${TEMP_IMAGE_NAME}"
+      fi
+  fi
+  
+  # Stop the running docker container
+  sleep 1
+  "${DOCKER_BINARY}" stop --time=0 ${CONTAINER_ID}
+}
+
+function checkout_tensorflow()
+{
+  if [[ "$#" != "3" ]]; then
+    die "Usage: ${FUNCNAME} <REPO_URL> <BRANCH/TAG/COMMIT-ID/PR-ID> <TF_BUILD_VERSION_IS_PR>"
+  fi
+
+  TF_REPO="${1}"
+  TF_BUILD_VERSION="${2}"
+  TF_BUILD_VERSION_IS_PR="${3}"
+  TENSORFLOW_DIR="tensorflow"
+
+  debug "Checking out ${TF_REPO}:${TF_BUILD_VERSION} into ${TENSORFLOW_DIR}"
+
+  # Clean any existing tensorflow sources
+  rm -rf "${TENSORFLOW_DIR}"
+
+  git clone ${TF_REPO} ${TENSORFLOW_DIR}
+  cd ${TENSORFLOW_DIR}
+  if [[ "${TF_BUILD_VERSION_IS_PR}" == "yes" ]]; then
+    # If TF_BUILD_VERSION is a PR number, then fetch first
+    git fetch origin pull/${TF_BUILD_VERSION}/head:pr-${TF_BUILD_VERSION}
+    git checkout pr-${TF_BUILD_VERSION}
+  else
+    git checkout ${TF_BUILD_VERSION}
+  fi
+  if [ $? -ne 0 ]; then
+    die "Unable to find ${TF_BUILD_VERSION} on ${TF_REPO}"
+  fi
+  cd ..
+}
+
+function tag_container()
+{
+  # Apply the final image name and tag
+  TEMP_IMAGE_NAME="${1}"
+  FINAL_IMG="${2}"
+
+  DOCKER_VER=$("${DOCKER_BINARY}" version | grep Version | head -1 | awk '{print $NF}')
+  if [[ -z "${DOCKER_VER}" ]]; then
+    die "ERROR: Failed to determine ${DOCKER_BINARY} version"
+  fi
+  DOCKER_MAJOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 1)
+  DOCKER_MINOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 2)
+
+  FORCE_TAG=""
+  if [[ "${DOCKER_MAJOR_VER}" -le 1 ]] && \
+    [[ "${DOCKER_MINOR_VER}" -le 9 ]]; then
+    FORCE_TAG="--force"
+  fi
+
+  "${DOCKER_BINARY}" tag ${FORCE_TAG} "${TEMP_IMAGE_NAME}" "${FINAL_IMG}" || \
+      die "Failed to tag intermediate docker image ${TEMP_IMAGE_NAME} as ${FINAL_IMG}"
+
+  debug "Successfully tagged docker image: ${FINAL_IMG}"
+}
+
+PYTHON_VERSIONS=("python3")
+if [[ ${BUILD_PY2_CONTAINERS} == "yes" ]]; then
+  PYTHON_VERSIONS+=("python")
+fi
+
+PLATFORMS=()
+if [[ ${BUILD_AVX_CONTAINERS} == "yes" ]]; then
+  PLATFORMS+=("sandybridge")
+fi
+
+if [[ ${BUILD_AVX2_CONTAINERS} == "yes" ]]; then
+  PLATFORMS+=("haswell")
+fi
+
+if [[ ${BUILD_SKX_CONTAINERS} == "yes" ]]; then
+  PLATFORMS+=("skylake")
+fi
+
+if [[ ${BUILD_CLX_CONTAINERS} == "yes" ]]; then
+  PLATFORMS+=("icelake")
+fi
+
+if [[ ${BUILD_ICX_CLIENT_CONTAINERS} == "yes" ]]; then
+  PLATFORMS+=("icelake-client")
+fi
+
+if [[ ${BUILD_ICX_SERVER_CONTAINERS} == "yes" ]]; then
+  PLATFORMS+=("icelake-server")
+fi
+
+# Checking out sources needs to be done only once
+checkout_tensorflow "${TF_REPO}" "${TF_BUILD_VERSION}" "${TF_BUILD_VERSION_IS_PR}"
+
+for PLATFORM in "${PLATFORMS[@]}"
+do
+  for PYTHON in "${PYTHON_VERSIONS[@]}"
+  do
+    # Clear the build args array
+    TF_DOCKER_BUILD_ARGS=("--build-arg TARGET_PLATFORM=${PLATFORM}")
+    TF_DOCKER_BUILD_ARGS+=("--build-arg ROOT_CONTAINER=${ROOT_CONTAINER}")
+    FINAL_TAG="${TF_DOCKER_BUILD_VERSION}"
+    ROOT_CONTAINER_TAG="${TF_ROOT_CONTAINER_TAG}"
+
+      if [[ ${PLATFORM} == "haswell" ]]; then
+        FINAL_TAG="${FINAL_TAG}-avx2"
+      fi
+
+      if [[ ${PLATFORM} == "skylake" ]]; then
+        FINAL_TAG="${FINAL_TAG}-avx512"
+      fi
+
+      if [[ ${PLATFORM} == "icelake" ]]; then
+        FINAL_TAG="${FINAL_TAG}-avx512-VNNI"
+      fi
+
+      if [[ ${PLATFORM} == "icelake-client" ]]; then
+        FINAL_TAG="${FINAL_TAG}-icx-client"
+      fi
+
+      if [[ ${PLATFORM} == "icelake-server" ]]; then
+        FINAL_TAG="${FINAL_TAG}-icx-server"
+      fi
+
+      # Add -devel-mkl to the image tag
+      FINAL_TAG="${FINAL_TAG}-devel-mkl"
+      if [[ "${PYTHON}" == "python3" ]]; then
+        TF_DOCKER_BUILD_ARGS+=("--build-arg WHL_DIR=/tmp/pip3")
+        TF_DOCKER_BUILD_ARGS+=("--build-arg PIP=pip3")
+      fi
+
+      TF_DOCKER_BUILD_ARGS+=("--build-arg PYTHON=${PYTHON}")
+      TF_DOCKER_BUILD_ARGS+=("--build-arg ROOT_CONTAINER_TAG=${ROOT_CONTAINER_TAG}")
+
+      # Intermediate image name with tag
+      TEMP_IMAGE_NAME="${USER}/tensorflow:${FINAL_TAG}"
+      build_container "${TEMP_IMAGE_NAME}" "${TF_DOCKER_BUILD_ARGS[@]}"
+      test_container "${TEMP_IMAGE_NAME}"
+      tag_container "${TEMP_IMAGE_NAME}" "${FINAL_IMAGE_NAME}:${FINAL_TAG}"
+  done
+done

tensorflow/tools/ci_build/linux/mkl/install_openmpi_horovod.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Install OpenMPI, OpenSSH and Horovod during Intel(R) MKL container build
+# Usage: install_openmpi_horovod.sh [OPENMPI_VERSION=<openmpi version>] [OPENMPI_DOWNLOAD_URL=<openmpi download url>] 
+# [HOROVOD_VERSION=<horovod version>]
+
+set -e
+
+# Set default
+OPENMPI_VERSION=${OPENMPI_VERSION:-openmpi-2.1.1}
+OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL:-https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.gz}
+INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT:-no}
+BUILD_SSH=${BUILD_SSH:-no}
+HOROVOD_VERSION=${HOROVOD_VERSION:-0.19.1}
+SSH_CONFIG_PATH=/etc/ssh
+
+# Install Open MPI
+echo "Installing OpenMPI version ${OPENMPI_VERSION} ..."
+echo "OpenMPI Download url ${OPENMPI_DOWNLOAD_URL} ..."
+
+mkdir /tmp/openmpi
+cd /tmp/openmpi
+curl -fSsL -O ${OPENMPI_DOWNLOAD_URL}
+tar zxf ${OPENMPI_VERSION}.tar.gz
+cd ${OPENMPI_VERSION}
+./configure --enable-mpirun-prefix-by-default
+make -j $(nproc) all
+make install
+ldconfig
+cd /
+rm -rf /tmp/openmpi
+
+# Create a wrapper for OpenMPI to allow running as root by default
+mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real
+echo '#!/bin/bash' > /usr/local/bin/mpirun
+echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun
+chmod a+x /usr/local/bin/mpirun
+
+# Configure OpenMPI to run good defaults:
+echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
+
+# Check mpi version
+echo 'OpenMPI version:'
+mpirun --version
+
+# Install OpenSSH for MPI to communicate between containers
+if [[ ${BUILD_SSH} == "yes" ]]; then
+	mkdir /tmp/buildssh
+	cd /tmp/buildssh && curl -fSsL -O http://www.zlib.net/zlib-1.2.11.tar.gz && tar -xzvf zlib-1.2.11.tar.gz && \
+		cd /tmp/buildssh/zlib-1.2.11 && ./configure && make && make install
+	cd /tmp/buildssh && curl -fSsL -O https://www.openssl.org/source/openssl-1.1.1.tar.gz && tar -xzvf openssl-1.1.1.tar.gz && \
+		cd  /tmp/buildssh/openssl-1.1.1 && ./config && make  && make test  && make install
+	cd /tmp/buildssh && curl -fSsL -O https://mirrors.sonic.net/pub/OpenBSD/OpenSSH/portable/openssh-8.4p1.tar.gz && \
+		tar -xzvf openssh-8.4p1.tar.gz && cd /tmp/buildssh/openssh-8.4p1 && \
+		./configure --with-md5-passwords  && make && \
+		groupadd sshd && useradd -M -g sshd -c 'sshd privsep' -d /var/empty -s /sbin/nologin sshd && passwd -l sshd && \
+		make install
+	apt-get clean && apt-get update && \
+	    apt-get install -y --no-install-recommends --fix-missing \
+	        libnuma-dev cmake
+        SSH_CONFIG_PATH=/usr/local/etc
+else
+	apt-get clean && apt-get update && \
+	    apt-get install -y --no-install-recommends --fix-missing \
+	        openssh-client openssh-server libnuma-dev cmake && \
+	    rm -rf /var/lib/apt/lists/*
+	if [[ $?  == "0" ]]; then
+	    echo "PASS: OpenSSH installation"
+	else
+	    yum -y update && yum -y install numactl-devel openssh-server openssh-clients cmake && \
+	        yum clean all
+	    if [[ $?  == "0" ]]; then
+	        echo "PASS: OpenSSH installation"
+	    else
+	        echo "Unsupported Linux distribution. Aborting!" && exit 1
+	    fi
+	fi
+fi
+mkdir -p /var/run/sshd
+grep -v StrictHostKeyChecking ${SSH_CONFIG_PATH}/ssh_config > ${SSH_CONFIG_PATH}/ssh_config.new
+# Allow OpenSSH to talk to containers without asking for confirmation
+echo " StrictHostKeyChecking no" >> ${SSH_CONFIG_PATH}/ssh_config.new
+mv ${SSH_CONFIG_PATH}/ssh_config.new ${SSH_CONFIG_PATH}/ssh_config
+
+# Install Horovod
+if [[ ${INSTALL_HOROVOD_FROM_COMMIT} == "yes" ]]; then
+	HOROVOD_WITH_TENSORFLOW=1
+	python3 -m pip install --no-cache-dir git+https://github.com/horovod/horovod.git@${HOROVOD_VERSION}
+else
+	HOROVOD_WITH_TENSORFLOW=1
+	python3 -m pip install --no-cache-dir horovod==${HOROVOD_VERSION}
+fi

tensorflow/tools/ci_build/linux/mkl/set-build-env.py

@@ -0,0 +1,381 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Configure build environment for certain Intel platforms."""
+
+import argparse
+import os
+import subprocess
+
+BASIC_BUILD_OPTS = ["--cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0", "--copt=-O3"]
+
+SECURE_BUILD_OPTS = [
+    "--copt=-Wformat", "--copt=-Wformat-security", "--copt=-fstack-protector",
+    "--copt=-fPIC", "--copt=-fpic", "--linkopt=-znoexecstack",
+    "--linkopt=-zrelro", "--linkopt=-znow", "--linkopt=-fstack-protector"
+]
+
+
+class IntelPlatform(object):
+  min_gcc_major_version_ = 0
+  min_gcc_minor_version_ = 0
+  host_gcc_major_version_ = 0
+  host_gcc_minor_version_ = 0
+  BAZEL_PREFIX_ = "--copt="
+  ARCH_PREFIX_ = "-march="
+  FLAG_PREFIX_ = "-m"
+
+  def __init__(self, min_gcc_major_version, min_gcc_minor_version):
+    self.min_gcc_minor_version_ = min_gcc_minor_version
+    self.min_gcc_major_version_ = min_gcc_major_version
+
+  # Return True or False depending on whether
+  # The platform optimization flags can be generated by
+  # the gcc version specified in the parameters
+  def set_host_gcc_version(self, gcc_major_version, gcc_minor_version):
+    # True only if the gcc version in the tuple is >=
+    # min_gcc_major_version_, min_gcc_minor_version_
+    if gcc_major_version < self.min_gcc_major_version_:
+      print("Your MAJOR version of GCC is too old: {}; "
+            "it must be at least {}.{}".format(gcc_major_version,
+                                               self.min_gcc_major_version_,
+                                               self.min_gcc_minor_version_))
+      return False
+    elif gcc_major_version == self.min_gcc_major_version_ and \
+          gcc_minor_version < self.min_gcc_minor_version_:
+      print("Your MINOR version of GCC is too old: {}; "
+            "it must be at least {}.{}".format(gcc_minor_version,
+                                               self.min_gcc_major_version_,
+                                               self.min_gcc_minor_version_))
+      return False
+    print("gcc version OK: {}.{}".format(gcc_major_version, gcc_minor_version))
+    self.host_gcc_major_version_ = gcc_major_version
+    self.host_gcc_minor_version_ = gcc_minor_version
+    return True
+
+  # return a string with all the necessary bazel formatted flags for this
+  # platform in this gcc environment
+  def get_bazel_gcc_flags(self):
+    raise NotImplementedError(self)
+
+  # Returns True if the host gcc version is older than the gcc version in which
+  # the new march flag became available.
+  # Specify the version in which the new name usage began
+  def use_old_arch_names(self, gcc_new_march_major_version,
+                         gcc_new_march_minor_version):
+    if self.host_gcc_major_version_ < gcc_new_march_major_version:
+      return True
+    elif self.host_gcc_major_version_ == gcc_new_march_major_version and \
+       self.host_gcc_minor_version_ < gcc_new_march_minor_version:
+      return True
+    return False
+
+
+class NehalemPlatform(IntelPlatform):
+
+  def __init__(self):
+    IntelPlatform.__init__(self, 4, 8)
+
+  def get_bazel_gcc_flags(self):
+    NEHALEM_ARCH_OLD = "corei7"
+    NEHALEM_ARCH_NEW = "nehalem"
+    if self.use_old_arch_names(4, 9):
+      return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+             NEHALEM_ARCH_OLD + " "
+    else:
+      return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+             NEHALEM_ARCH_NEW + " "
+
+
+class SandyBridgePlatform(IntelPlatform):
+
+  def __init__(self):
+    IntelPlatform.__init__(self, 4, 8)
+
+  def get_bazel_gcc_flags(self):
+    SANDYBRIDGE_ARCH_OLD = "corei7-avx"
+    SANDYBRIDGE_ARCH_NEW = "sandybridge"
+    if self.use_old_arch_names(4, 9):
+      return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+             SANDYBRIDGE_ARCH_OLD + " "
+    else:
+      return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+             SANDYBRIDGE_ARCH_NEW + " "
+
+
+class HaswellPlatform(IntelPlatform):
+
+  def __init__(self):
+    IntelPlatform.__init__(self, 4, 8)
+
+  def get_bazel_gcc_flags(self):
+    HASWELL_ARCH_OLD = "core-avx2"  # Only missing the POPCNT instruction
+    HASWELL_ARCH_NEW = "haswell"
+    POPCNT_FLAG = "popcnt"
+    if self.use_old_arch_names(4, 9):
+      ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+                HASWELL_ARCH_OLD + " "
+      return ret_val + self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + \
+             POPCNT_FLAG + " "
+    else:
+      return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+             HASWELL_ARCH_NEW + " "
+
+
+class SkylakePlatform(IntelPlatform):
+
+  def __init__(self):
+    IntelPlatform.__init__(self, 4, 9)
+
+  def get_bazel_gcc_flags(self):
+    SKYLAKE_ARCH_OLD = "broadwell"  # Only missing the POPCNT instruction
+    SKYLAKE_ARCH_NEW = "skylake-avx512"
+    # the flags that broadwell is missing: pku, clflushopt, clwb, avx512vl,
+    # avx512bw, avx512dq. xsavec and xsaves are available in gcc 5.x
+    # but for now, just exclude them.
+    AVX512_FLAGS = ["avx512f", "avx512cd"]
+    if self.use_old_arch_names(6, 1):
+      ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+                SKYLAKE_ARCH_OLD + " "
+      for flag in AVX512_FLAGS:
+        ret_val += self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + flag + " "
+      return ret_val
+    else:
+      return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+             SKYLAKE_ARCH_NEW + " "
+
+
+class CascadelakePlatform(IntelPlatform):
+
+  def __init__(self):
+    IntelPlatform.__init__(self, 8, 3)
+
+  def get_bazel_gcc_flags(self):
+    CASCADELAKE_ARCH_OLD = "skylake-avx512"  # Only missing the POPCNT instruction
+    CASCADELAKE_ARCH_NEW = "cascadelake"
+    # the flags that broadwell is missing: pku, clflushopt, clwb, avx512vl, avx512bw, avx512dq
+    VNNI_FLAG = "avx512vnni"
+    if IntelPlatform.use_old_arch_names(self, 9, 1):
+      ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+        CASCADELAKE_ARCH_OLD + " "
+      return ret_val + self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + \
+             VNNI_FLAG + " "
+    else:
+      return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+             CASCADELAKE_ARCH_NEW + " "
+
+
+class IcelakeClientPlatform(IntelPlatform):
+
+  def __init__(self):
+    IntelPlatform.__init__(self, 8, 4)
+
+  def get_bazel_gcc_flags(self):
+    ICELAKE_ARCH_OLD = "skylake-avx512"
+    ICELAKE_ARCH_NEW = "icelake-client"
+    AVX512_FLAGS = ["avx512f", "avx512cd"]
+    if IntelPlatform.use_old_arch_names(self, 8, 4):
+      ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+        ICELAKE_ARCH_OLD + " "
+      for flag in AVX512_FLAGS:
+        ret_val += self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + flag + " "
+      return ret_val
+    else:
+      return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+             ICELAKE_ARCH_NEW + " "
+
+
+class IcelakeServerPlatform(IntelPlatform):
+
+  def __init__(self):
+    IntelPlatform.__init__(self, 8, 4)
+
+  def get_bazel_gcc_flags(self):
+    ICELAKE_ARCH_OLD = "skylake-avx512"
+    ICELAKE_ARCH_NEW = "icelake-server"
+    AVX512_FLAGS = ["avx512f", "avx512cd"]
+    if IntelPlatform.use_old_arch_names(self, 8, 4):
+      ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+        ICELAKE_ARCH_OLD + " "
+      for flag in AVX512_FLAGS:
+        ret_val += self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + flag + " "
+      return ret_val
+    else:
+      return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
+             ICELAKE_ARCH_NEW + " "
+
+
+class BuildEnvSetter(object):
+  """Prepares the proper environment settings for various Intel platforms."""
+  default_platform_ = "haswell"
+
+  PLATFORMS_ = {
+      "nehalem": NehalemPlatform(),
+      "sandybridge": SandyBridgePlatform(),
+      "haswell": HaswellPlatform(),
+      "skylake": SkylakePlatform(),
+      "cascadelake": CascadelakePlatform(),
+      "icelake-client": IcelakeClientPlatform(),
+      "icelake-server": IcelakeServerPlatform(),
+  }
+
+  def __init__(self):
+    self.args = None
+    self.bazel_flags_ = "build "
+    self.target_platform_ = None
+
+  # Return a tuple of the current gcc version
+  def get_gcc_version(self):
+    gcc_major_version = 0
+    gcc_minor_version = 0
+    # check to see if gcc is present
+    gcc_path = ""
+    gcc_path_cmd = "command -v gcc"
+    try:
+      gcc_path = subprocess.check_output(gcc_path_cmd, shell=True,
+                                         stderr=subprocess.STDOUT).\
+        strip()
+      print("gcc located here: {}".format(gcc_path))
+      if not os.access(gcc_path, os.F_OK | os.X_OK):
+        raise ValueError(
+            "{} does not exist or is not executable.".format(gcc_path))
+
+      gcc_output = subprocess.check_output(
+          [gcc_path, "-dumpfullversion", "-dumpversion"],
+          stderr=subprocess.STDOUT).strip()
+      # handle python2 vs 3 (bytes vs str type)
+      if isinstance(gcc_output, bytes):
+        gcc_output = gcc_output.decode("utf-8")
+      print("gcc version: {}".format(gcc_output))
+      gcc_info = gcc_output.split(".")
+      gcc_major_version = int(gcc_info[0])
+      gcc_minor_version = int(gcc_info[1])
+    except subprocess.CalledProcessException as e:
+      print("Problem getting gcc info: {}".format(e))
+      gcc_major_version = 0
+      gcc_minor_version = 0
+    return gcc_major_version, gcc_minor_version
+
+  def parse_args(self):
+    """Set up argument parser, and parse CLI args."""
+    arg_parser = argparse.ArgumentParser(
+        description="Parse the arguments for the "
+        "TensorFlow build environment "
+        " setter")
+    arg_parser.add_argument(
+        "--disable-mkl",
+        dest="disable_mkl",
+        help="Turn off MKL. By default the compiler flag "
+        "--config=mkl is enabled.",
+        action="store_true")
+    arg_parser.add_argument(
+        "--disable-v2",
+        dest="disable_v2",
+        help="Build TensorFlow v1 rather than v2. By default the "
+        " compiler flag --config=v2 is enabled.",
+        action="store_true")
+    arg_parser.add_argument(
+        "--enable-bfloat16",
+        dest="enable_bfloat16",
+        help="Enable bfloat16 build. By default it is "
+        " disabled if no parameter is passed.",
+        action="store_true")
+    arg_parser.add_argument(
+        "--enable-dnnl1",
+        dest="enable_dnnl1",
+        help="Enable dnnl1 build. By default it is "
+        " disabled if no parameter is passed.",
+        action="store_true")
+    arg_parser.add_argument(
+        "-s",
+        "--secure-build",
+        dest="secure_build",
+        help="Enable secure build flags.",
+        action="store_true")
+    arg_parser.add_argument(
+        "-p",
+        "--platform",
+        choices=self.PLATFORMS_.keys(),
+        help="The target platform.",
+        dest="target_platform",
+        default=self.default_platform_)
+    arg_parser.add_argument(
+        "-f",
+        "--bazelrc-file",
+        dest="bazelrc_file",
+        help="The full path to the bazelrc file into which "
+        "the build command will be written. The path "
+        "will be relative to the container "
+        " environment.",
+        required=True)
+
+    self.args = arg_parser.parse_args()
+
+  def validate_args(self):
+    # Check the bazelrc file
+    if os.path.exists(self.args.bazelrc_file):
+      if os.path.isfile(self.args.bazelrc_file):
+        self._debug("The file {} exists and will be deleted.".format(
+            self.args.bazelrc_file))
+      elif os.path.isdir(self.args.bazelrc_file):
+        print("You can't write bazel config to \"{}\" "
+              "because it is a directory".format(self.args.bazelrc_file))
+        return False
+
+    # Validate gcc with the requested platform
+    gcc_major_version, gcc_minor_version = self.get_gcc_version()
+    if gcc_major_version == 0 or \
+       not self.target_platform_.set_host_gcc_version(
+           gcc_major_version, gcc_minor_version):
+      return False
+
+    return True
+
+  def set_build_args(self):
+    """Generate Bazel build flags."""
+    for flag in BASIC_BUILD_OPTS:
+      self.bazel_flags_ += "{} ".format(flag)
+    if self.args.secure_build:
+      for flag in SECURE_BUILD_OPTS:
+        self.bazel_flags_ += "{} ".format(flag)
+    if not self.args.disable_mkl:
+      self.bazel_flags_ += "--config=mkl "
+    if self.args.disable_v2:
+      self.bazel_flags_ += "--config=v1 "
+    if self.args.enable_dnnl1:
+      self.bazel_flags_ += "--define build_with_mkl_dnn_v1_only=true "
+    if self.args.enable_bfloat16:
+      self.bazel_flags_ += "--copt=-DENABLE_INTEL_MKL_BFLOAT16 "
+
+    self.bazel_flags_ += self.target_platform_.get_bazel_gcc_flags()
+
+  def write_build_args(self):
+    self._debug("Writing build flags: {}".format(self.bazel_flags_))
+    with open(self.args.bazelrc_file, "w") as f:
+      f.write(self.bazel_flags_ + "\n")
+
+  def _debug(self, msg):
+    print(msg)
+
+  def go(self):
+    self.parse_args()
+    self.target_platform_ = self.PLATFORMS_.get(self.args.target_platform)
+    if self.validate_args():
+      self.set_build_args()
+      self.write_build_args()
+    else:
+      print("Error.")
+
+env_setter = BuildEnvSetter()
+env_setter.go()

tensorflow/tools/ci_build/linux/ppc64le/cpu/run_py2.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export CC_OPT_FLAGS='-mcpu=power8 -mtune=power8'
+export PYTHON_BIN_PATH=`which python2`
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only --config=opt \
+    --test_output=errors --test_size_filters=small,medium -- \
+    //tensorflow/... -//tensorflow/compiler/...

tensorflow/tools/ci_build/linux/ppc64le/cpu/run_py3.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export CC_OPT_FLAGS='-mcpu=power8 -mtune=power8'
+export PYTHON_BIN_PATH=`which python3`
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only --config=opt \
+    --test_output=errors --test_size_filters=small,medium -- \
+    //tensorflow/... -//tensorflow/compiler/...

tensorflow/tools/ci_build/linux/ppc64le/gpu/run_py2.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+LT_JOBS=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader | wc -l)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo "Bazel will use ${LT_JOBS} local test job(s)."
+echo ""
+
+# Run configure.
+export PYTHON_BIN_PATH=`which python2`
+export CC_OPT_FLAGS='-mcpu=power8 -mtune=power8'
+
+export TF_NEED_CUDA=1
+export TF_CUDA_COMPUTE_CAPABILITIES=3.7
+
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --config=cuda --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-no_gpu,-benchmark-test -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 \
+    --test_output=errors --local_test_jobs=${LT_JOBS} --build_tests_only --config=opt \
+    --test_size_filters=small,medium \
+    --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute -- \
+    //tensorflow/... -//tensorflow/compiler/...

tensorflow/tools/ci_build/linux/ppc64le/gpu/run_py3.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+LT_JOBS=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader | wc -l)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo "Bazel will use ${LT_JOBS} local test job(s)."
+echo ""
+
+# Run configure.
+export PYTHON_BIN_PATH=`which python3`
+export CC_OPT_FLAGS='-mcpu=power8 -mtune=power8'
+
+export TF_NEED_CUDA=1
+export TF_CUDA_COMPUTE_CAPABILITIES=3.7
+
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --config=cuda --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-no_gpu,-benchmark-test -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 \
+    --test_output=errors --local_test_jobs=${LT_JOBS} --build_tests_only --config=opt \
+    --test_size_filters=small,medium \
+    --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute -- \
+    //tensorflow/... -//tensorflow/compiler/...

tensorflow/tools/ci_build/linux/rocm/rocm_py310_pip.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+set -e
+set -x
+
+source tensorflow/tools/ci_build/release/common.sh
+
+install_ubuntu_16_python_pip_deps python3.10
+
+install_bazelisk
+
+# Export required variables for running pip.sh
+export OS_TYPE="UBUNTU"
+export CONTAINER_TYPE="ROCM"
+export TF_PYTHON_VERSION='python3.10'
+export PYTHON_BIN_PATH="$(which ${TF_PYTHON_VERSION})"
+
+# .bazelrc does not have all config options listed within it for ROCm (as it does for others)
+# So explicitly calling the configure script here to properly setup the ROCm config options
+yes "" | TF_NEED_ROCM=1 ./configure
+
+# Setup some env vars needed by the underlying script(s) for building and testing
+export N_BUILD_JOBS=$(grep -c ^processor /proc/cpuinfo)
+export TF_GPU_COUNT=$(lspci|grep 'controller'|grep 'AMD/ATI'|wc -l)
+export TF_TESTS_PER_GPU=1
+export N_TEST_JOBS=$(expr ${TF_GPU_COUNT} \* ${TF_TESTS_PER_GPU})
+
+
+# Get the default test targets for bazel.
+source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh
+
+#Add filter tags specific to ROCm version
+rocm_major_version=`cat /opt/rocm/.info/version | cut -d "." -f 1`
+rocm_minor_version=`cat /opt/rocm/.info/version | cut -d "." -f 2`
+TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=""
+if [[ $rocm_major_version -ge 5 ]]; then
+	if [[ $rocm_minor_version -lt 3 ]]; then
+		TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=",no_rocm_pre_53"	
+	fi
+else
+    TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=",no_rocm_pre_53"	
+fi
+
+# # Export optional variables for running pip.sh
+export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_excluded,-oss_serial,-no_oss_py39,-no_rocm${TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC}'
+export TF_BUILD_FLAGS="--config=release_base "
+export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \
+ --test_env=TF2_BEHAVIOR=1 \
+ --local_test_jobs=${N_TEST_JOBS} \
+ --test_env=TF_GPU_COUNT=$TF_GPU_COUNT \
+ --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
+ --test_env=HSA_TOOLS_LIB=libroctracer64.so \
+ --test_timeout 600,900,2400,7200 \
+ --build_tests_only \
+ --test_output=errors \
+ --test_sharding_strategy=disabled \
+ --define=no_tensorflow_py_deps=true \
+ --test_lang_filters=py \
+ --verbose_failures=true \
+ --keep_going "
+export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... "
+export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean"
+export IS_NIGHTLY="${IS_NIGHTLY:-0}"
+export TF_PROJECT_NAME="tensorflow_rocm"  # single pip package!
+export TF_PIP_TEST_ROOT="pip_test"
+
+# To build both tensorflow and tensorflow-gpu pip packages
+export TF_BUILD_BOTH_GPU_PACKAGES=0
+
+./tensorflow/tools/ci_build/builds/pip_new.sh

tensorflow/tools/ci_build/linux/rocm/rocm_py36_pip.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+set -e
+set -x
+
+source tensorflow/tools/ci_build/release/common.sh
+
+install_ubuntu_16_python_pip_deps python3.6
+
+install_bazelisk
+
+# Export required variables for running pip.sh
+export OS_TYPE="UBUNTU"
+export CONTAINER_TYPE="ROCM"
+export TF_PYTHON_VERSION='python3.6'
+export PYTHON_BIN_PATH="$(which ${TF_PYTHON_VERSION})"
+
+# .bazelrc does not have all config options listed within it for ROCm (as it does for others)
+# So explicitly calling the configure script here to properly setup the ROCm config options
+yes "" | TF_NEED_ROCM=1 ./configure
+
+# Setup some env vars needed by the underlying script(s) for building and testing
+export N_BUILD_JOBS=$(grep -c ^processor /proc/cpuinfo)
+export TF_GPU_COUNT=$(lspci|grep 'controller'|grep 'AMD/ATI'|wc -l)
+export TF_TESTS_PER_GPU=1
+export N_TEST_JOBS=$(expr ${TF_GPU_COUNT} \* ${TF_TESTS_PER_GPU})
+
+
+# Get the default test targets for bazel.
+source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh
+
+# # Export optional variables for running pip.sh
+export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_excluded,-oss_serial,-no_oss_py36,-no_rocm'
+export TF_BUILD_FLAGS="--config=release_base "
+export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \
+ --test_env=TF2_BEHAVIOR=1 \
+ --local_test_jobs=${N_TEST_JOBS} \
+ --test_env=TF_GPU_COUNT=$TF_GPU_COUNT \
+ --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
+ --test_timeout 600,900,2400,7200 \
+ --build_tests_only \
+ --test_output=errors \
+ --test_sharding_strategy=disabled \
+ --define=no_tensorflow_py_deps=true \
+ --test_lang_filters=py \
+ --verbose_failures=true \
+ --keep_going "
+export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... "
+export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean"
+export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo.
+export TF_PROJECT_NAME="tensorflow_rocm"  # single pip package!
+export TF_PIP_TEST_ROOT="pip_test"
+
+# To build both tensorflow and tensorflow-gpu pip packages
+export TF_BUILD_BOTH_GPU_PACKAGES=0
+
+./tensorflow/tools/ci_build/builds/pip_new.sh

tensorflow/tools/ci_build/linux/rocm/rocm_py37_pip.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+set -e
+set -x
+
+source tensorflow/tools/ci_build/release/common.sh
+
+install_ubuntu_16_python_pip_deps python3.7
+
+install_bazelisk
+
+# Export required variables for running pip.sh
+export OS_TYPE="UBUNTU"
+export CONTAINER_TYPE="ROCM"
+export TF_PYTHON_VERSION='python3.7'
+export PYTHON_BIN_PATH="$(which ${TF_PYTHON_VERSION})"
+
+# .bazelrc does not have all config options listed within it for ROCm (as it does for others)
+# So explicitly calling the configure script here to properly setup the ROCm config options
+yes "" | TF_NEED_ROCM=1 ./configure
+
+# Setup some env vars needed by the underlying script(s) for building and testing
+export N_BUILD_JOBS=$(grep -c ^processor /proc/cpuinfo)
+export TF_GPU_COUNT=$(lspci|grep 'controller'|grep 'AMD/ATI'|wc -l)
+export TF_TESTS_PER_GPU=1
+export N_TEST_JOBS=$(expr ${TF_GPU_COUNT} \* ${TF_TESTS_PER_GPU})
+
+
+# Get the default test targets for bazel.
+source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh
+
+#Add filter tags specific to ROCm version
+rocm_major_version=`cat /opt/rocm/.info/version | cut -d "." -f 1`
+rocm_minor_version=`cat /opt/rocm/.info/version | cut -d "." -f 2`
+TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=""
+if [[ $rocm_major_version -ge 5 ]]; then
+	if [[ $rocm_minor_version -lt 3 ]]; then
+		TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=",no_rocm_pre_53"	
+	fi
+else
+    TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=",no_rocm_pre_53"	
+fi
+
+
+
+# # Export optional variables for running pip.sh
+export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_excluded,-oss_serial,-no_oss_py39,-no_rocm'${TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC}
+export TF_BUILD_FLAGS="--config=release_base "
+export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \
+ --test_env=TF2_BEHAVIOR=1 \
+ --local_test_jobs=${N_TEST_JOBS} \
+ --test_env=TF_GPU_COUNT=$TF_GPU_COUNT \
+ --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
+ --test_timeout 600,900,2400,7200 \
+ --build_tests_only \
+ --test_output=errors \
+ --test_sharding_strategy=disabled \
+ --define=no_tensorflow_py_deps=true \
+ --test_lang_filters=py \
+ --verbose_failures=true \
+ --keep_going "
+export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... "
+export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean"
+export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo.
+export TF_PROJECT_NAME="tensorflow_rocm"  # single pip package!
+export TF_PIP_TEST_ROOT="pip_test"
+
+# To build both tensorflow and tensorflow-gpu pip packages
+export TF_BUILD_BOTH_GPU_PACKAGES=0
+
+./tensorflow/tools/ci_build/builds/pip_new.sh

tensorflow/tools/ci_build/linux/rocm/rocm_py38_pip.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+set -e
+set -x
+
+source tensorflow/tools/ci_build/release/common.sh
+
+install_ubuntu_16_python_pip_deps python3.8
+
+install_bazelisk
+
+# Export required variables for running pip.sh
+export OS_TYPE="UBUNTU"
+export CONTAINER_TYPE="ROCM"
+export TF_PYTHON_VERSION='python3.8'
+export PYTHON_BIN_PATH="$(which ${TF_PYTHON_VERSION})"
+
+# .bazelrc does not have all config options listed within it for ROCm (as it does for others)
+# So explicitly calling the configure script here to properly setup the ROCm config options
+yes "" | TF_NEED_ROCM=1 ./configure
+
+# Setup some env vars needed by the underlying script(s) for building and testing
+export N_BUILD_JOBS=$(grep -c ^processor /proc/cpuinfo)
+export TF_GPU_COUNT=$(lspci|grep 'controller'|grep 'AMD/ATI'|wc -l)
+export TF_TESTS_PER_GPU=1
+export N_TEST_JOBS=$(expr ${TF_GPU_COUNT} \* ${TF_TESTS_PER_GPU})
+
+
+# Get the default test targets for bazel.
+source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh
+
+#Add filter tags specific to ROCm version
+rocm_major_version=`cat /opt/rocm/.info/version | cut -d "." -f 1`
+rocm_minor_version=`cat /opt/rocm/.info/version | cut -d "." -f 2`
+TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=""
+if [[ $rocm_major_version -ge 5 ]]; then
+	if [[ $rocm_minor_version -lt 3 ]]; then
+		TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=",no_rocm_pre_53"	
+	fi
+else
+    TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=",no_rocm_pre_53"	
+fi
+
+
+
+# # Export optional variables for running pip.sh
+export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_excluded,-oss_serial,-no_oss_py39,-no_rocm'${TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC}
+export TF_BUILD_FLAGS="--config=release_base "
+export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \
+ --test_env=TF2_BEHAVIOR=1 \
+ --local_test_jobs=${N_TEST_JOBS} \
+ --test_env=TF_GPU_COUNT=$TF_GPU_COUNT \
+ --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
+ --test_timeout 600,900,2400,7200 \
+ --build_tests_only \
+ --test_output=errors \
+ --test_sharding_strategy=disabled \
+ --define=no_tensorflow_py_deps=true \
+ --test_lang_filters=py \
+ --verbose_failures=true \
+ --keep_going "
+export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... "
+export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean"
+export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo.
+export TF_PROJECT_NAME="tensorflow_rocm"  # single pip package!
+export TF_PIP_TEST_ROOT="pip_test"
+
+# To build both tensorflow and tensorflow-gpu pip packages
+export TF_BUILD_BOTH_GPU_PACKAGES=0
+
+./tensorflow/tools/ci_build/builds/pip_new.sh

tensorflow/tools/ci_build/linux/rocm/rocm_py39_pip.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+set -e
+set -x
+
+source tensorflow/tools/ci_build/release/common.sh
+
+install_ubuntu_16_python_pip_deps python3.9
+
+install_bazelisk
+
+# Export required variables for running pip.sh
+export OS_TYPE="UBUNTU"
+export CONTAINER_TYPE="ROCM"
+export TF_PYTHON_VERSION='python3.9'
+export PYTHON_BIN_PATH="$(which ${TF_PYTHON_VERSION})"
+
+# .bazelrc does not have all config options listed within it for ROCm (as it does for others)
+# So explicitly calling the configure script here to properly setup the ROCm config options
+yes "" | TF_NEED_ROCM=1 ./configure
+
+# Setup some env vars needed by the underlying script(s) for building and testing
+export N_BUILD_JOBS=$(grep -c ^processor /proc/cpuinfo)
+export TF_GPU_COUNT=$(lspci|grep 'controller'|grep 'AMD/ATI'|wc -l)
+export TF_TESTS_PER_GPU=1
+export N_TEST_JOBS=$(expr ${TF_GPU_COUNT} \* ${TF_TESTS_PER_GPU})
+
+
+# Get the default test targets for bazel.
+source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh
+
+#Add filter tags specific to ROCm version
+rocm_major_version=`cat /opt/rocm/.info/version | cut -d "." -f 1`
+rocm_minor_version=`cat /opt/rocm/.info/version | cut -d "." -f 2`
+TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=""
+if [[ $rocm_major_version -ge 5 ]]; then
+	if [[ $rocm_minor_version -lt 3 ]]; then
+		TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=",no_rocm_pre_53"	
+	fi
+else
+    TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC=",no_rocm_pre_53"	
+fi
+
+# # Export optional variables for running pip.sh
+export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_excluded,-oss_serial,-no_oss_py39,-no_rocm'${TF_TEST_FILTER_TAGS_ROCM_VERSION_SPECIFIC}
+export TF_BUILD_FLAGS="--config=release_base "
+export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \
+ --test_env=TF2_BEHAVIOR=1 \
+ --local_test_jobs=${N_TEST_JOBS} \
+ --test_env=TF_GPU_COUNT=$TF_GPU_COUNT \
+ --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
+ --test_timeout 600,900,2400,7200 \
+ --build_tests_only \
+ --test_output=errors \
+ --test_sharding_strategy=disabled \
+ --define=no_tensorflow_py_deps=true \
+ --test_lang_filters=py \
+ --verbose_failures=true \
+ --keep_going "
+export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... "
+export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean"
+export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo.
+export TF_PROJECT_NAME="tensorflow_rocm"  # single pip package!
+export TF_PIP_TEST_ROOT="pip_test"
+
+# To build both tensorflow and tensorflow-gpu pip packages
+export TF_BUILD_BOTH_GPU_PACKAGES=0
+
+./tensorflow/tools/ci_build/builds/pip_new.sh

tensorflow/tools/ci_build/linux/rocm/run_cpu.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+set -e
+set -x
+
+N_BUILD_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_BUILD_JOBS} concurrent build job(s) and ${N_BUILD__JOBS} concurrent test job(s)."
+echo ""
+
+# Run configure.
+export PYTHON_BIN_PATH=`which python3`
+
+export TF_NEED_ROCM=0
+
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test \
+      -k \
+      --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-multi_gpu,-tpu,-no_rocm,-benchmark-test,-v1only \
+      --jobs=${N_BUILD_JOBS} \
+      --local_test_jobs=${N_BUILD_JOBS} \
+      --test_timeout 600,900,2400,7200 \
+      --build_tests_only \
+      --test_output=errors \
+      --test_sharding_strategy=disabled \
+      --test_size_filters=small,medium,large \
+      -- \
+      //tensorflow/... \
+      -//tensorflow/core/tpu/... \
+      -//tensorflow/lite/... \

tensorflow/tools/ci_build/linux/rocm/run_gpu_multi.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+set -e
+set -x
+
+N_BUILD_JOBS=$(grep -c ^processor /proc/cpuinfo)
+N_TEST_JOBS=1 # run tests serially
+
+echo ""
+echo "Bazel will use ${N_BUILD_JOBS} concurrent build job(s) and ${N_TEST_JOBS} concurrent test job(s)."
+echo ""
+
+# First positional argument (if any) specifies the ROCM_INSTALL_DIR
+ROCM_INSTALL_DIR=/opt/rocm-5.3.0
+if [[ -n $1 ]]; then
+    ROCM_INSTALL_DIR=$1
+fi
+
+# Run configure.
+export PYTHON_BIN_PATH=`which python3`
+
+export TF_NEED_ROCM=1
+export ROCM_PATH=$ROCM_INSTALL_DIR
+
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test \
+      --config=rocm \
+      -k \
+      --test_tag_filters=-no_gpu,-no_rocm \
+      --jobs=${N_BUILD_JOBS} \
+      --local_test_jobs=${N_TEST_JOBS} \
+      --test_timeout 600,900,2400,7200 \
+      --build_tests_only \
+      --test_output=errors \
+      --test_sharding_strategy=disabled \
+      --test_size_filters=small,medium,large \
+      --cache_test_results=no \
+      --test_env=TF_PER_DEVICE_MEMORY_LIMIT_MB=2048 \
+      -- \
+//tensorflow/core/common_runtime/gpu:gpu_device_unified_memory_test_2gpu \
+//tensorflow/core/kernels:collective_nccl_test_2gpu \
+//tensorflow/core/nccl:nccl_manager_test_2gpu \
+//tensorflow/python/distribute/integration_test:mwms_peer_failure_test_2gpu \
+//tensorflow/python/distribute:checkpoint_utils_test_2gpu \
+//tensorflow/python/distribute:checkpointing_test_2gpu \
+//tensorflow/python/distribute:collective_all_reduce_strategy_test_xla_2gpu \
+//tensorflow/python/distribute:custom_training_loop_gradient_test_2gpu \
+//tensorflow/python/distribute:custom_training_loop_input_test_2gpu \
+//tensorflow/python/distribute:distribute_utils_test_2gpu \
+//tensorflow/python/distribute:input_lib_test_2gpu \
+//tensorflow/python/distribute:input_lib_type_spec_test_2gpu \
+//tensorflow/python/distribute:metrics_v1_test_2gpu \
+//tensorflow/python/distribute:mirrored_variable_test_2gpu \
+//tensorflow/python/distribute:parameter_server_strategy_test_2gpu \
+//tensorflow/python/distribute:ps_values_test_2gpu \
+//tensorflow/python/distribute:random_generator_test_2gpu \
+//tensorflow/python/distribute:test_util_test_2gpu \
+//tensorflow/python/distribute:tf_function_test_2gpu \
+//tensorflow/python/distribute:vars_test_2gpu \
+//tensorflow/python/distribute:warm_starting_util_test_2gpu \
+//tensorflow/python/keras/distribute:collective_all_reduce_strategy_test_2gpu \
+//tensorflow/python/keras/distribute:collective_all_reduce_strategy_test_xla_2gpu \
+//tensorflow/python/keras/distribute:ctl_correctness_test_2gpu \
+//tensorflow/python/keras/distribute:custom_training_loop_optimizer_test_2gpu \
+//tensorflow/python/keras/distribute:keras_metrics_test_2gpu \
+//tensorflow/python/keras/distribute:keras_models_test_2gpu \
+//tensorflow/python/keras/distribute:keras_optimizer_v2_test_2gpu \
+//tensorflow/python/keras/distribute:keras_stateful_lstm_model_correctness_test_2gpu \
+//tensorflow/python/keras/distribute:mirrored_strategy_test_2gpu \
+//tensorflow/python/keras/distribute:mirrored_variable_test_2gpu \
+//tensorflow/python/keras/distribute:multi_worker_test_2gpu \
+//tensorflow/python/keras/layers/preprocessing:category_crossing_distribution_test_2gpu \
+//tensorflow/python/keras/layers/preprocessing:category_encoding_distribution_test_2gpu \
+//tensorflow/python/keras/layers/preprocessing:discretization_distribution_test_2gpu \
+//tensorflow/python/keras/layers/preprocessing:hashing_distribution_test_2gpu \
+//tensorflow/python/keras/layers/preprocessing:image_preprocessing_distribution_test_2gpu \
+//tensorflow/python/keras/layers/preprocessing:index_lookup_distribution_test_2gpu \
+//tensorflow/python/keras/layers/preprocessing:text_vectorization_distribution_test_2gpu \
+//tensorflow/python/kernel_tests:dynamic_partition_op_test_2gpu \
+//tensorflow/python/training:saver_test_2gpu \
+
+
+
+
+# no_rocm : //tensorflow/python/keras/distribute:keras_dnn_correctness_test_2gpu \
+# no_rocm : //tensorflow/python/keras/distribute:keras_embedding_model_correctness_test_2gpu \
+      
+# TIMEOUT : //tensorflow/python/distribute:values_test_2gpu \
+# TIMEOUT : //tensorflow/python/keras/distribute:keras_image_model_correctness_test_2gpu \
+# TIMEOUT : //tensorflow/python/keras/distribute:keras_rnn_model_correctness_test_2gpu \
+# TIMEOUT : //tensorflow/python/keras/distribute:saved_model_mixed_api_test_2gpu \
+# TIMEOUT : //tensorflow/python/keras/distribute:saved_model_save_load_test_2gpu \
+
+# Started timing-out with ROCm 4.1
+# TIMEOUT : //tensorflow/python/keras/distribute:keras_premade_models_test_2gpu \
+
+# Became FLAKY with  ROCm 4.1
+# FLAKY : //tensorflow/python/distribute:strategy_common_test_2gpu \
+# FLAKY : //tensorflow/python/distribute:strategy_common_test_xla_2gpu \
+# FLAKY : //tensorflow/python/distribute:strategy_gather_test_2gpu \
+# FLAKY : //tensorflow/python/distribute:strategy_gather_test_xla_2gpu \
+# FLAKY : //tensorflow/python/keras/distribute:custom_training_loop_metrics_test_2gpu \
+# FLAKY : //tensorflow/python/keras/distribute:custom_training_loop_models_test_2gpu \
+
+# FAILED : //tensorflow/python/distribute/v1:cross_device_ops_test_2gpu \
+# FAILED : //tensorflow/python/distribute:cross_device_ops_test_2gpu \
+# FAILED : //tensorflow/python/distribute:mirrored_strategy_test_2gpu \
+# FAILED : //tensorflow/python/keras/distribute:distribute_strategy_test_2gpu \
+# FAILED : //tensorflow/python/kernel_tests:collective_ops_test_2gpu \
+# FAILED : //tensorflow/python:collective_ops_gpu_test_2gpu \
+# FAILED : //tensorflow/python:nccl_ops_test_2gpu \
+
+# FAILED ON CI Node only : //tensorflow/python/distribute:collective_all_reduce_strategy_test_2gpu \
+# See run : http://ml-ci.amd.com:21096/job/tensorflow/job/github-prs-rocmfork-develop-upstream/job/rocm-latest-ubuntu-gpu-multi/216/console
+
+# FAILED ON CI Node only : //tensorflow/python/keras/distribute:keras_save_load_test_2gpu \
+# Starting with ROCm 4.1, see run : http://ml-ci.amd.com:21096/job/tensorflow/job/github-prs-rocmfork-develop-upstream/job/rocm-latest-ubuntu-gpu-multi/241/console
+
+# FAILED  //tensorflow/python/keras/distribute:minimize_loss_test_2gpu \
+# potential breaking commit : https://github.com/tensorflow/tensorflow/commit/74e39c8fa60079862597c9db506cd15b2443a5a2
+
+# NO MORE MULTI_GPU : //tensorflow/python/keras/distribute:checkpointing_test_2gpu \
+# multi_gpu tag was commented out in this commit : https://github.com/tensorflow/tensorflow/commit/b87d02a3f8d8b55045bf4250dd72e746357a3eba

tensorflow/tools/ci_build/linux/rocm/run_gpu_single.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+set -e
+set -x
+
+N_BUILD_JOBS=$(grep -c ^processor /proc/cpuinfo)
+TF_GPU_COUNT=$(lspci|grep 'controller'|grep 'AMD/ATI'|wc -l)
+TF_TESTS_PER_GPU=1
+N_TEST_JOBS=$(expr ${TF_GPU_COUNT} \* ${TF_TESTS_PER_GPU})
+
+echo ""
+echo "Bazel will use ${N_BUILD_JOBS} concurrent build job(s) and ${N_TEST_JOBS} concurrent test job(s)."
+echo ""
+
+# First positional argument (if any) specifies the ROCM_INSTALL_DIR
+ROCM_INSTALL_DIR=/opt/rocm-5.3.0
+if [[ -n $1 ]]; then
+    ROCM_INSTALL_DIR=$1
+fi
+
+# Run configure.
+export PYTHON_BIN_PATH=`which python3`
+
+export TF_NEED_ROCM=1
+export ROCM_PATH=$ROCM_INSTALL_DIR
+
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test \
+      --config=rocm \
+      -k \
+      --test_tag_filters=gpu,-no_oss,-oss_excluded,-oss_serial,-no_gpu,-no_rocm,-benchmark-test,-rocm_multi_gpu,-tpu,-v1only \
+      --jobs=${N_BUILD_JOBS} \
+      --local_test_jobs=${N_TEST_JOBS} \
+      --test_env=TF_GPU_COUNT=$TF_GPU_COUNT \
+      --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
+      --test_timeout 600,900,2400,7200 \
+      --build_tests_only \
+      --test_output=errors \
+      --test_sharding_strategy=disabled \
+      --test_size_filters=small,medium,large \
+      --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \
+      -- \
+      //tensorflow/... \
+      -//tensorflow/core/tpu/... \
+      -//tensorflow/lite/... \
+      -//tensorflow/compiler/tf2tensorrt/... \