wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/tools/ci_build/linux/mkl/Dockerfile.devel-mkl
Executable
+113
View File
@@ -0,0 +1,113 @@
# Copyright 2026 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
ARG ROOT_CONTAINER_TAG=devel
ARG ROOT_CONTAINER=tensorflow/tensorflow
FROM ${ROOT_CONTAINER}:${ROOT_CONTAINER_TAG}
LABEL maintainer="Clayne Robison <clayne.b.robison@intel.com>"
# These parameters can be overridden
ARG PYTHON="python"
ARG WHL_DIR="/tmp/pip"
ARG PIP="pip"
ARG TARGET_PLATFORM="haswell"
ARG CONFIG_V2_DISABLE=""
ARG CONFIG_BFLOAT16_BUILD=""
ARG ENABLE_SECURE_BUILD
ARG BAZEL_VERSION=""
ARG ENABLE_DNNL1=""
ARG ENABLE_HOROVOD=""
ARG ENABLE_GCC8=""
ARG OPENMPI_VERSION=""
ARG OPENMPI_DOWNLOAD_URL=""
ARG HOROVOD_VERSION=""
ARG INSTALL_HOROVOD_FROM_COMMIT=""
ARG BUILD_SSH=""
ARG TF_NIGHTLY_FLAG=""
ARG RELEASE_CONTAINER=""
ENV DEBIAN_FRONTEND=noninteractive
# Upgrade Bazel version if argument is passed
RUN if [ "${BAZEL_VERSION}" != "" ]; then \
curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
chmod +x bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
rm -rf bazel-$BAZEL_VERSION-installer-linux-x86_64.sh; \
fi
# Upgrade gcc-8 if argument is passed
RUN if [ "${ENABLE_GCC8}" = "yes" ]; then \
add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
apt-get update && \
apt-get install gcc-8 g++-8 -y && \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8 --slave /usr/bin/gcov gcov /usr/bin/gcov-8 && \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 70 --slave /usr/bin/g++ g++ /usr/bin/g++-7 --slave /usr/bin/gcov gcov /usr/bin/gcov-7 ;\
fi
# Download and build TensorFlow from the latest sources found in the root container
# make sure that if they pass in a tag, that it is loaded or we'll get an error
WORKDIR /
COPY tensorflow/ /tensorflow/
WORKDIR /tensorflow
RUN yes "" | ${PYTHON} configure.py
ENV CI_BUILD_PYTHON ${PYTHON}
# This script detects the version of gcc in the container, sets the appropriate
# compiler flags based on parameters
ADD set-build-env.py .
RUN ${PYTHON} set-build-env.py -p ${TARGET_PLATFORM} -f /root/.mkl.bazelrc \
${CONFIG_V2_DISABLE} ${ENABLE_SECURE_BUILD} ${CONFIG_BFLOAT16_BUILD} ${ENABLE_DNNL1}
# Pull the compiler flags we just wrote into root user's .bazelrc file
RUN echo "import /root/.mkl.bazelrc" >>/root/.bazelrc
# Install futures>=0.17.1 for Python2.7 compatibility mode
RUN ${PIP} install future>=0.17.1
RUN bazel --bazelrc=/root/.bazelrc build -c opt \
tensorflow/tools/pip_package:build_pip_package && \
bazel-bin/tensorflow/tools/pip_package/build_pip_package "${TF_NIGHTLY_FLAG}" "${WHL_DIR}" && \
${PIP} --no-cache-dir install --upgrade "${WHL_DIR}"/*.whl && \
rm -rf /root/.cache
# Clean up Bazel cache when done.
# Install OpenMPI/Horovod
COPY install_openmpi_horovod.sh .
RUN if [ "${ENABLE_HOROVOD}" = "yes" ]; then \
chmod +x install_openmpi_horovod.sh && \
OPENMPI_VERSION=${OPENMPI_VERSION} OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL} BUILD_SSH=${BUILD_SSH} \
INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT} HOROVOD_VERSION=${HOROVOD_VERSION} ./install_openmpi_horovod.sh && \
rm -rf install_openmpi_horovod.sh; \
fi
# Remove crypto python packages for software compliance check.
RUN if [ "${RELEASE_CONTAINER}" = "yes" ]; then \
${PIP} uninstall --yes cryptography && \
rm -rf /usr/lib/python3/dist-packages/pycrypto-2.6.1.egg-info; \
fi
# TensorBoard
EXPOSE 6006
# IPython
EXPOSE 8888
WORKDIR /root
tensorflow/tools/ci_build/linux/mkl/basic-mkl-gpu-test.sh
Executable
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Usage: basic_mkl_test.sh
# Helper function to traverse directories up until given file is found.
function upsearch () {
test / == "$PWD" && return || \
test -e "$1" && echo "$PWD" && return || \
cd .. && upsearch "$1"
}
# Set up WORKSPACE.
WORKSPACE="${WORKSPACE:-$(upsearch WORKSPACE)}"
BUILD_TAG=mkl-gpu-ci-test CI_BUILD_USER_FORCE_BADNAME=yes ${WORKSPACE}/tensorflow/tools/ci_build/ci_build.sh gpu tensorflow/tools/ci_build/linux/gpu/run_mkl.sh
tensorflow/tools/ci_build/linux/mkl/basic-mkl-test.sh
Executable
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env bash
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Usage: basic_mkl_test.sh
# Helper function to traverse directories up until given file is found.
function upsearch () {
test / == "$PWD" && return || \
test -e "$1" && echo "$PWD" && return || \
cd .. && upsearch "$1"
}
# Set up WORKSPACE.
WORKSPACE="${WORKSPACE:-$(upsearch WORKSPACE)}"
OMP_NUM_THREADS=${OMP_NUM_THREADS:-8}
if [[ ! -z ${OMP_NUM_THREADS} ]]; then
if [[ ${OMP_NUM_THREADS} -gt 112 ]] || [[ ${OMP_NUM_THREADS} < 0 ]]; then
>&2 echo "Usage: OMP_NUM_THREADS value should be between 0 and 112"
exit 1
fi
fi
BUILD_TAG=mkl-ci-test CI_BUILD_USER_FORCE_BADNAME=yes ${WORKSPACE}/tensorflow/tools/ci_build/ci_build.sh cpu OMP_NUM_THREADS=${OMP_NUM_THREADS} tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
Executable
+384
View File
@@ -0,0 +1,384 @@
#!/usr/bin/env bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Build a whl and container with Intel(R) MKL support
# Usage: build-dev-container.sh
DEBUG=1
DOCKER_BINARY="docker"
TMP_DIR=$(pwd)
# Helper function to traverse directories up until given file is found.
function upsearch () {
test / == "$PWD" && return || \
test -e "$1" && echo "$PWD" && return || \
cd .. && upsearch "$1"
}
function debug()
{
if [[ ${DEBUG} == 1 ]] ; then
echo $1
fi
}
function die()
{
echo $1
exit 1
}
# Set up WORKSPACE.
WORKSPACE="${WORKSPACE:-$(upsearch WORKSPACE)}"
ROOT_CONTAINER=${ROOT_CONTAINER:-tensorflow/tensorflow}
TF_ROOT_CONTAINER_TAG=${ROOT_CONTAINER_TAG:-devel}
# TF_BUILD_VERSION can be either a tag, branch, commit ID or PR number.
# For a PR, set TF_BUILD_VERSION_IS_PR="yes"
TF_BUILD_VERSION=${TF_DOCKER_BUILD_DEVEL_BRANCH:-master}
TF_BUILD_VERSION_IS_PR=${TF_DOCKER_BUILD_DEVEL_BRANCH_IS_PR:-no}
TF_REPO=${TF_REPO:-https://github.com/tensorflow/tensorflow}
FINAL_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME:-intel-mkl/tensorflow}
TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION:-nightly}
BUILD_AVX_CONTAINERS=${BUILD_AVX_CONTAINERS:-no}
BUILD_AVX2_CONTAINERS=${BUILD_AVX2_CONTAINERS:-no}
BUILD_SKX_CONTAINERS=${BUILD_SKX_CONTAINERS:-no}
BUILD_CLX_CONTAINERS=${BUILD_CLX_CONTAINERS:-no}
BUILD_ICX_CLIENT_CONTAINERS=${BUILD_ICX_CLIENT_CONTAINERS:-no}
BUILD_ICX_SERVER_CONTAINERS=${BUILD_ICX_SERVER_CONTAINERS:-no}
CONTAINER_PORT=${TF_DOCKER_BUILD_PORT:-8888}
BUILD_TF_V2_CONTAINERS=${BUILD_TF_V2_CONTAINERS:-yes}
BUILD_TF_BFLOAT16_CONTAINERS=${BUILD_TF_BFLOAT16_CONTAINERS:-no}
ENABLE_SECURE_BUILD=${ENABLE_SECURE_BUILD:-no}
BAZEL_VERSION=${BAZEL_VERSION}
BUILD_PY2_CONTAINERS=${BUILD_PY2_CONTAINERS:-no}
ENABLE_DNNL1=${ENABLE_DNNL1:-no}
ENABLE_HOROVOD=${ENABLE_HOROVOD:-no}
INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT:-no}
ENABLE_GCC8=${ENABLE_GCC8:-no}
OPENMPI_VERSION=${OPENMPI_VERSION}
OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL}
HOROVOD_VERSION=${HOROVOD_VERSION}
BUILD_SSH=${BUILD_SSH:-no}
IS_NIGHTLY=${IS_NIGHTLY:-no}
RELEASE_CONTAINER=${RELEASE_CONTAINER:-no}
debug "ROOT_CONTAINER=${ROOT_CONTAINER}"
debug "TF_ROOT_CONTAINER_TAG=${TF_ROOT_CONTAINER_TAG}"
debug "TF_BUILD_VERSION=${TF_BUILD_VERSION}"
debug "TF_BUILD_VERSION_IS_PR=${TF_BUILD_VERSION_IS_PR}"
debug "FINAL_IMAGE_NAME=${FINAL_IMAGE_NAME}"
debug "TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION}"
debug "BUILD_AVX_CONTAINERS=${BUILD_AVX_CONTAINERS}"
debug "BUILD_AVX2_CONTAINERS=${BUILD_AVX2_CONTAINERS}"
debug "BUILD_SKX_CONTAINERS=${BUILD_SKX_CONTAINERS}"
debug "BUILD_CLX_CONTAINERS=${BUILD_CLX_CONTAINERS}"
debug "BUILD_ICX_CLIENT_CONTAINERS=${BUILD_ICX_CLIENT_CONTAINERS}"
debug "BUILD_ICX_SERVER_CONTAINERS=${BUILD_ICX_SERVER_CONTAINERS}"
debug "BUILD_TF_V2_CONTAINERS=${BUILD_TF_V2_CONTAINERS}"
debug "BUILD_TF_BFLOAT16_CONTAINERS=${BUILD_TF_BFLOAT16_CONTAINERS}"
debug "ENABLE_SECURE_BUILD=${ENABLE_SECURE_BUILD}"
debug "TMP_DIR=${TMP_DIR}"
debug "BAZEL_VERSION=${BAZEL_VERSION}"
debug "ENABLE_GCC8=${ENABLE_GCC8}"
debug "BUILD_PY2_CONTAINERS=${BUILD_PY2_CONTAINERS}"
debug "ENABLE_DNNL1=${ENABLE_DNNL1}"
debug "ENABLE_HOROVOD=${ENABLE_HOROVOD}"
debug "INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT}"
debug "OPENMPI_VERSION=${OPENMPI_VERSION}"
debug "OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL}"
debug "HOROVOD_VERSION=${HOROVOD_VERSION}"
debug "BUILD_SSH=${BUILD_SSH}"
debug "IS_NIGHTLY=${IS_NIGHTLY}"
debug "RELEASE_CONTAINER=${RELEASE_CONTAINER}"
function build_container()
{
if [[ $# -lt 2 ]]; then
die "Usage: build_container <TEMP_IMAGE_NAME> <TF_DOCKER_BUILD_ARGS>."
fi
TEMP_IMAGE_NAME=${1}
debug "TEMP_IMAGE_NAME=${TEMP_IMAGE_NAME}"
shift
TF_DOCKER_BUILD_ARGS=("${@}")
# Add the proxy info build args. This will be later on passed to docker as
# --build-arg so that users behind corporate proxy can build the images
TF_DOCKER_BUILD_ARGS+=("--build-arg http_proxy=${http_proxy}")
TF_DOCKER_BUILD_ARGS+=("--build-arg https_proxy=${https_proxy}")
TF_DOCKER_BUILD_ARGS+=("--build-arg socks_proxy=${socks_proxy}")
TF_DOCKER_BUILD_ARGS+=("--build-arg no_proxy=${no_proxy}")
# In general having uppercase proxies is a good idea because different
# applications running inside Docker may only honor uppercase proxies
TF_DOCKER_BUILD_ARGS+=("--build-arg HTTP_PROXY=${HTTP_PROXY}")
TF_DOCKER_BUILD_ARGS+=("--build-arg HTTPS_PROXY=${HTTPS_PROXY}")
TF_DOCKER_BUILD_ARGS+=("--build-arg SOCKS_PROXY=${SOCKS_PROXY}")
TF_DOCKER_BUILD_ARGS+=("--build-arg NO_PROXY=${NO_PROXY}")
#Add --config=v2 build arg for TF v2
if [[ ${BUILD_TF_V2_CONTAINERS} == "no" ]]; then
TF_DOCKER_BUILD_ARGS+=("--build-arg CONFIG_V2_DISABLE=--disable-v2")
fi
#Add build arg for bfloat16 build
if [[ ${BUILD_TF_BFLOAT16_CONTAINERS} == "yes" ]]; then
TF_DOCKER_BUILD_ARGS+=("--build-arg CONFIG_BFLOAT16_BUILD=--enable-bfloat16")
fi
#Add build arg for Secure Build
if [[ ${ENABLE_SECURE_BUILD} == "yes" ]]; then
TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_SECURE_BUILD=--secure-build")
fi
# Add build arg for DNNL1
if [[ ${ENABLE_DNNL1} == "yes" ]]; then
TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_DNNL1=--enable-dnnl1")
fi
# BAZEL Version
if [[ ${BAZEL_VERSION} != "" ]]; then
TF_DOCKER_BUILD_ARGS+=("--build-arg BAZEL_VERSION=${BAZEL_VERSION}")
fi
# Add build arg for installing OpenMPI/Horovod
if [[ ${ENABLE_HOROVOD} == "yes" ]]; then
TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_HOROVOD=${ENABLE_HOROVOD}")
TF_DOCKER_BUILD_ARGS+=("--build-arg OPENMPI_VERSION=${OPENMPI_VERSION}")
TF_DOCKER_BUILD_ARGS+=("--build-arg OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL}")
TF_DOCKER_BUILD_ARGS+=("--build-arg HOROVOD_VERSION=${HOROVOD_VERSION}")
TF_DOCKER_BUILD_ARGS+=("--build-arg INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT}")
TF_DOCKER_BUILD_ARGS+=("--build-arg BUILD_SSH=${BUILD_SSH}")
fi
# Add build arg --nightly_flag for the nightly build
if [[ ${IS_NIGHTLY} == "yes" ]]; then
TF_DOCKER_BUILD_ARGS+=("--build-arg TF_NIGHTLY_FLAG=--nightly_flag")
fi
# Add build arg GCC8 install
TF_DOCKER_BUILD_ARGS+=("--build-arg ENABLE_GCC8=${ENABLE_GCC8}")
TF_DOCKER_BUILD_ARGS+=("--build-arg RELEASE_CONTAINER=${RELEASE_CONTAINER}")
# Perform docker build
debug "Building docker image with image name and tag: ${TEMP_IMAGE_NAME}"
CMD="${DOCKER_BINARY} build ${TF_DOCKER_BUILD_ARGS[@]} --no-cache --pull -t ${TEMP_IMAGE_NAME} -f Dockerfile.devel-mkl ."
debug "CMD=${CMD}"
${CMD}
if [[ $? == "0" ]]; then
debug "${DOCKER_BINARY} build of ${TEMP_IMAGE_NAME} succeeded"
else
die "FAIL: ${DOCKER_BINARY} build of ${TEMP_IMAGE_NAME} failed"
fi
}
function test_container()
{
if [[ "$#" != "1" ]]; then
die "Usage: ${FUNCNAME} <TEMP_IMAGE_NAME>"
fi
TEMP_IMAGE_NAME=${1}
# Make sure that there is no other containers of the same image running
if "${DOCKER_BINARY}" ps | grep -q "${TEMP_IMAGE_NAME}"; then
die "ERROR: It appears that there are docker containers of the image "\
"${TEMP_IMAGE_NAME} running. Please stop them before proceeding"
fi
# Start a docker container from the newly-built docker image
DOCKER_RUN_LOG="${TMP_DIR}/docker_run.log"
debug " Log file is at: ${DOCKER_RUN_LOG}"
debug "Running docker container from image ${TEMP_IMAGE_NAME}..."
RUN_CMD="${DOCKER_BINARY} run --rm -d -p ${CONTAINER_PORT}:${CONTAINER_PORT} ${TEMP_IMAGE_NAME} tail -f /dev/null 2>&1 > ${DOCKER_RUN_LOG}"
debug "RUN_CMD=${RUN_CMD}"
${RUN_CMD}
# Get the container ID
CONTAINER_ID=""
while [[ -z ${CONTAINER_ID} ]]; do
sleep 1
debug "Polling for container ID..."
CONTAINER_ID=$("${DOCKER_BINARY}" ps | grep "${TEMP_IMAGE_NAME}" | awk '{print $1}')
done
debug "ID of the running docker container: ${CONTAINER_ID}"
debug "Performing basic sanity checks on the running container..."
{
${DOCKER_BINARY} exec ${CONTAINER_ID} bash -c "${PYTHON} -c 'from tensorflow.python import _pywrap_util_port; print(_pywrap_util_port.IsMklEnabled())'"
echo "PASS: MKL enabled test in ${TEMP_IMAGE_NAME}"
} || {
${DOCKER_BINARY} exec ${CONTAINER_ID} bash -c "${PYTHON} -c 'from tensorflow.python import pywrap_tensorflow; print(pywrap_tensorflow.IsMklEnabled())'"
echo "PASS: Old MKL enabled in ${TEMP_IMAGE_NAME}"
} || {
die "FAIL: MKL enabled test in ${TEMP_IMAGE_NAME}"
}
# Test to check if horovod is installed successfully
if [[ ${ENABLE_HOROVOD} == "yes" ]]; then
debug "Test horovod in the container..."
${DOCKER_BINARY} exec ${CONTAINER_ID} bash -c "${PYTHON} -c 'import horovod.tensorflow as hvd;'"
if [[ $? == "0" ]]; then
echo "PASS: HOROVOD installation test in ${TEMP_IMAGE_NAME}"
else
die "FAIL: HOROVOD installation test in ${TEMP_IMAGE_NAME}"
fi
fi
# Stop the running docker container
sleep 1
"${DOCKER_BINARY}" stop --time=0 ${CONTAINER_ID}
}
function checkout_tensorflow()
{
if [[ "$#" != "3" ]]; then
die "Usage: ${FUNCNAME} <REPO_URL> <BRANCH/TAG/COMMIT-ID/PR-ID> <TF_BUILD_VERSION_IS_PR>"
fi
TF_REPO="${1}"
TF_BUILD_VERSION="${2}"
TF_BUILD_VERSION_IS_PR="${3}"
TENSORFLOW_DIR="tensorflow"
debug "Checking out ${TF_REPO}:${TF_BUILD_VERSION} into ${TENSORFLOW_DIR}"
# Clean any existing tensorflow sources
rm -rf "${TENSORFLOW_DIR}"
git clone ${TF_REPO} ${TENSORFLOW_DIR}
cd ${TENSORFLOW_DIR}
if [[ "${TF_BUILD_VERSION_IS_PR}" == "yes" ]]; then
# If TF_BUILD_VERSION is a PR number, then fetch first
git fetch origin pull/${TF_BUILD_VERSION}/head:pr-${TF_BUILD_VERSION}
git checkout pr-${TF_BUILD_VERSION}
else
git checkout ${TF_BUILD_VERSION}
fi
if [ $? -ne 0 ]; then
die "Unable to find ${TF_BUILD_VERSION} on ${TF_REPO}"
fi
cd ..
}
function tag_container()
{
# Apply the final image name and tag
TEMP_IMAGE_NAME="${1}"
FINAL_IMG="${2}"
DOCKER_VER=$("${DOCKER_BINARY}" version | grep Version | head -1 | awk '{print $NF}')
if [[ -z "${DOCKER_VER}" ]]; then
die "ERROR: Failed to determine ${DOCKER_BINARY} version"
fi
DOCKER_MAJOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 1)
DOCKER_MINOR_VER=$(echo "${DOCKER_VER}" | cut -d. -f 2)
FORCE_TAG=""
if [[ "${DOCKER_MAJOR_VER}" -le 1 ]] && \
[[ "${DOCKER_MINOR_VER}" -le 9 ]]; then
FORCE_TAG="--force"
fi
"${DOCKER_BINARY}" tag ${FORCE_TAG} "${TEMP_IMAGE_NAME}" "${FINAL_IMG}" || \
die "Failed to tag intermediate docker image ${TEMP_IMAGE_NAME} as ${FINAL_IMG}"
debug "Successfully tagged docker image: ${FINAL_IMG}"
}
PYTHON_VERSIONS=("python3")
if [[ ${BUILD_PY2_CONTAINERS} == "yes" ]]; then
PYTHON_VERSIONS+=("python")
fi
PLATFORMS=()
if [[ ${BUILD_AVX_CONTAINERS} == "yes" ]]; then
PLATFORMS+=("sandybridge")
fi
if [[ ${BUILD_AVX2_CONTAINERS} == "yes" ]]; then
PLATFORMS+=("haswell")
fi
if [[ ${BUILD_SKX_CONTAINERS} == "yes" ]]; then
PLATFORMS+=("skylake")
fi
if [[ ${BUILD_CLX_CONTAINERS} == "yes" ]]; then
PLATFORMS+=("icelake")
fi
if [[ ${BUILD_ICX_CLIENT_CONTAINERS} == "yes" ]]; then
PLATFORMS+=("icelake-client")
fi
if [[ ${BUILD_ICX_SERVER_CONTAINERS} == "yes" ]]; then
PLATFORMS+=("icelake-server")
fi
# Checking out sources needs to be done only once
checkout_tensorflow "${TF_REPO}" "${TF_BUILD_VERSION}" "${TF_BUILD_VERSION_IS_PR}"
for PLATFORM in "${PLATFORMS[@]}"
do
for PYTHON in "${PYTHON_VERSIONS[@]}"
do
# Clear the build args array
TF_DOCKER_BUILD_ARGS=("--build-arg TARGET_PLATFORM=${PLATFORM}")
TF_DOCKER_BUILD_ARGS+=("--build-arg ROOT_CONTAINER=${ROOT_CONTAINER}")
FINAL_TAG="${TF_DOCKER_BUILD_VERSION}"
ROOT_CONTAINER_TAG="${TF_ROOT_CONTAINER_TAG}"
if [[ ${PLATFORM} == "haswell" ]]; then
FINAL_TAG="${FINAL_TAG}-avx2"
fi
if [[ ${PLATFORM} == "skylake" ]]; then
FINAL_TAG="${FINAL_TAG}-avx512"
fi
if [[ ${PLATFORM} == "icelake" ]]; then
FINAL_TAG="${FINAL_TAG}-avx512-VNNI"
fi
if [[ ${PLATFORM} == "icelake-client" ]]; then
FINAL_TAG="${FINAL_TAG}-icx-client"
fi
if [[ ${PLATFORM} == "icelake-server" ]]; then
FINAL_TAG="${FINAL_TAG}-icx-server"
fi
# Add -devel-mkl to the image tag
FINAL_TAG="${FINAL_TAG}-devel-mkl"
if [[ "${PYTHON}" == "python3" ]]; then
TF_DOCKER_BUILD_ARGS+=("--build-arg WHL_DIR=/tmp/pip3")
TF_DOCKER_BUILD_ARGS+=("--build-arg PIP=pip3")
fi
TF_DOCKER_BUILD_ARGS+=("--build-arg PYTHON=${PYTHON}")
TF_DOCKER_BUILD_ARGS+=("--build-arg ROOT_CONTAINER_TAG=${ROOT_CONTAINER_TAG}")
# Intermediate image name with tag
TEMP_IMAGE_NAME="${USER}/tensorflow:${FINAL_TAG}"
build_container "${TEMP_IMAGE_NAME}" "${TF_DOCKER_BUILD_ARGS[@]}"
test_container "${TEMP_IMAGE_NAME}"
tag_container "${TEMP_IMAGE_NAME}" "${FINAL_IMAGE_NAME}:${FINAL_TAG}"
done
done
tensorflow/tools/ci_build/linux/mkl/install_openmpi_horovod.sh
Executable
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env bash
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Install OpenMPI, OpenSSH and Horovod during Intel(R) MKL container build
# Usage: install_openmpi_horovod.sh [OPENMPI_VERSION=<openmpi version>] [OPENMPI_DOWNLOAD_URL=<openmpi download url>]
# [HOROVOD_VERSION=<horovod version>]
set -e
# Set default
OPENMPI_VERSION=${OPENMPI_VERSION:-openmpi-2.1.1}
OPENMPI_DOWNLOAD_URL=${OPENMPI_DOWNLOAD_URL:-https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.gz}
INSTALL_HOROVOD_FROM_COMMIT=${INSTALL_HOROVOD_FROM_COMMIT:-no}
BUILD_SSH=${BUILD_SSH:-no}
HOROVOD_VERSION=${HOROVOD_VERSION:-0.19.1}
SSH_CONFIG_PATH=/etc/ssh
# Install Open MPI
echo "Installing OpenMPI version ${OPENMPI_VERSION} ..."
echo "OpenMPI Download url ${OPENMPI_DOWNLOAD_URL} ..."
mkdir /tmp/openmpi
cd /tmp/openmpi
curl -fSsL -O ${OPENMPI_DOWNLOAD_URL}
tar zxf ${OPENMPI_VERSION}.tar.gz
cd ${OPENMPI_VERSION}
./configure --enable-mpirun-prefix-by-default
make -j $(nproc) all
make install
ldconfig
cd /
rm -rf /tmp/openmpi
# Create a wrapper for OpenMPI to allow running as root by default
mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real
echo '#!/bin/bash' > /usr/local/bin/mpirun
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun
chmod a+x /usr/local/bin/mpirun
# Configure OpenMPI to run good defaults:
echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
# Check mpi version
echo 'OpenMPI version:'
mpirun --version
# Install OpenSSH for MPI to communicate between containers
if [[ ${BUILD_SSH} == "yes" ]]; then
mkdir /tmp/buildssh
cd /tmp/buildssh && curl -fSsL -O http://www.zlib.net/zlib-1.2.11.tar.gz && tar -xzvf zlib-1.2.11.tar.gz && \
cd /tmp/buildssh/zlib-1.2.11 && ./configure && make && make install
cd /tmp/buildssh && curl -fSsL -O https://www.openssl.org/source/openssl-1.1.1.tar.gz && tar -xzvf openssl-1.1.1.tar.gz && \
cd /tmp/buildssh/openssl-1.1.1 && ./config && make && make test && make install
cd /tmp/buildssh && curl -fSsL -O https://mirrors.sonic.net/pub/OpenBSD/OpenSSH/portable/openssh-8.4p1.tar.gz && \
tar -xzvf openssh-8.4p1.tar.gz && cd /tmp/buildssh/openssh-8.4p1 && \
./configure --with-md5-passwords && make && \
groupadd sshd && useradd -M -g sshd -c 'sshd privsep' -d /var/empty -s /sbin/nologin sshd && passwd -l sshd && \
make install
apt-get clean && apt-get update && \
apt-get install -y --no-install-recommends --fix-missing \
libnuma-dev cmake
SSH_CONFIG_PATH=/usr/local/etc
else
apt-get clean && apt-get update && \
apt-get install -y --no-install-recommends --fix-missing \
openssh-client openssh-server libnuma-dev cmake && \
rm -rf /var/lib/apt/lists/*
if [[ $? == "0" ]]; then
echo "PASS: OpenSSH installation"
else
yum -y update && yum -y install numactl-devel openssh-server openssh-clients cmake && \
yum clean all
if [[ $? == "0" ]]; then
echo "PASS: OpenSSH installation"
else
echo "Unsupported Linux distribution. Aborting!" && exit 1
fi
fi
fi
mkdir -p /var/run/sshd
grep -v StrictHostKeyChecking ${SSH_CONFIG_PATH}/ssh_config > ${SSH_CONFIG_PATH}/ssh_config.new
# Allow OpenSSH to talk to containers without asking for confirmation
echo " StrictHostKeyChecking no" >> ${SSH_CONFIG_PATH}/ssh_config.new
mv ${SSH_CONFIG_PATH}/ssh_config.new ${SSH_CONFIG_PATH}/ssh_config
# Install Horovod
if [[ ${INSTALL_HOROVOD_FROM_COMMIT} == "yes" ]]; then
HOROVOD_WITH_TENSORFLOW=1
python3 -m pip install --no-cache-dir git+https://github.com/horovod/horovod.git@${HOROVOD_VERSION}
else
HOROVOD_WITH_TENSORFLOW=1
python3 -m pip install --no-cache-dir horovod==${HOROVOD_VERSION}
fi
tensorflow/tools/ci_build/linux/mkl/set-build-env.py
+381
View File
@@ -0,0 +1,381 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Configure build environment for certain Intel platforms."""
import argparse
import os
import subprocess
BASIC_BUILD_OPTS = ["--cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0", "--copt=-O3"]
SECURE_BUILD_OPTS = [
"--copt=-Wformat", "--copt=-Wformat-security", "--copt=-fstack-protector",
"--copt=-fPIC", "--copt=-fpic", "--linkopt=-znoexecstack",
"--linkopt=-zrelro", "--linkopt=-znow", "--linkopt=-fstack-protector"
]
class IntelPlatform(object):
min_gcc_major_version_ = 0
min_gcc_minor_version_ = 0
host_gcc_major_version_ = 0
host_gcc_minor_version_ = 0
BAZEL_PREFIX_ = "--copt="
ARCH_PREFIX_ = "-march="
FLAG_PREFIX_ = "-m"
def __init__(self, min_gcc_major_version, min_gcc_minor_version):
self.min_gcc_minor_version_ = min_gcc_minor_version
self.min_gcc_major_version_ = min_gcc_major_version
# Return True or False depending on whether
# The platform optimization flags can be generated by
# the gcc version specified in the parameters
def set_host_gcc_version(self, gcc_major_version, gcc_minor_version):
# True only if the gcc version in the tuple is >=
# min_gcc_major_version_, min_gcc_minor_version_
if gcc_major_version < self.min_gcc_major_version_:
print("Your MAJOR version of GCC is too old: {}; "
"it must be at least {}.{}".format(gcc_major_version,
self.min_gcc_major_version_,
self.min_gcc_minor_version_))
return False
elif gcc_major_version == self.min_gcc_major_version_ and \
gcc_minor_version < self.min_gcc_minor_version_:
print("Your MINOR version of GCC is too old: {}; "
"it must be at least {}.{}".format(gcc_minor_version,
self.min_gcc_major_version_,
self.min_gcc_minor_version_))
return False
print("gcc version OK: {}.{}".format(gcc_major_version, gcc_minor_version))
self.host_gcc_major_version_ = gcc_major_version
self.host_gcc_minor_version_ = gcc_minor_version
return True
# return a string with all the necessary bazel formatted flags for this
# platform in this gcc environment
def get_bazel_gcc_flags(self):
raise NotImplementedError(self)
# Returns True if the host gcc version is older than the gcc version in which
# the new march flag became available.
# Specify the version in which the new name usage began
def use_old_arch_names(self, gcc_new_march_major_version,
gcc_new_march_minor_version):
if self.host_gcc_major_version_ < gcc_new_march_major_version:
return True
elif self.host_gcc_major_version_ == gcc_new_march_major_version and \
self.host_gcc_minor_version_ < gcc_new_march_minor_version:
return True
return False
class NehalemPlatform(IntelPlatform):
def __init__(self):
IntelPlatform.__init__(self, 4, 8)
def get_bazel_gcc_flags(self):
NEHALEM_ARCH_OLD = "corei7"
NEHALEM_ARCH_NEW = "nehalem"
if self.use_old_arch_names(4, 9):
return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
NEHALEM_ARCH_OLD + " "
else:
return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
NEHALEM_ARCH_NEW + " "
class SandyBridgePlatform(IntelPlatform):
def __init__(self):
IntelPlatform.__init__(self, 4, 8)
def get_bazel_gcc_flags(self):
SANDYBRIDGE_ARCH_OLD = "corei7-avx"
SANDYBRIDGE_ARCH_NEW = "sandybridge"
if self.use_old_arch_names(4, 9):
return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
SANDYBRIDGE_ARCH_OLD + " "
else:
return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
SANDYBRIDGE_ARCH_NEW + " "
class HaswellPlatform(IntelPlatform):
def __init__(self):
IntelPlatform.__init__(self, 4, 8)
def get_bazel_gcc_flags(self):
HASWELL_ARCH_OLD = "core-avx2" # Only missing the POPCNT instruction
HASWELL_ARCH_NEW = "haswell"
POPCNT_FLAG = "popcnt"
if self.use_old_arch_names(4, 9):
ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
HASWELL_ARCH_OLD + " "
return ret_val + self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + \
POPCNT_FLAG + " "
else:
return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
HASWELL_ARCH_NEW + " "
class SkylakePlatform(IntelPlatform):
def __init__(self):
IntelPlatform.__init__(self, 4, 9)
def get_bazel_gcc_flags(self):
SKYLAKE_ARCH_OLD = "broadwell" # Only missing the POPCNT instruction
SKYLAKE_ARCH_NEW = "skylake-avx512"
# the flags that broadwell is missing: pku, clflushopt, clwb, avx512vl,
# avx512bw, avx512dq. xsavec and xsaves are available in gcc 5.x
# but for now, just exclude them.
AVX512_FLAGS = ["avx512f", "avx512cd"]
if self.use_old_arch_names(6, 1):
ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
SKYLAKE_ARCH_OLD + " "
for flag in AVX512_FLAGS:
ret_val += self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + flag + " "
return ret_val
else:
return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
SKYLAKE_ARCH_NEW + " "
class CascadelakePlatform(IntelPlatform):
def __init__(self):
IntelPlatform.__init__(self, 8, 3)
def get_bazel_gcc_flags(self):
CASCADELAKE_ARCH_OLD = "skylake-avx512" # Only missing the POPCNT instruction
CASCADELAKE_ARCH_NEW = "cascadelake"
# the flags that broadwell is missing: pku, clflushopt, clwb, avx512vl, avx512bw, avx512dq
VNNI_FLAG = "avx512vnni"
if IntelPlatform.use_old_arch_names(self, 9, 1):
ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
CASCADELAKE_ARCH_OLD + " "
return ret_val + self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + \
VNNI_FLAG + " "
else:
return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
CASCADELAKE_ARCH_NEW + " "
class IcelakeClientPlatform(IntelPlatform):
def __init__(self):
IntelPlatform.__init__(self, 8, 4)
def get_bazel_gcc_flags(self):
ICELAKE_ARCH_OLD = "skylake-avx512"
ICELAKE_ARCH_NEW = "icelake-client"
AVX512_FLAGS = ["avx512f", "avx512cd"]
if IntelPlatform.use_old_arch_names(self, 8, 4):
ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
ICELAKE_ARCH_OLD + " "
for flag in AVX512_FLAGS:
ret_val += self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + flag + " "
return ret_val
else:
return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
ICELAKE_ARCH_NEW + " "
class IcelakeServerPlatform(IntelPlatform):
def __init__(self):
IntelPlatform.__init__(self, 8, 4)
def get_bazel_gcc_flags(self):
ICELAKE_ARCH_OLD = "skylake-avx512"
ICELAKE_ARCH_NEW = "icelake-server"
AVX512_FLAGS = ["avx512f", "avx512cd"]
if IntelPlatform.use_old_arch_names(self, 8, 4):
ret_val = self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
ICELAKE_ARCH_OLD + " "
for flag in AVX512_FLAGS:
ret_val += self.BAZEL_PREFIX_ + self.FLAG_PREFIX_ + flag + " "
return ret_val
else:
return self.BAZEL_PREFIX_ + self.ARCH_PREFIX_ + \
ICELAKE_ARCH_NEW + " "
class BuildEnvSetter(object):
"""Prepares the proper environment settings for various Intel platforms."""
default_platform_ = "haswell"
PLATFORMS_ = {
"nehalem": NehalemPlatform(),
"sandybridge": SandyBridgePlatform(),
"haswell": HaswellPlatform(),
"skylake": SkylakePlatform(),
"cascadelake": CascadelakePlatform(),
"icelake-client": IcelakeClientPlatform(),
"icelake-server": IcelakeServerPlatform(),
}
def __init__(self):
self.args = None
self.bazel_flags_ = "build "
self.target_platform_ = None
# Return a tuple of the current gcc version
def get_gcc_version(self):
gcc_major_version = 0
gcc_minor_version = 0
# check to see if gcc is present
gcc_path = ""
gcc_path_cmd = "command -v gcc"
try:
gcc_path = subprocess.check_output(gcc_path_cmd, shell=True,
stderr=subprocess.STDOUT).\
strip()
print("gcc located here: {}".format(gcc_path))
if not os.access(gcc_path, os.F_OK | os.X_OK):
raise ValueError(
"{} does not exist or is not executable.".format(gcc_path))
gcc_output = subprocess.check_output(
[gcc_path, "-dumpfullversion", "-dumpversion"],
stderr=subprocess.STDOUT).strip()
# handle python2 vs 3 (bytes vs str type)
if isinstance(gcc_output, bytes):
gcc_output = gcc_output.decode("utf-8")
print("gcc version: {}".format(gcc_output))
gcc_info = gcc_output.split(".")
gcc_major_version = int(gcc_info[0])
gcc_minor_version = int(gcc_info[1])
except subprocess.CalledProcessException as e:
print("Problem getting gcc info: {}".format(e))
gcc_major_version = 0
gcc_minor_version = 0
return gcc_major_version, gcc_minor_version
def parse_args(self):
"""Set up argument parser, and parse CLI args."""
arg_parser = argparse.ArgumentParser(
description="Parse the arguments for the "
"TensorFlow build environment "
" setter")
arg_parser.add_argument(
"--disable-mkl",
dest="disable_mkl",
help="Turn off MKL. By default the compiler flag "
"--config=mkl is enabled.",
action="store_true")
arg_parser.add_argument(
"--disable-v2",
dest="disable_v2",
help="Build TensorFlow v1 rather than v2. By default the "
" compiler flag --config=v2 is enabled.",
action="store_true")
arg_parser.add_argument(
"--enable-bfloat16",
dest="enable_bfloat16",
help="Enable bfloat16 build. By default it is "
" disabled if no parameter is passed.",
action="store_true")
arg_parser.add_argument(
"--enable-dnnl1",
dest="enable_dnnl1",
help="Enable dnnl1 build. By default it is "
" disabled if no parameter is passed.",
action="store_true")
arg_parser.add_argument(
"-s",
"--secure-build",
dest="secure_build",
help="Enable secure build flags.",
action="store_true")
arg_parser.add_argument(
"-p",
"--platform",
choices=self.PLATFORMS_.keys(),
help="The target platform.",
dest="target_platform",
default=self.default_platform_)
arg_parser.add_argument(
"-f",
"--bazelrc-file",
dest="bazelrc_file",
help="The full path to the bazelrc file into which "
"the build command will be written. The path "
"will be relative to the container "
" environment.",
required=True)
self.args = arg_parser.parse_args()
def validate_args(self):
# Check the bazelrc file
if os.path.exists(self.args.bazelrc_file):
if os.path.isfile(self.args.bazelrc_file):
self._debug("The file {} exists and will be deleted.".format(
self.args.bazelrc_file))
elif os.path.isdir(self.args.bazelrc_file):
print("You can't write bazel config to \"{}\" "
"because it is a directory".format(self.args.bazelrc_file))
return False
# Validate gcc with the requested platform
gcc_major_version, gcc_minor_version = self.get_gcc_version()
if gcc_major_version == 0 or \
not self.target_platform_.set_host_gcc_version(
gcc_major_version, gcc_minor_version):
return False
return True
def set_build_args(self):
"""Generate Bazel build flags."""
for flag in BASIC_BUILD_OPTS:
self.bazel_flags_ += "{} ".format(flag)
if self.args.secure_build:
for flag in SECURE_BUILD_OPTS:
self.bazel_flags_ += "{} ".format(flag)
if not self.args.disable_mkl:
self.bazel_flags_ += "--config=mkl "
if self.args.disable_v2:
self.bazel_flags_ += "--config=v1 "
if self.args.enable_dnnl1:
self.bazel_flags_ += "--define build_with_mkl_dnn_v1_only=true "
if self.args.enable_bfloat16:
self.bazel_flags_ += "--copt=-DENABLE_INTEL_MKL_BFLOAT16 "
self.bazel_flags_ += self.target_platform_.get_bazel_gcc_flags()
def write_build_args(self):
self._debug("Writing build flags: {}".format(self.bazel_flags_))
with open(self.args.bazelrc_file, "w") as f:
f.write(self.bazel_flags_ + "\n")
def _debug(self, msg):
print(msg)
def go(self):
self.parse_args()
self.target_platform_ = self.PLATFORMS_.get(self.args.target_platform)
if self.validate_args():
self.set_build_args()
self.write_build_args()
else:
print("Error.")
env_setter = BuildEnvSetter()
env_setter.go()