wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 6 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/tools/tf_sig_build_dockerfiles/Dockerfile
+86
View File
@@ -0,0 +1,86 @@
# Copyright 2026 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
################################################################################
FROM ubuntu:22.04@sha256:104ae83764a5119017b8e8d6218fa0832b09df65aae7d5a6de29a85d813da2fb as builder
################################################################################
# Install devtoolset build dependencies
COPY setup.packages.sh setup.packages.sh
COPY builder.packages.txt builder.packages.txt
RUN /setup.packages.sh /builder.packages.txt
# Install devtoolset-9 in /dt9 with glibc 2.17 and libstdc++ 4.8, for building
# manylinux2014-compatible packages.
COPY builder.devtoolset/fixlinks.sh /fixlinks.sh
COPY builder.devtoolset/rpm-patch.sh /rpm-patch.sh
COPY builder.devtoolset/build_devtoolset.sh /build_devtoolset.sh
COPY builder.devtoolset/glibc2.17-inline.patch /glibc2.17-inline.patch
RUN /build_devtoolset.sh devtoolset-9 /dt9
################################################################################
FROM nvidia/cuda:12.3.2-base-ubuntu22.04@sha256:8cecfe099315f73127d6d5cc43fce32c7ffff4ea0460eefac48f2b7d811ce857 as devel
################################################################################
COPY --from=builder /dt9 /dt9
# Install required development packages but delete unneeded CUDA bloat
# CUDA must be cleaned up in the same command to prevent Docker layer bloating
COPY setup.sources.sh /setup.sources.sh
COPY setup.packages.sh /setup.packages.sh
COPY setup.cuda.sh /setup.cuda.sh
COPY devel.packages.txt /devel.packages.txt
RUN /setup.sources.sh && /setup.packages.sh /devel.packages.txt && /setup.cuda.sh
# Make sure clang is on the path
RUN ln -s /usr/lib/llvm-18/bin/clang /usr/bin/clang
# Install various tools.
# - bats: bash unit testing framework
# - bazelisk: always use the correct bazel version
# - buildifier: clean bazel build deps
# - buildozer: clean bazel build deps
# - gcloud SDK: communicate with Google Cloud Platform (GCP) for RBE, CI
# - patchelf: Utility tool to modify existing ELF executables and libraries
RUN git clone --branch v1.7.0 https://github.com/bats-core/bats-core.git && bats-core/install.sh /usr/local && rm -rf bats-core
RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 -O /usr/local/bin/bazel && chmod +x /usr/local/bin/bazel
RUN wget https://github.com/bazelbuild/buildtools/releases/download/3.5.0/buildifier -O /usr/local/bin/buildifier && chmod +x /usr/local/bin/buildifier
RUN wget https://github.com/bazelbuild/buildtools/releases/download/3.5.0/buildozer -O /usr/local/bin/buildozer && chmod +x /usr/local/bin/buildozer
RUN curl -sSL https://sdk.cloud.google.com > /tmp/gcloud && bash /tmp/gcloud --install-dir=~/usr/local/bin --disable-prompts
# Download and install patchelf v0.18.0 from GitHub. The default Ubuntu focal
# packages only provide the "0.10-2build1" version. We use patchelf to manipulate
# certain shared libraries during the wheel building process (https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/pip_package/build_pip_package.sh#L255-L262).
# When we use Patchelf versions <0.12, those shared libraries end up with a
# corrupted PT_NOTE program header. This was fixed in v0.12, see https://github.com/NixOS/patchelf/commit/43a33482b501b0f5ee9da312aabfca3806570cc9.
RUN wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz && tar -zxvf patchelf-0.18.0-x86_64.tar.gz -C /usr && rm -rf patchelf-0.18.0-x86_64.tar.gz
# All lines past this point are reset when $CACHEBUSTER is set. We need this
# for Python specifically because we install some nightly packages which are
# likely to change daily.
ARG CACHEBUSTER=0
RUN echo $CACHEBUSTER
# Setup Python environment. PYTHON_VERSION is e.g. "python3.8"
ARG PYTHON_VERSION
COPY setup.python.sh /setup.python.sh
COPY devel.requirements.txt /devel.requirements.txt
RUN /setup.python.sh $PYTHON_VERSION devel.requirements.txt
# Setup build and environment
COPY devel.usertools /usertools
COPY devel.bashrc /root/.bashrc
# Don't use the bazel cache when a new docker image is created.
RUN echo build --action_env=DOCKER_CACHEBUSTER=$(date +%s%N)$RANDOM >> /etc/bazel.bazelrc
RUN echo build --host_action_env=DOCKER_HOST_CACHEBUSTER=$(date +%s%N)$RANDOM >> /etc/bazel.bazelrc
tensorflow/tools/tf_sig_build_dockerfiles/README.md
+542
View File
@@ -0,0 +1,542 @@
# TF SIG Build Dockerfiles
Standard Dockerfiles for TensorFlow builds, used internally at Google.
Maintainer: @angerson (TensorFlow OSS DevInfra; SIG Build)
* * *
These docker containers are for building and testing TensorFlow in CI
environments (and for users replicating those CI builds). They are openly
developed in TF SIG Build, verified by Google developers, and published to
tensorflow/build on [Docker Hub](https://hub.docker.com/r/tensorflow/build/).
The TensorFlow OSS DevInfra team uses these containers for most of our
Linux-based CI, including `tf-nightly` tests and Pip packages and TF release
packages for TensorFlow 2.9 onwards.
## Tags
These Dockerfiles are built and deployed to [Docker
Hub](https://hub.docker.com/r/tensorflow/build/) via [Github
Actions](https://github.com/tensorflow/tensorflow/blob/master/.github/workflows/sigbuild-docker.yml).
The tags are defined as such:
- The `latest` tags are kept up-to-date to build TensorFlow's `master` branch.
- The `version number` tags target the corresponding TensorFlow version. We
continuously build the `current-tensorflow-version + 1` tag, so when a new
TensorFlow branch is cut, that Dockerfile is frozen to support that branch.
- We support the same Python versions that TensorFlow does.
## Updating the Containers
For simple changes, you can adjust the source files and then make a PR. Send it
to @angerson for review. We have presubmits that will make sure your change
still builds a container. After approval and submission, our GitHub Actions
workflow deploys the containers to Docker Hub.
- To update Python packages, look at `devel.requirements.txt`
- To update system packages, look at `devel.packages.txt`
- To update the way `bazel build` works, look at `devel.usertools/*.bazelrc`.
To rebuild the containers locally after making changes, use this command from
this directory:
```bash
DOCKER_BUILDKIT=1 docker build \
--build-arg PYTHON_VERSION=python3.10 --target=devel -t my-tf-devel .
```
It will take a long time to build devtoolset and install CUDA packages. After
it's done, you can use the commands below to test your changes. Just replace
`tensorflow/build:latest-python3.10` with `my-tf-devel` to use your image
instead.
### Automatic GCR.io Builds for Presubmits
TensorFlow team members (i.e. Google employees) can apply a `Build and deploy
to gcr.io for staging` tag to their PRs to the Dockerfiles, as long as the PR
is being developed on a branch of this repository, not a fork. Unfortunately
this is not available for non-Googler contributors for security reasons.
## Run the TensorFlow Team's Nightly Test Suites with Docker
The TensorFlow DevInfra team runs a daily test suite that builds `tf-nightly`
and runs a `bazel test` suite on both the Pip package (the "pip" tests) and
on the source code itself (the "nonpip" tests). These test scripts are often
referred to as "The Nightly Tests" and can be a common reason for a TF PR to be
reverted. The build scripts aren't visible to external users, but they use
the configuration files which are included in these containers. Our test suites,
which include the build of `tf-nightly`, are easy to replicate with these
containers, and here is how you can do it.
Presubmits are not using these containers... yet.
Here are some important notes to keep in mind:
- The Ubuntu CI jobs that build the `tf-nightly` package build at the GitHub
`nightly` tag. You can see the specific commit of a `tf-nightly` package on
pypi.org in `tf.version.GIT_VERSION`, which will look something like
`v1.12.1-67282-g251085598b7`. The final section, `g251085598b7`, is a short
git hash.
- If you interrupt a `docker exec` command with `ctrl-c`, you will get your
shell back but the command will continue to run. You cannot reattach to it,
but you can kill it with `docker kill tf` (or `docker kill the-container-name`).
This will destroy your container but will not harm your work since it's
mounted. If you have any suggestions for handling this better, let us know.
Now let's build `tf-nightly`.
1. Set up your directories:
- A directory with the TensorFlow source code, e.g. `/tmp/tensorflow`
- A directory for TensorFlow packages built in the container, e.g. `/tmp/packages`
- A directory for your local bazel cache (can be empty), e.g. `/tmp/bazelcache`
2. Choose the Docker container to use from [Docker
Hub](https://hub.docker.com/r/tensorflow/build/tags). The options for the
`master` branch are:
- `tensorflow/build:latest-python3.12`
- `tensorflow/build:latest-python3.11`
- `tensorflow/build:latest-python3.10`
For this example we'll use `tensorflow/build:latest-python3.10`.
3. Pull the container you decided to use.
```bash
docker pull tensorflow/build:latest-python3.10
```
4. Start a backgrounded Docker container with the three folders mounted.
- Mount the TensorFlow source code to `/tf/tensorflow`.
- Mount the directory for built packages to `/tf/pkg`.
- Mount the bazel cache to `/tf/cache`. You don't need `/tf/cache` if
you're going to use the remote cache.
Here are the arguments we're using:
- `--name tf`: Names the container `tf` so we can refer to it later.
- `-w /tf/tensorflow`: All commands run in the `/tf/tensorflow` directory,
where the TF source code is.
- `-it`: Makes the container interactive for running commands
- `-d`: Makes the container start in the background, so we can send
commands to it instead of running commands from inside.
And `-v` is for mounting directories into the container.
```bash
docker run --name tf -w /tf/tensorflow -it -d \
--env TF_PYTHON_VERSION=3.10 \
-v "/tmp/packages:/tf/pkg" \
-v "/tmp/tensorflow:/tf/tensorflow" \
-v "/tmp/bazelcache:/tf/cache" \
tensorflow/build:latest-python3.10 \
bash
```
Note: if you wish to use your own Google Cloud Platform credentials for
e.g. RBE, you may also wish to set `-v
$HOME/.config/gcloud:/root/.config/gcloud` to make your credentials
available to bazel. You don't need to do this unless you know what you're
doing.
Now you can continue on to any of:
- Build `tf-nightly` and then (optionally) run a test suite on the pip package
(the "pip" suite)
- Run a test suite on the TF code directly (the "nonpip" suite)
- Build the libtensorflow packages (the "libtensorflow" suite)
- Run a code-correctness check (the "code_check" suite)
### Build `tf-nightly` and run Pip tests
1. Apply the `update_version.py` script that changes the TensorFlow version to
`X.Y.Z.devYYYYMMDD`. This is used for `tf-nightly` on PyPI and is technically
optional.
```bash
docker exec tf python3 tensorflow/tools/ci_build/update_version.py --nightly
```
2. Build TensorFlow by following the instructions under one of the collapsed
sections below. You can build both CPU and GPU packages without a GPU. TF
DevInfra's remote cache is better for building TF only once, but if you
build over and over, it will probably be better in the long run to use a
local cache. We're not sure about which is best for most users, so let us
know on [Gitter](https://gitter.im/tensorflow/sig-build).
This step will take a long time, since you're building TensorFlow. GPU takes
much longer to build. Choose one and click on the arrow to expand the
commands:
<details><summary>TF Nightly CPU - Remote Cache</summary>
Build the sources with Bazel:
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
build --config=sigbuild_remote_cache \
tensorflow/tools/pip_package:build_pip_package
```
And then construct the pip package:
```
docker exec tf \
./bazel-bin/tensorflow/tools/pip_package/build_pip_package \
/tf/pkg \
--cpu \
--nightly_flag
```
</details>
<details><summary>TF Nightly GPU - Remote Cache</summary>
Build the sources with Bazel:
```
docker exec tf bazel --bazelrc=/usertools/gpu.bazelrc \
build --config=sigbuild_remote_cache \
tensorflow/tools/pip_package:build_pip_package
```
And then construct the pip package:
```
docker exec tf \
./bazel-bin/tensorflow/tools/pip_package/build_pip_package \
/tf/pkg \
--nightly_flag
```
</details>
<details><summary>TF Nightly CPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
Build the sources with Bazel:
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
build --config=sigbuild_local_cache \
tensorflow/tools/pip_package:build_pip_package
```
And then construct the pip package:
```
docker exec tf \
./bazel-bin/tensorflow/tools/pip_package/build_pip_package \
/tf/pkg \
--cpu \
--nightly_flag
```
</details>
<details><summary>TF Nightly GPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
Build the sources with Bazel:
```
docker exec tf \
bazel --bazelrc=/usertools/gpu.bazelrc \
build --config=sigbuild_local_cache \
tensorflow/tools/pip_package:build_pip_package
```
And then construct the pip package:
```
docker exec tf \
./bazel-bin/tensorflow/tools/pip_package/build_pip_package \
/tf/pkg \
--nightly_flag
```
</details>
3. Run the helper script that checks for manylinux compliance, renames the
wheels, and then checks the size of the packages.
```
docker exec tf /usertools/rename_and_verify_wheels.sh
```
4. Take a look at the new wheel packages you built! They may be owned by `root`
because of how Docker volume permissions work.
```
ls -al /tmp/packages
```
5. To continue on to running the Pip tests, create a venv and install the
testing packages:
```
docker exec tf /usertools/setup_venv_test.sh bazel_pip "/tf/pkg/tf_nightly*.whl"
```
6. And now run the tests depending on your target platform: `--config=pip`
includes the same test suite that is run by the DevInfra team every night.
If you want to run a specific test instead of the whole suite, pass
`--config=pip_venv` instead, and then set the target on the command like
normal.
<details><summary>TF Nightly CPU - Remote Cache</summary>
Build the sources with Bazel:
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
test --config=sigbuild_remote_cache \
--config=pip
```
</details>
<details><summary>TF Nightly GPU - Remote Cache</summary>
Build the sources with Bazel:
```
docker exec tf bazel --bazelrc=/usertools/gpu.bazelrc \
test --config=sigbuild_remote_cache \
--config=pip
```
</details>
<details><summary>TF Nightly CPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
Build the sources with Bazel:
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
test --config=sigbuild_local_cache \
--config=pip
```
</details>
<details><summary>TF Nightly GPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
Build the sources with Bazel:
```
docker exec tf \
bazel --bazelrc=/usertools/gpu.bazelrc \
test --config=sigbuild_local_cache \
--config=pip
```
</details>
### Run Nonpip Tests
1. Run the tests depending on your target platform. `--config=nonpip` includes
the same test suite that is run by the DevInfra team every night. If you
want to run a specific test instead of the whole suite, you do not need
`--config=nonpip` at all; just set the target on the command line like usual.
<details><summary>TF Nightly CPU - Remote Cache</summary>
Build the sources with Bazel:
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
test --config=sigbuild_remote_cache \
--config=nonpip
```
</details>
<details><summary>TF Nightly GPU - Remote Cache</summary>
Build the sources with Bazel:
```
docker exec tf bazel --bazelrc=/usertools/gpu.bazelrc \
test --config=sigbuild_remote_cache \
--config=nonpip
```
</details>
<details><summary>TF Nightly CPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
Build the sources with Bazel:
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
test --config=sigbuild_local_cache \
--config=nonpip
```
</details>
<details><summary>TF Nightly GPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
Build the sources with Bazel:
```
docker exec tf \
bazel --bazelrc=/usertools/gpu.bazelrc \
test --config=sigbuild_local_cache \
--config=nonpip
```
</details>
### Test, build and package libtensorflow
1. Run the tests depending on your target platform.
`--config=libtensorflow_test` includes the same test suite that is run by
the DevInfra team every night. If you want to run a specific test instead of
the whole suite, just set the target on the command line like usual.
<details><summary>TF Nightly CPU - Remote Cache</summary>
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
test --config=sigbuild_remote_cache \
--config=libtensorflow_test
```
</details>
<details><summary>TF Nightly GPU - Remote Cache</summary>
```
docker exec tf bazel --bazelrc=/usertools/gpu.bazelrc \
test --config=sigbuild_remote_cache \
--config=libtensorflow_test
```
</details>
<details><summary>TF Nightly CPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
test --config=sigbuild_local_cache \
--config=libtensorflow_test
```
</details>
<details><summary>TF Nightly GPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
```
docker exec tf \
bazel --bazelrc=/usertools/gpu.bazelrc \
test --config=sigbuild_local_cache \
--config=libtensorflow_test
```
</details>
1. Build the libtensorflow packages.
<details><summary>TF Nightly CPU - Remote Cache</summary>
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
build --config=sigbuild_remote_cache \
--config=libtensorflow_build
```
</details>
<details><summary>TF Nightly GPU - Remote Cache</summary>
```
docker exec tf bazel --bazelrc=/usertools/gpu.bazelrc \
build --config=sigbuild_remote_cache \
--config=libtensorflow_build
```
</details>
<details><summary>TF Nightly CPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
```
docker exec tf bazel --bazelrc=/usertools/cpu.bazelrc \
build --config=sigbuild_local_cache \
--config=libtensorflow_build
```
</details>
<details><summary>TF Nightly GPU - Local Cache</summary>
Make sure you have a directory mounted to the container in `/tf/cache`!
```
docker exec tf \
bazel --bazelrc=/usertools/gpu.bazelrc \
build --config=sigbuild_local_cache \
--config=libtensorflow_build
```
</details>
1. Run the `repack_libtensorflow.sh` utility to repack and rename the archives.
<details><summary>CPU</summary>
```
docker exec tf /usertools/repack_libtensorflow.sh /tf/pkg "-cpu-linux-x86_64"
```
</details>
<details><summary>GPU</summary>
```
docker exec tf /usertools/repack_libtensorflow.sh /tf/pkg "-gpu-linux-x86_64"
```
</details>
### Run a code check
1. Every night the TensorFlow team runs `code_check_full`, which contains a
suite of checks that were gradually introduced over TensorFlow's lifetime
to prevent certain unsable code states. This check has supplanted the old
"sanity" or "ci_sanity" checks.
```
docker exec tf bats /usertools/code_check_full.bats --timing --formatter junit
```
### Clean Up
1. Shut down and remove the container when you are finished.
```
docker stop tf
docker rm tf
```
tensorflow/tools/tf_sig_build_dockerfiles/builder.devtoolset/build_devtoolset.sh
Executable
+198
View File
@@ -0,0 +1,198 @@
#!/bin/bash -eu
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Builds a devtoolset cross-compiler targeting manylinux 2010 (glibc 2.12 /
# libstdc++ 4.4) or manylinux2014 (glibc 2.17 / libstdc++ 4.8).
VERSION="$1"
TARGET="$2"
case "${VERSION}" in
devtoolset-7)
LIBSTDCXX_VERSION="6.0.24"
LIBSTDCXX_ABI="gcc4-compatible"
;;
devtoolset-9)
LIBSTDCXX_VERSION="6.0.28"
LIBSTDCXX_ABI="new"
;;
*)
echo "Usage: $0 {devtoolset-7|devtoolset-9} <target-directory>"
echo "Use 'devtoolset-7' to build a manylinux2010 compatible toolchain or 'devtoolset-9' to build a manylinux2014 compatible toolchain"
exit 1
;;
esac
mkdir -p "${TARGET}"
# Download glibc's shared and development libraries based on the value of the
# `VERSION` parameter.
# Note: 'Templatizing' this and the other conditional branches would require
# defining several variables (version, os, path) making it difficult to maintain
# and extend for future modifications.
case "${VERSION}" in
devtoolset-7)
# Download binary glibc 2.12 shared library release.
wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6_2.12.1-0ubuntu6_amd64.deb" && \
unar "libc6_2.12.1-0ubuntu6_amd64.deb" && \
tar -C "${TARGET}" -xvzf "libc6_2.12.1-0ubuntu6_amd64/data.tar.gz" && \
rm -rf "libc6_2.12.1-0ubuntu6_amd64.deb" "libc6_2.12.1-0ubuntu6_amd64"
# Download binary glibc 2.12 development library release.
wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \
unar "libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \
tar -C "${TARGET}" -xvzf "libc6-dev_2.12.1-0ubuntu6_amd64/data.tar.gz" && \
rm -rf "libc6-dev_2.12.1-0ubuntu6_amd64.deb" "libc6-dev_2.12.1-0ubuntu6_amd64"
;;
devtoolset-9)
# Download binary glibc 2.17 shared library release.
wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6_2.17-0ubuntu5.1_amd64.deb" && \
unar "libc6_2.17-0ubuntu5.1_amd64.deb" && \
tar -C "${TARGET}" -xvzf "libc6_2.17-0ubuntu5.1_amd64/data.tar.gz" && \
rm -rf "libc6_2.17-0ubuntu5.1_amd64.deb" "libc6_2.17-0ubuntu5.1_amd64"
# Download binary glibc 2.17 development library release.
wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6-dev_2.17-0ubuntu5.1_amd64.deb" && \
unar "libc6-dev_2.17-0ubuntu5.1_amd64.deb" && \
tar -C "${TARGET}" -xvzf "libc6-dev_2.17-0ubuntu5.1_amd64/data.tar.gz" && \
rm -rf "libc6-dev_2.17-0ubuntu5.1_amd64.deb" "libc6-dev_2.17-0ubuntu5.1_amd64"
;;
esac
# Put the current kernel headers from ubuntu in place.
ln -s "/usr/include/linux" "/${TARGET}/usr/include/linux"
ln -s "/usr/include/asm-generic" "/${TARGET}/usr/include/asm-generic"
ln -s "/usr/include/x86_64-linux-gnu/asm" "/${TARGET}/usr/include/asm"
# Symlinks in the binary distribution are set up for installation in /usr, we
# need to fix up all the links to stay within /${TARGET}.
/fixlinks.sh "/${TARGET}"
# Patch to allow non-glibc 2.12 compatible builds to work.
sed -i '54i#define TCP_USER_TIMEOUT 18' "/${TARGET}/usr/include/netinet/tcp.h"
# Download specific version of libstdc++ shared library based on the value of
# the `VERSION` parameter
case "${VERSION}" in
devtoolset-7)
# Download binary libstdc++ 4.4 release we are going to link against.
# We only need the shared library, as we're going to develop against the
# libstdc++ provided by devtoolset.
wget "http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.4/libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \
unar "libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \
tar -C "/${TARGET}" -xvzf "libstdc++6_4.4.3-4ubuntu5_amd64/data.tar.gz" "./usr/lib/libstdc++.so.6.0.13" && \
rm -rf "libstdc++6_4.4.3-4ubuntu5_amd64.deb" "libstdc++6_4.4.3-4ubuntu5_amd64"
;;
devtoolset-9)
# Download binary libstdc++ 4.8 shared library release
wget "http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.8/libstdc++6_4.8.1-10ubuntu8_amd64.deb" && \
unar "libstdc++6_4.8.1-10ubuntu8_amd64.deb" && \
tar -C "/${TARGET}" -xvzf "libstdc++6_4.8.1-10ubuntu8_amd64/data.tar.gz" "./usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.18" && \
rm -rf "libstdc++6_4.8.1-10ubuntu8_amd64.deb" "libstdc++6_4.8.1-10ubuntu8_amd64"
;;
esac
mkdir -p "${TARGET}-src"
cd "${TARGET}-src"
# Build a devtoolset cross-compiler based on our glibc 2.12/glibc 2.17 sysroot setup.
case "${VERSION}" in
devtoolset-7)
wget "http://vault.centos.org/centos/6/sclo/Source/rh/devtoolset-7/devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm"
rpm2cpio "devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm" |cpio -idmv
tar -xvjf "gcc-7.3.1-20180303.tar.bz2" --strip 1
;;
devtoolset-9)
wget "https://vault.centos.org/centos/7/sclo/Source/rh/devtoolset-9-gcc-9.3.1-2.2.el7.src.rpm"
rpm2cpio "devtoolset-9-gcc-9.3.1-2.2.el7.src.rpm" |cpio -idmv
tar -xvf "gcc-9.3.1-20200408.tar.xz" --strip 1
;;
esac
# Apply the devtoolset patches to gcc.
/rpm-patch.sh "gcc.spec"
./contrib/download_prerequisites
mkdir -p "${TARGET}-build"
cd "${TARGET}-build"
"${TARGET}-src/configure" \
--prefix=/"${TARGET}/usr" \
--with-sysroot="/${TARGET}" \
--disable-bootstrap \
--disable-libmpx \
--disable-libsanitizer \
--disable-libunwind-exceptions \
--disable-libunwind-exceptions \
--disable-lto \
--disable-multilib \
--enable-__cxa_atexit \
--enable-gnu-indirect-function \
--enable-gnu-unique-object \
--enable-initfini-array \
--enable-languages="c,c++" \
--enable-linker-build-id \
--enable-plugin \
--enable-shared \
--enable-threads=posix \
--with-default-libstdcxx-abi=${LIBSTDCXX_ABI} \
--with-gcc-major-version-only \
--with-linker-hash-style="gnu" \
--with-tune="generic" \
&& \
make -j 42 && \
make install
# Create the devtoolset libstdc++ linkerscript that links dynamically against
# the system libstdc++ 4.4 and provides all other symbols statically.
case "${VERSION}" in
devtoolset-7)
mv "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}" \
"/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}.backup"
echo -e "OUTPUT_FORMAT(elf64-x86-64)\nINPUT ( libstdc++.so.6.0.13 -lstdc++_nonshared44 )" \
> "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}"
cp "./x86_64-pc-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared44.a" \
"/${TARGET}/usr/lib"
;;
devtoolset-9)
# Note that the installation path for libstdc++ here is /${TARGET}/usr/lib64/
mv "/${TARGET}/usr/lib64/libstdc++.so.${LIBSTDCXX_VERSION}" \
"/${TARGET}/usr/lib64/libstdc++.so.${LIBSTDCXX_VERSION}.backup"
echo -e "OUTPUT_FORMAT(elf64-x86-64)\nINPUT ( libstdc++.so.6.0.18 -lstdc++_nonshared44 )" \
> "/${TARGET}/usr/lib64/libstdc++.so.${LIBSTDCXX_VERSION}"
cp "./x86_64-pc-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared44.a" \
"/${TARGET}/usr/lib64"
;;
esac
# Link in architecture specific includes from the system; note that we cannot
# link in the whole x86_64-linux-gnu folder, as otherwise we're overlaying
# system gcc paths that we do not want to find.
# TODO(klimek): Automate linking in all non-gcc / non-kernel include
# directories.
mkdir -p "/${TARGET}/usr/include/x86_64-linux-gnu"
PYTHON_VERSIONS=("python3.10" "python3.11" "python3.12")
for v in "${PYTHON_VERSIONS[@]}"; do
ln -s "/usr/local/include/${v}" "/${TARGET}/usr/include/x86_64-linux-gnu/${v}"
done
# Patch glibc to be compatable with modern clang
case "${VERSION}" in
devtoolset-9)
cd /
patch -p0 < /glibc2.17-inline.patch
;;
esac
tensorflow/tools/tf_sig_build_dockerfiles/builder.devtoolset/fixlinks.sh
Executable
+28
View File
@@ -0,0 +1,28 @@
#!/bin/bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Re-direct all links in $1 that point to /lib... to point to $1/lib... instead.
BASE="$1"
find "${BASE}" -type l | \
while read l ; do
if [[ "$(readlink "$l")" == /lib* ]]; then
ORIG="$(readlink "$l")";
rm "$l";
ln -s "${BASE}${ORIG}" "$l"
fi
done
tensorflow/tools/tf_sig_build_dockerfiles/builder.devtoolset/glibc2.17-inline.patch
+11
View File
@@ -0,0 +1,11 @@
--- /dt9/usr/include/x86_64-linux-gnu/sys/cdefs.h 2013-09-30 13:58:17.000000000 +0000
+++ /dt9/usr/include/x86_64-linux-gnu/sys/cdefs.new.h 2022-11-04 17:17:31.727061220 +0000
@@ -320,7 +320,7 @@
/* GCC 4.3 and above with -std=c99 or -std=gnu99 implements ISO C99
inline semantics, unless -fgnu89-inline is used. */
-#if (!defined __cplusplus || __GNUC_PREREQ (4,3)) && defined __GNUC__
+#if (!defined __cplusplus || __GNUC_PREREQ (4,3) || defined __clang__) && defined __GNUC__
# if defined __GNUC_STDC_INLINE__ || defined __cplusplus
# define __extern_inline extern __inline __attribute__ ((__gnu_inline__))
# define __extern_always_inline \
tensorflow/tools/tf_sig_build_dockerfiles/builder.devtoolset/rpm-patch.sh
Executable
+28
View File
@@ -0,0 +1,28 @@
#!/bin/bash -eu
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Given an RPM spec file $1, apply its patches.
SPEC="$1"
grep '%patch' "${SPEC}" |while read cmd ; do
N=$(echo "${cmd}" |sed 's,%patch\([0-9]\+\).*,\1,')
file=$(grep "Patch$N:" "${SPEC}" |sed 's,.*: ,,')
parg=$(echo "${cmd}" |sed 's,.*\(-p[0-9]\).*,\1,')
if [[ ! "${file}" =~ doxygen && "${cmd}" != \#* ]]; then
echo "patch ${parg} -s < ${file}"
patch ${parg} -s < "${file}"
fi
done
tensorflow/tools/tf_sig_build_dockerfiles/builder.packages.txt
+32
View File
@@ -0,0 +1,32 @@
# Packages needed to install Python from source
# See https://github.com/pyenv/pyenv/wiki#suggested-build-environment
build-essential
curl
libbz2-dev
libffi-dev
liblzma-dev
libncurses5-dev
libreadline-dev
libsqlite3-dev
libssl-dev
libxml2-dev
libxmlsec1-dev
llvm
make
tk-dev
wget
xz-utils
zlib1g-dev
git
# Packages needed to build devtoolset
file
flex
g++
make
patch
rpm2cpio
unar
wget
xz-utils
cpio
tensorflow/tools/tf_sig_build_dockerfiles/devel.bashrc
+26
View File
@@ -0,0 +1,26 @@
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ==============================================================================
# Do not print anything if this is not being used interactively
[ -z "$PS1" ] && return
# Set up attractive prompt
export PS1="\[\e[31m\]tf-docker\[\e[m\] \[\e[33m\]\w\[\e[m\] > "
export TERM=xterm-256color
alias grep="grep --color=auto"
alias ls="ls --color=auto"
# Fix nvidia-docker
ldconfig
tensorflow/tools/tf_sig_build_dockerfiles/devel.packages.txt
+76
View File
@@ -0,0 +1,76 @@
# All required CUDA packages
cuda-compat-12-3
cuda-command-line-tools-12-3
cuda-cudart-dev-12-3
cuda-nvcc-12-3
cuda-cupti-12-3
cuda-nvprune-12-3
cuda-libraries-12-3
cuda-libraries-dev-12-3
cuda-nvml-dev-12-3
libcufft-12-3
libcurand-12-3
libcusolver-dev-12-3
libcusparse-dev-12-3
libcublas-12-3
libcublas-dev-12-3
libnccl-dev=2.19.3-1+cuda12.3
libnccl2=2.19.3-1+cuda12.3
# CuDNN: https://docs.nvidia.com/deeplearning/sdk/cudnn-install/index.html#ubuntu-network-installation
libcudnn8-dev=8.9.6.50-1+cuda12.2
libcudnn8=8.9.6.50-1+cuda12.2
# TensorRT: See https://docs.nvidia.com/deeplearning/sdk/tensorrt-install-guide/index.html#maclearn-net-repo-install-rpm
libnvinfer-headers-dev=8.6.1.6-1+cuda12.0
libnvinfer-headers-plugin-dev=8.6.1.6-1+cuda12.0
libnvinfer-plugin8=8.6.1.6-1+cuda12.0
libnvinfer8=8.6.1.6-1+cuda12.0
libnvinfer-dev=8.6.1.6-1+cuda12.0
libnvinfer-plugin-dev=8.6.1.6-1+cuda12.0
# This can be removed once NVIDIA publishes a cuda-12.3.2 Docker image.
# For now it ensures that we install at least version 12.3.107 of PTXAS,
# since 12.3.103 has a bug.
cuda-compiler-12-3=12.3.2-1
# Other build-related tools
apt-transport-https
autoconf
automake
build-essential
ca-certificates
llvm-18
clang-18
clang-tidy-18
lld-18
clang-format-12
colordiff
curl
ffmpeg
gdb
git
jq
less
libcurl3-dev
libcurl4-openssl-dev
libfreetype6-dev
libhdf5-serial-dev
libssl-dev
libtool
libzmq3-dev
mlocate
moreutils
openjdk-21-jdk
openjdk-21-jre-headless
parallel
pkg-config
python3-dev
python3-setuptools
rsync
software-properties-common
sudo
swig
unzip
vim
wget
zip
zlib1g-dev
tensorflow/tools/tf_sig_build_dockerfiles/devel.requirements.txt
+51
View File
@@ -0,0 +1,51 @@
# Cloned from internal ci_build dependencies
# See here for format definitions: https://www.python.org/dev/peps/pep-0440/#version-specifiers
#
# To have reproducible builds, these dependencies should be pinned always.
# Prefer pinning to the same version as in setup.py for now.
# This will change in the future.
absl-py ~= 1.0.0
astunparse ~= 1.6.3
flatbuffers ~= 25.9.23
google_pasta ~= 0.2
h5py ~= 3.11.0 # Earliest version for NumPy 2.0
ml_dtypes ~= 0.5.1 # Earliest version with mxfloat types
# TODO(b/366266944): Support older versions of NumPy to support TFX. Remove when
# Apache Beam upgrades to newer NumPy.
numpy ~= 1.26.0
opt_einsum ~= 3.3.0
packaging ~= 23.2
protobuf ~= 3.20.3
six ~= 1.16.0
termcolor ~= 2.1.1
typing_extensions ~= 4.8.0
wheel ~= 0.41.2
setuptools >= 70.0.0
wrapt ~= 1.14.1
# We need to pin the gast dependency exactly
gast == 0.4.0
# Finally, install tensorboard, and keras
# Note that here we want the latest version that matches TF major.minor version
# Note that we must use nightly here as these are used in nightly jobs
# For release jobs, we will pin these on the release branch
# Note that the CACHEBUSTER variable, set in the CI builds, will force these to
# be the latest version.
keras-nightly ~= 2.14.0.dev
tb-nightly ~= 2.13.0.a
# Test dependencies
grpcio ~= 1.59.0 # Earliest version for Python 3.12
portpicker ~= 1.6.0
scipy ~= 1.13.0 # Earliest version for NumPy 2.0
# Required for TFLite import from JAX tests
jax ~= 0.4.1; python_version <= '3.11'
jaxlib ~= 0.4.1; python_version <= '3.11' # Earliest version for Python 3.11
# For uploading
auditwheel ~= 5.0.0
twine ~= 3.6.0
# For user tool scripts
junitparser ~= 2.2.0
lxml ~= 4.9.1
pylint ~= 2.13.9
# For using Python 3.11 with Bazel 6 (b/286090018)
lit ~= 17.0.2
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/code_check_changed_files.bats
+76
View File
@@ -0,0 +1,76 @@
# vim: filetype=bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
setup_file() {
cd /tf/tensorflow
bazel version # Start the bazel server
# Without this, git errors if /tf/tensorflow directory owner is different
git config --global --add safe.directory /tf/tensorflow
# Note that you could generate a list of all the affected targets with e.g.:
# bazel query $(paste -sd "+" $BATS_FILE_TMPDIR/changed_files) --keep_going
# Only shows Added, Changed, Modified, Renamed, and Type-changed files
if [[ "$(git rev-parse --abbrev-ref HEAD)" = "pull_branch" ]]; then
# TF's CI runs 'git fetch origin "pull/PR#/merge:pull_branch"'
# To get the as-merged branch during the CI tests
git diff --diff-filter ACMRT --name-only pull_branch^ pull_branch > $BATS_FILE_TMPDIR/changed_files
else
# If the branch is not present, then diff against origin/master
git diff --diff-filter ACMRT --name-only origin/master > $BATS_FILE_TMPDIR/changed_files
fi
}
# Note: this is excluded on the full code base, since any submitted code must
# have passed Google's internal style guidelines.
@test "Check buildifier formatting on BUILD files" {
echo "buildifier formatting is recommended. Here are the suggested fixes:"
echo "============================="
grep -e 'BUILD' $BATS_FILE_TMPDIR/changed_files \
| xargs buildifier -v -mode=diff -diff_command="git diff --no-index"
}
# Note: this is excluded on the full code base, since any submitted code must
# have passed Google's internal style guidelines.
@test "Check formatting for C++ files" {
skip "clang-format doesn't match internal clang-format checker"
echo "clang-format is recommended. Here are the suggested changes:"
echo "============================="
grep -e '\.h$' -e '\.cc$' $BATS_FILE_TMPDIR/changed_files > $BATS_TEST_TMPDIR/files || true
if [[ ! -s $BATS_TEST_TMPDIR/files ]]; then return 0; fi
xargs -a $BATS_TEST_TMPDIR/files -i -n1 -P $(nproc --all) \
bash -c 'clang-format-12 --style=Google {} | git diff --no-index {} -' \
| tee $BATS_TEST_TMPDIR/needs_help.txt
echo "You can use clang-format --style=Google -i <file> to apply changes to a file."
[[ ! -s $BATS_TEST_TMPDIR/needs_help.txt ]]
}
# Note: this is excluded on the full code base, since any submitted code must
# have passed Google's internal style guidelines.
@test "Check pylint for Python files" {
echo "Python formatting is recommended. Here are the pylint errors:"
echo "============================="
grep -e "\.py$" $BATS_FILE_TMPDIR/changed_files > $BATS_TEST_TMPDIR/files || true
if [[ ! -s $BATS_TEST_TMPDIR/files ]]; then return 0; fi
xargs -a $BATS_TEST_TMPDIR/files -n1 -P $(nproc --all) \
python -m pylint --rcfile=tensorflow/tools/ci_build/pylintrc --score false \
| grep -v "**** Module" \
| tee $BATS_TEST_TMPDIR/needs_help.txt
[[ ! -s $BATS_TEST_TMPDIR/needs_help.txt ]]
}
teardown_file() {
bazel shutdown
}
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/code_check_full.bats
+309
View File
@@ -0,0 +1,309 @@
# vim: filetype=bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
setup_file() {
cd /tf/tensorflow
bazel version # Start the bazel server
}
# Do a bazel query specifically for the licenses checker. It searches for
# targets matching the provided query, which start with // or @ but not
# //tensorflow (so it looks for //third_party, //external, etc.), and then
# gathers the list of all packages (i.e. directories) which contain those
# targets.
license_query() {
bazel cquery --experimental_cc_shared_library "$1" --keep_going \
| grep -e "^//" -e "^@" \
| grep -E -v "^//tensorflow" \
| sed -e 's|:.*||' \
| sort -u
}
# Verify that, given a build target and a license-list generator target, all of
# the dependencies of that target which include a license notice file are then
# included when generating that license. Necessary because the license targets
# in TensorFlow are manually enumerated rather than generated automatically.
do_external_licenses_check(){
BUILD_TARGET="$1"
LICENSES_TARGET="$2"
# grep patterns for targets which are allowed to be missing from the licenses
cat > $BATS_TEST_TMPDIR/allowed_to_be_missing <<EOF
@absl_py//absl
@bazel_tools//platforms
@bazel_tools//third_party/
@bazel_tools//tools
@local
@com_google_absl//absl
@pybind11_abseil//pybind11_abseil
@org_tensorflow//
@com_github_googlecloudplatform_google_cloud_cpp//google
@com_github_grpc_grpc//src/compiler
@platforms//os
@ruy//
EOF
# grep patterns for targets which are allowed to be extra licenses
cat > $BATS_TEST_TMPDIR/allowed_to_be_extra <<EOF
@xla//third_party/mkl
@xla//third_party/mkl_dnn
@absl_py//
@bazel_tools//src
@bazel_tools//platforms
@bazel_tools//tools/
@org_tensorflow//tensorflow
@com_google_absl//
@pybind11_abseil//pybind11_abseil
//external
@local
@com_github_googlecloudplatform_google_cloud_cpp//
@embedded_jdk//
^//$
@ruy//
EOF
license_query "attr('licenses', 'notice', deps($BUILD_TARGET))" > $BATS_TEST_TMPDIR/expected_licenses
license_query "deps($LICENSES_TARGET)" > $BATS_TEST_TMPDIR/actual_licenses
# Column 1 is left only, Column 2 is right only, Column 3 is shared lines
# Select lines unique to actual_licenses, i.e. extra licenses.
comm -1 -3 $BATS_TEST_TMPDIR/expected_licenses $BATS_TEST_TMPDIR/actual_licenses | grep -v -f $BATS_TEST_TMPDIR/allowed_to_be_extra > $BATS_TEST_TMPDIR/actual_extra_licenses || true
# Select lines unique to expected_licenses, i.e. missing licenses
comm -2 -3 $BATS_TEST_TMPDIR/expected_licenses $BATS_TEST_TMPDIR/actual_licenses | grep -v -f $BATS_TEST_TMPDIR/allowed_to_be_missing > $BATS_TEST_TMPDIR/actual_missing_licenses || true
if [[ -s $BATS_TEST_TMPDIR/actual_extra_licenses ]]; then
echo "Please remove the following extra licenses from $LICENSES_TARGET:"
cat $BATS_TEST_TMPDIR/actual_extra_licenses
fi
if [[ -s $BATS_TEST_TMPDIR/actual_missing_licenses ]]; then
echo "Please include the missing licenses for the following packages in $LICENSES_TARGET:"
cat $BATS_TEST_TMPDIR/actual_missing_licenses
fi
# Fail if either of the two "extras" or "missing" lists are present. If so,
# then the user will see the above error messages.
[[ ! -s $BATS_TEST_TMPDIR/actual_extra_licenses ]] && [[ ! -s $BATS_TEST_TMPDIR/actual_missing_licenses ]]
}
@test "Pip package generated license includes all dependencies' licenses" {
do_external_licenses_check \
"//tensorflow/tools/pip_package:build_pip_package" \
"//tensorflow/tools/pip_package:licenses"
}
@test "Libtensorflow generated license includes all dependencies' licenses" {
do_external_licenses_check \
"//tensorflow:libtensorflow.so" \
"//tensorflow/tools/lib_package:clicenses_generate"
}
@test "Java library generated license includes all dependencies' licenses" {
do_external_licenses_check \
"//tensorflow/java:libtensorflow_jni.so" \
"//tensorflow/tools/lib_package:jnilicenses_generate"
}
# This test ensures that all the targets built into the Python package include
# their dependencies. It's a rewritten version of the "smoke test", an older
# Python script that was very difficult to understand. See
# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/pip_package/pip_smoke_test.py
@test "Pip package includes all required //tensorflow dependencies" {
# grep patterns for packages whose dependencies can be ignored
cat > $BATS_TEST_TMPDIR/ignore_deps_for_these_packages <<EOF
//tensorflow/lite
//tensorflow/compiler/mlir/lite
//tensorflow/compiler/mlir/tfrt
//tensorflow/core/runtime_fallback
//tensorflow/core/tfrt
//tensorflow/python/kernel_tests/signal
//tensorflow/examples
//tensorflow/tools/android
//tensorflow/python/eager/benchmarks
EOF
# grep patterns for files and targets which don't need to be in the pip
# package, ever.
cat > $BATS_TEST_TMPDIR/ignore_these_deps <<EOF
benchmark
_test$
_test.py$
_test_cpu$
_test_cpu.py$
_test_gpu$
_test_gpu.py$
_test_lib$
//tensorflow/cc/saved_model:saved_model_test_files
//tensorflow/cc/saved_model:saved_model_half_plus_two
//tensorflow:no_tensorflow_py_deps
//tensorflow/tools/pip_package:win_pip_package_marker
//tensorflow/core:image_testdata
//tensorflow/core/kernels/cloud:bigquery_reader_ops
//tensorflow/python:extra_py_tests_deps
//tensorflow/python:mixed_precision
//tensorflow/python:tf_optimizer
//tensorflow/python:compare_test_proto_py
//tensorflow/python/framework:test_ops_2
//tensorflow/python/framework:test_file_system.so
//tensorflow/python/debug:grpc_tensorflow_server.par
//tensorflow/python/feature_column:vocabulary_testdata
//tensorflow/python/util:nest_test_main_lib
//tensorflow/lite/experimental/examples/lstm:rnn_cell
//tensorflow/lite/experimental/examples/lstm:rnn_cell.py
//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test
//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test.py
//tensorflow/lite/python:interpreter
//tensorflow/lite/python:interpreter_test
//tensorflow/lite/python:interpreter.py
//tensorflow/lite/python:interpreter_test.py
EOF
# Get the full list of files and targets which get included into the pip
# package
bazel query --keep_going 'deps(//tensorflow/tools/pip_package:build_pip_package)' | sort -u > $BATS_TEST_TMPDIR/pip_deps
# Find all Python py_test targets not tagged "no_pip" or "manual", excluding
# any targets in ignored packages. Combine this list of targets into a bazel
# query list (e.g. the list becomes "target+target2+target3")
bazel query --keep_going 'kind(py_test, //tensorflow/python/...) - attr("tags", "no_pip|manual", //tensorflow/python/...)' | grep -v -f $BATS_TEST_TMPDIR/ignore_deps_for_these_packages | paste -sd "+" - > $BATS_TEST_TMPDIR/deps
# Find all one-step dependencies of those tests which are from //tensorflow
# (since external deps will come from Python-level pip dependencies),
# excluding dependencies and files that are known to be unneccessary.
# This creates a list of targets under //tensorflow that are required for
# TensorFlow python tests.
bazel query --keep_going "deps($(cat $BATS_TEST_TMPDIR/deps), 1)" | grep "^//tensorflow" | grep -v -f $BATS_TEST_TMPDIR/ignore_these_deps | sort -u > $BATS_TEST_TMPDIR/required_deps
# Find if any required dependencies are missing from the list of dependencies
# included in the pip package.
# (comm: Column 1 is left, Column 2 is right, Column 3 is shared lines)
comm -2 -3 $BATS_TEST_TMPDIR/required_deps $BATS_TEST_TMPDIR/pip_deps > $BATS_TEST_TMPDIR/missing_deps || true
if [[ -s $BATS_TEST_TMPDIR/missing_deps ]]; then
cat <<EOF
One or more test dependencies are not in the pip package.
If these test dependencies need to be in the TensorFlow pip package, please
add them to //tensorflow/tools/pip_package/BUILD. Otherwise, add the no_pip tag
to the test, or change code_check_full.bats in the SIG Build repository. That's
https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/code_check_full.bats
Here are the affected tests:
EOF
while read dep; do
echo "For dependency $dep:"
# For every missing dependency, find the tests which directly depend on
# it, and print that list for debugging. Not really clear if this is
# helpful since the only examples I've seen are enormous.
bazel query "rdeps(kind(py_test, $(cat $BATS_TEST_TMPDIR/deps)), $dep, 1)"
done < $BATS_TEST_TMPDIR/missing_deps
exit 1
fi
}
# The Python package is not allowed to depend on any CUDA packages.
@test "Pip package doesn't depend on CUDA" {
bazel cquery \
--experimental_cc_shared_library \
--@local_config_cuda//:enable_cuda \
--repo_env=HERMETIC_CUDA_VERSION="12.3.2" \
--repo_env=HERMETIC_CUDNN_VERSION="8.9.7.29" \
"somepath(//tensorflow/tools/pip_package:build_pip_package, " \
"@local_config_cuda//cuda:cudart + "\
"@local_config_cuda//cuda:cudart + "\
"@local_config_cuda//cuda:cuda_driver + "\
"@local_config_cuda//cuda:cudnn + "\
"@local_config_cuda//cuda:cusolver + "\
"@local_config_tensorrt//:tensorrt)" --keep_going > $BATS_TEST_TMPDIR/out
cat <<EOF
There was a path found connecting //tensorflow/tools/pip_package:build_pip_package
to a banned CUDA dependency. Here's the output from bazel query:
EOF
cat $BATS_TEST_TMPDIR/out
[[ ! -s $BATS_TEST_TMPDIR/out ]]
}
@test "Pip package doesn't depend on CUDA for static builds (i.e. Windows)" {
bazel cquery \
--experimental_cc_shared_library \
--@local_config_cuda//:enable_cuda \
--repo_env=HERMETIC_CUDA_VERSION="12.3.2" \
--repo_env=HERMETIC_CUDNN_VERSION="8.9.7.29" \
--define framework_shared_object=false \
"somepath(//tensorflow/tools/pip_package:build_pip_package, " \
"@local_config_cuda//cuda:cudart + "\
"@local_config_cuda//cuda:cudart + "\
"@local_config_cuda//cuda:cuda_driver + "\
"@local_config_cuda//cuda:cudnn + "\
"@local_config_cuda//cuda:cusolver + "\
"@local_config_tensorrt//:tensorrt)" --keep_going > $BATS_TEST_TMPDIR/out
cat <<EOF
There was a path found connecting //tensorflow/tools/pip_package:build_pip_package
to a banned CUDA dependency when '--define framework_shared_object=false' is set.
This means that a CUDA target was probably included via an is_static condition,
used when targeting platforms like Windows where we build statically instead
of dynamically. Here's the output from bazel query:
EOF
cat $BATS_TEST_TMPDIR/out
[[ ! -s $BATS_TEST_TMPDIR/out ]]
}
@test "All tensorflow.org/code links point to real files" {
for i in $(grep -onI 'https://www.tensorflow.org/code/[a-zA-Z0-9/._-]\+' -r tensorflow); do
target=$(echo $i | sed 's!.*https://www.tensorflow.org/code/!!g')
if [[ ! -f $target ]] && [[ ! -d $target ]]; then
echo "$i" >> errors.txt
fi
if [[ -e errors.txt ]]; then
echo "Broken links found:"
cat errors.txt
rm errors.txt
false
fi
done
}
@test "No duplicate files on Windows" {
cat <<EOF
Please rename files so there are no repeats. For example, README.md and
Readme.md would be the same file on Windows. In this test, you would get a
warning for "readme.md" because it makes everything lowercase. There are
repeats of these filename(s) with different casing:
EOF
find . | tr '[A-Z]' '[a-z]' | sort | uniq -d | tee $BATS_FILE_TMPDIR/repeats
[[ ! -s $BATS_FILE_TMPDIR/repeats ]]
}
# It's unclear why, but running this on //tensorflow/... is faster than running
# only on affected targets, usually. There are targets in //tensorflow/lite that
# don't pass --nobuild, so they're on their own.
#
# Although buildifier checks for formatting as well, "bazel build nobuild"
# checks for cross-file issues like bad includes or missing BUILD definitions.
#
# We can't test on the windows toolchains because they're using a legacy
# toolchain format (or something) that specifies the toolchain directly instead
# of as a "repository". They can't be valid on Linux because Linux can't do
# anything with a Windows-only toolchain, and bazel errors if trying to build
# that directory.
@test "bazel nobuild passes on all of TF except TF Lite and win toolchains" {
bazel build --experimental_cc_shared_library --nobuild --keep_going -- //tensorflow/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/win/... -//tensorflow/tools/toolchains/win_1803/...
}
teardown_file() {
bazel shutdown
}
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/cpu.bazelrc
+43
View File
@@ -0,0 +1,43 @@
# This bazelrc can build a CPU-supporting TF package.
# Convenient cache configurations
# Use a cache directory mounted to /tf/cache. Very useful!
build:sigbuild_local_cache --disk_cache=/tf/cache
# Use the public-access TF DevInfra cache (read only)
build:sigbuild_remote_cache --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache/september2022" --remote_upload_local_results=false
# Write to the TF DevInfra cache (only works for internal TF CI)
build:sigbuild_remote_cache_push --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache/september2022" --google_default_credentials
# Change the value of CACHEBUSTER when upgrading the toolchain, or when testing
# different compilation methods. E.g. for a PR to test a new CUDA version, set
# the CACHEBUSTER to the PR number.
build --action_env=CACHEBUSTER=565341047
# Build options for CPU Linux
build --config=release_cpu_linux
# "nonpip" tests are regular py_test tests.
# Pass --config=nonpip to run the same suite of tests. If you want to run just
# one test for investigation, you don't need --config=nonpip; just run the
# bazel test invocation as normal.
test:nonpip_filters --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only,-no_oss_py39,-no_oss_py310
test:nonpip_filters --build_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only,-no_oss_py39,-no_oss_py310
test:nonpip_filters --test_lang_filters=py --test_size_filters=small,medium
test:nonpip --config=nonpip_filters -- //tensorflow/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/...
# For building libtensorflow archives
test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //tensorflow/tools/lib_package:libtensorflow_java_test
build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip
# For outputting Build Event Protocol files
build:build_event_export --build_event_json_file=/tf/pkg/bep.json
# For Remote Build Execution.
# TODO(kanglan): Replace the old rbe configs with rbe_linux_cpu in linux cpu jobs and remove the `rbe` config here.
build:rbe --config=rbe_linux_cpu
# For continuous builds
test:pycpp_filters --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only
test:pycpp_filters --build_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only
test:pycpp_filters --test_lang_filters=cc,py --test_size_filters=small,medium
test:pycpp --config=pycpp_filters -- //tensorflow/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/...
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/cpu_gcc.bazelrc
+85
View File
@@ -0,0 +1,85 @@
# This bazelrc can build a CPU-supporting TF package.
# Convenient cache configurations
# Use a cache directory mounted to /tf/cache. Very useful!
build:sigbuild_local_cache --disk_cache=/tf/cache
# Use the public-access TF DevInfra cache (read only)
build:sigbuild_remote_cache --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache/september2022" --remote_upload_local_results=false
# Write to the TF DevInfra cache (only works for internal TF CI)
build:sigbuild_remote_cache_push --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache/september2022" --google_default_credentials
# Change the value of CACHEBUSTER when upgrading the toolchain, or when testing
# different compilation methods. E.g. for a PR to test a new CUDA version, set
# the CACHEBUSTER to the PR number.
build --action_env=CACHEBUSTER=501872366
# Use Python 3.X as installed in container image
build --action_env PYTHON_BIN_PATH="/usr/bin/python3"
build --action_env PYTHON_LIB_PATH="/usr/lib/tf_python"
build --python_path="/usr/bin/python3"
# Build TensorFlow v2
build --define=tf_api_version=2 --action_env=TF2_BEHAVIOR=1
# Target the AVX instruction set
build --copt=-mavx --host_copt=-mavx
# Store performance profiling log in the mounted artifact directory.
# The profile can be viewed by visiting chrome://tracing in a Chrome browser.
# See https://docs.bazel.build/versions/main/skylark/performance.html#performance-profiling
build --profile=/tf/pkg/profile.json.gz
# Use the NVCC toolchain to compile for manylinux2014
build --crosstool_top="@sigbuild-r2.17_config_cuda//crosstool:toolchain"
# Test-related settings below this point.
test --build_tests_only --keep_going --test_output=errors --verbose_failures=true
test --local_test_jobs=HOST_CPUS
test --test_env=LD_LIBRARY_PATH
# Give only the list of failed tests at the end of the log
test --test_summary=short
# "nonpip" tests are regular py_test tests.
# Pass --config=nonpip to run the same suite of tests. If you want to run just
# one test for investigation, you don't need --config=nonpip; just run the
# bazel test invocation as normal.
test:nonpip_filters --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only,-no_oss_py39,-no_oss_py310
test:nonpip_filters --build_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only,-no_oss_py39,-no_oss_py310
test:nonpip_filters --test_lang_filters=py --test_size_filters=small,medium
test:nonpip --config=nonpip_filters -- //tensorflow/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/...
# For building libtensorflow archives
test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //tensorflow/tools/lib_package:libtensorflow_java_test
build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip
# For outputting Build Event Protocol files
build:build_event_export --build_event_json_file=/tf/pkg/bep.json
# For Remote Build Execution.
build:rbe --google_default_credentials
build:rbe --bes_backend=buildeventservice.googleapis.com
build:rbe --bes_results_url="https://source.cloud.google.com/results/invocations"
build:rbe --bes_timeout=600s
build:rbe --define=EXECUTOR=remote
build:rbe --jobs=800
build:rbe --remote_executor=grpcs://remotebuildexecution.googleapis.com
build:rbe --remote_timeout=3600
build:rbe --spawn_strategy=remote,worker,standalone,local
build:rbe --remote_download_toplevel
build:rbe --action_env=PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin"
build:rbe --linkopt=-lrt --host_linkopt=-lrt --linkopt=-lm --host_linkopt=-lm # Unclear why this is here
build:rbe --host_crosstool_top="@sigbuild-r2.17_config_cuda//crosstool:toolchain"
build:rbe --crosstool_top="@sigbuild-r2.17_config_cuda//crosstool:toolchain"
build:rbe --extra_toolchains="@sigbuild-r2.17_config_cuda//crosstool:toolchain-linux-x86_64"
build:rbe --extra_execution_platforms="@sigbuild-r2.17_config_platform//:platform"
build:rbe --host_platform="@sigbuild-r2.17_config_platform//:platform"
build:rbe --platforms="@sigbuild-r2.17_config_platform//:platform"
# Python config is the same across all containers because the binary is the same
build:rbe --repo_env=TF_PYTHON_CONFIG_REPO="@sigbuild-r2.17_config_python"
build:rbe --remote_instance_name=projects/tensorflow-testing/instances/default_instance
build:rbe --project_id="tensorflow-testing"
# For continuous builds
test:pycpp_filters --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only
test:pycpp_filters --build_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-tpu,-benchmark-test,-v1only
test:pycpp_filters --test_lang_filters=cc,py --test_size_filters=small,medium
test:pycpp --config=pycpp_filters -- //tensorflow/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/...
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/get_test_list.sh
Executable
+24
View File
@@ -0,0 +1,24 @@
#!/bin/bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Usage: get_test_list.sh OUTPUT BAZEL_TEST_COMMAND...
# Writes the list of tests that would be run from BAZEL_TEST_COMMAND to OUTPUT.
# Hides all extra output and always exits with success for now.
OUTPUT=$1
shift
"$@" --test_summary=short --check_tests_up_to_date 2>/dev/null | sort -u | awk '{print $1}' | grep "^//" | tee $OUTPUT
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/gpu.bazelrc
+42
View File
@@ -0,0 +1,42 @@
# This bazelrc can build a GPU-supporting TF package.
# Convenient cache configurations
# Use a cache directory mounted to /tf/cache. Very useful!
build:sigbuild_local_cache --disk_cache=/tf/cache
# Use the public-access TF DevInfra cache (read only)
build:sigbuild_remote_cache --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache/september2022" --remote_upload_local_results=false
# Write to the TF DevInfra cache (only works for internal TF CI)
build:sigbuild_remote_cache_push --remote_cache="https://storage.googleapis.com/tensorflow-devinfra-bazel-cache/september2022" --google_default_credentials
# Change the value of CACHEBUSTER when upgrading the toolchain, or when testing
# different compilation methods. E.g. for a PR to test a new CUDA version, set
# the CACHEBUSTER to the PR number.
build --action_env=CACHEBUSTER=565341047
# Build options for GPU Linux
build --config=release_gpu_linux
# "nonpip" tests are regular py_test tests.
# Pass --config=nonpip to run the same suite of tests. If you want to run just
# one test for investigation, you don't need --config=nonpip; just run the
# bazel test invocation as normal.
test:nonpip_filters --test_tag_filters=gpu,requires-gpu,-no_gpu,-no_oss,-oss_excluded,-oss_serial,-benchmark-test,-no_cuda11,-no_oss_py39,-no_oss_py310
test:nonpip_filters --build_tag_filters=gpu,requires-gpu,-no_gpu,-no_oss,-oss_excluded,-oss_serial,-benchmark-test,-no_cuda11,-no_oss_py39,-no_oss_py310
test:nonpip_filters --test_lang_filters=py --test_size_filters=small,medium
test:nonpip --config=nonpip_filters -- //tensorflow/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/...
# For building libtensorflow archives
test:libtensorflow_test -- //tensorflow/tools/lib_package:libtensorflow_test //tensorflow/tools/lib_package:libtensorflow_java_test
build:libtensorflow_build -- //tensorflow/tools/lib_package:libtensorflow.tar.gz //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz //tensorflow/java:libtensorflow.jar //tensorflow/java:libtensorflow-src.jar //tensorflow/tools/lib_package:libtensorflow_proto.zip
# For outputting Build Event Protocol files
build:build_event_export --build_event_json_file=/tf/pkg/bep.json
# For Remote build execution -- GPU configuration
# TODO(kanglan): Replace the old rbe configs with rbe_linux_cuda in linux cuda jobs and remove the `rbe` config here.
build:rbe --config=rbe_linux_cuda
# For continuous builds
test:pycpp_filters --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-benchmark-test,-v1only,gpu,-no_gpu,-no_gpu_presubmit,-no_cuda11
test:pycpp_filters --build_tag_filters=-no_oss,-oss_excluded,-oss_serial,-benchmark-test,-v1only,gpu,-no_gpu,-no_gpu_presubmit,-no_cuda11
test:pycpp_filters --test_lang_filters=cc,py --test_size_filters=small,medium
test:pycpp --config=pycpp_filters -- //tensorflow/... -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/...
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/nvidia.bazelrc
Symlink
+1
View File
@@ -0,0 +1 @@
gpu.bazelrc
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/rename_and_verify_wheels.sh
Executable
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Check and rename wheels with auditwheel. Inserts the platform tags like
# "manylinux_xyz" into the wheel filename.
set -euxo pipefail
for wheel in /tf/pkg/*.whl; do
echo "Checking and renaming $wheel..."
time python3 -m auditwheel repair --plat manylinux2014_x86_64 "$wheel" --wheel-dir /tf/pkg 2>&1 | tee check.txt
# We don't need the original wheel if it was renamed
new_wheel=$(grep --extended-regexp --only-matching '/tf/pkg/\S+.whl' check.txt)
if [[ "$new_wheel" != "$wheel" ]]; then
rm "$wheel"
wheel="$new_wheel"
fi
TF_WHEEL="$wheel" bats /usertools/wheel_verification.bats --timing
done
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/squash_testlogs.py
Executable
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
# pylint:disable=protected-access
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Merge all JUnit test.xml files in one directory into one.
Usage: squash_testlogs.py START_DIRECTORY OUTPUT_FILE
Example: squash_testlogs.py /tf/pkg/testlogs /tf/pkg/merged.xml
Recursively find all the JUnit test.xml files in one directory, and merge any
of them that contain failures into one file. The TensorFlow DevInfra team
uses this to generate a simple overview of an entire pip and nonpip test
invocation, since the normal logs that Bazel creates are too large for the
internal invocation viewer.
"""
import collections
import os
import re
import subprocess
import sys
from junitparser import JUnitXml
result = JUnitXml()
try:
files = subprocess.check_output(
["grep", "-rlE", '(failures|errors)="[1-9]', sys.argv[1]])
except subprocess.CalledProcessError as e:
print("No failures found to log!")
exit(0)
# For test cases, only show the ones that failed that have text (a log)
seen = collections.Counter()
runfiles_matcher = re.compile(r"(/.*\.runfiles/)")
for f in files.strip().splitlines():
# Just ignore any failures, they're probably not important
try:
r = JUnitXml.fromfile(f)
except Exception as e: # pylint: disable=broad-except
print("Ignoring this XML parse failure in {}: ".format(f), str(e))
source_file = re.search(r"/(bazel_pip|tensorflow)/.*",
f.decode("utf-8")).group(0)
for testsuite in r:
testsuite._elem.set("source_file", source_file)
# Remove empty testcases
for p in testsuite._elem.xpath(".//testcase"):
if not len(p): # pylint: disable=g-explicit-length-test
testsuite._elem.remove(p)
# Change "testsuite > testcase,system-out" to "testsuite > testcase > error"
for p in testsuite._elem.xpath(".//system-out"):
for c in p.getparent().xpath(".//error | .//failure"):
c.text = p.text
p.getparent().remove(p)
# Remove duplicate results of the same exact test (e.g. due to retry
# attempts)
for p in testsuite._elem.xpath(".//error | .//failure"):
# Sharded tests have target names like this:
# WindowOpsTest.test_tflite_convert0 (<function hann_window at
# 0x7fc61728dd40>, 10, False, tf.float32)
# Where 0x... is a thread ID (or something) that is not important for
# debugging, but breaks this "number of failures" counter because it's
# different for repetitions of the same test. We use re.sub(r"0x\w+")
# to remove it.
key = re.sub(r"0x\w+", "", p.getparent().get("name", "")) + p.text
if key in seen:
testsuite._elem.remove(p.getparent())
seen[key] += 1
# Remove this testsuite if it doesn't have anything in it any more
if len(testsuite) == 0: # pylint: disable=g-explicit-length-test
r._elem.remove(testsuite._elem)
if len(r) > 0: # pylint: disable=g-explicit-length-test
result += r
# Insert the number of failures for each test to help identify flakes
# need to clarify for shard
for p in result._elem.xpath(".//error | .//failure"):
key = re.sub(r"0x\w+", "", p.getparent().get("name", "")) + p.text
p.text = runfiles_matcher.sub("[testroot]/", p.text)
source_file = p.getparent().getparent().get("source_file", "")
p.text += f"\nNOTE: From {source_file}"
if "bazel_pip" in source_file:
p.text += ("\nNOTE: This is a --config=pip test. Remove 'bazel_pip' to find"
" the file.")
n_failures = seen[key]
p.text += f"\nNOTE: Number of failures for this test: {seen[key]}."
p.text += "\n Most TF jobs run tests three times to root out flakes."
if seen[key] == 3:
p.text += ("\n Since there were three failures, this is not flaky, and"
" it")
p.text += "\n probably caused the Kokoro invocation to fail."
else:
p.text += ("\n Since there were not three failures, this is probably a"
" flake.")
p.text += ("\n Flakes make this pkg/pip_and_nonpip_tests target show "
"as failing,")
p.text += "\n but do not make the Kokoro invocation fail."
os.makedirs(os.path.dirname(sys.argv[2]), exist_ok=True)
result.update_statistics()
result.write(sys.argv[2])
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/wheel_verification.bats
+74
View File
@@ -0,0 +1,74 @@
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Suite of verification tests for the SINGLE TensorFlow wheel in /tf/pkg
# or whatever path is set as $TF_WHEEL.
setup_file() {
cd /tf/pkg
if [[ -z "$TF_WHEEL" ]]; then
export TF_WHEEL=$(find /tf/pkg -iname "*.whl")
fi
}
teardown_file() {
rm -rf /tf/venv
}
@test "Wheel is manylinux2014 (manylinux_2_17) compliant" {
python3 -m auditwheel show "$TF_WHEEL" > audit.txt
grep --quiet 'This constrains the platform tag to "manylinux_2_17_x86_64"' audit.txt
}
@test "Wheel conforms to upstream size limitations" {
WHEEL_MEGABYTES=$(stat --format %s "$TF_WHEEL" | awk '{print int($1/(1024*1024))}')
# Googlers: search for "test_tf_whl_size"
case "$TF_WHEEL" in
# CPU:
*cpu*manylinux*) LARGEST_OK_SIZE=240 ;;
# GPU:
*manylinux*) LARGEST_OK_SIZE=580 ;;
# Unknown:
*)
echo "The wheel's name is in an unknown format."
exit 1
;;
esac
# >&3 forces output in bats even if the test passes. See
# https://bats-core.readthedocs.io/en/stable/writing-tests.html#printing-to-the-terminal
echo "# Size of $TF_WHEEL is $WHEEL_MEGABYTES / $LARGEST_OK_SIZE megabytes." >&3
test "$WHEEL_MEGABYTES" -le "$LARGEST_OK_SIZE"
}
# Note: this runs before the tests further down the file, so TF is installed in
# the venv and the venv is active when those tests run. The venv gets cleaned
# up in teardown_file() above.
# LibTPU is necessary if building a tpu package, and it is installed via
# "-f <libtpu_download_url>". See tensorflow/setup.py.
@test "Wheel is installable" {
python3 -m venv /tf/venv
source /tf/venv/bin/activate
python3 -m pip install "$TF_WHEEL" -f https://storage.googleapis.com/libtpu-releases/index.html
}
@test "TensorFlow is importable" {
source /tf/venv/bin/activate
python3 -c 'import tensorflow as tf; t1=tf.constant([1,2,3,4]); t2=tf.constant([5,6,7,8]); print(tf.add(t1,t2).shape)'
}
# Is this still useful?
@test "TensorFlow has Keras" {
source /tf/venv/bin/activate
python3 -c 'import sys; import tensorflow as tf; sys.exit(0 if "keras" in tf.keras.__name__ else 1)'
}
tensorflow/tools/tf_sig_build_dockerfiles/setup.cuda.sh
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# setup.cuda.sh: Clean up and prepare the CUDA installation on the container.
# TODO(@perfinion) Review this file
# Delete uneccessary static libraries
find /usr/local/cuda-*/lib*/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete
rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v*.a
# Link the libcuda stub to the location where tensorflow is searching for it and
# reconfigure dynamic linker run-time bindings
ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf
ldconfig
tensorflow/tools/tf_sig_build_dockerfiles/setup.packages.sh
Executable
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# setup.packages.sh: Given a list of Ubuntu packages, install them and clean up.
# Usage: setup.packages.sh <package_list.txt>
set -e
# Prevent apt install tzinfo from asking our location (assumes UTC)
export DEBIAN_FRONTEND=noninteractive
apt-get update
# Remove commented lines and blank lines
apt-get install -y --no-install-recommends $(sed -e '/^\s*#.*$/d' -e '/^\s*$/d' "$1" | sort -u)
rm -rf /var/lib/apt/lists/*
tensorflow/tools/tf_sig_build_dockerfiles/setup.python.sh
Executable
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# setup.python.sh: Install a specific Python version and packages for it.
# Usage: setup.python.sh <pyversion> <requirements.txt>
set -xe
source ~/.bashrc
VERSION=$1
REQUIREMENTS=$2
# Install Python packages for this container's version
cat >pythons.txt <<EOF
$VERSION
$VERSION-dev
$VERSION-venv
EOF
/setup.packages.sh pythons.txt
# Re-link pyconfig.h from x86_64-linux-gnu into the devtoolset directory
# for any Python version present
pushd /usr/include/x86_64-linux-gnu
for f in $(ls | grep python); do
# set up symlink for devtoolset-9
rm -f /dt9/usr/include/x86_64-linux-gnu/$f
ln -s /usr/include/x86_64-linux-gnu/$f /dt9/usr/include/x86_64-linux-gnu/$f
done
popd
# Setup links for TensorFlow to compile.
# Referenced in devel.usertools/*.bazelrc
ln -sf /usr/bin/$VERSION /usr/bin/python3
ln -sf /usr/bin/$VERSION /usr/bin/python
ln -sf /usr/lib/$VERSION /usr/lib/tf_python
# Python 3.10 include headers fix:
# sysconfig.get_path('include') incorrectly points to /usr/local/include/python
# map /usr/include/python3.10 to /usr/local/include/python3.10
if [[ ! -f "/usr/local/include/$VERSION" ]]; then
ln -sf /usr/include/$VERSION /usr/local/include/$VERSION
fi
# Install pip
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python3 get-pip.py
python3 -m pip install --no-cache-dir --upgrade pip
python3 -m pip install -U setuptools
# Disable the cache dir to save image space, and install packages
python3 -m pip install --no-cache-dir -r $REQUIREMENTS -U
tensorflow/tools/tf_sig_build_dockerfiles/setup.sources.sh
Executable
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# setup.python.sh: Install a specific Python version and packages for it.
# Usage: setup.python.sh <pyversion> <requirements.txt>
# Sets up custom apt sources for our TF images.
# Prevent apt install tzinfo from asking our location (assumes UTC)
export DEBIAN_FRONTEND=noninteractive
# Set up shared custom sources
apt-get update
apt-get install -y gnupg ca-certificates
# Deadsnakes: https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F23C5A6CF475977595C89F51BA6932366A755776
# Explicitly request Nvidia repo keys
# See: https://forums.developer.nvidia.com/t/invalid-public-key-for-cuda-apt-repository/212901/11
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
# LLVM/Clang: https://apt.llvm.org/
apt-key adv --fetch-keys https://apt.llvm.org/llvm-snapshot.gpg.key
# Set up custom sources
cat >/etc/apt/sources.list.d/custom.list <<SOURCES
# Nvidia CUDA packages: 18.04 has more available than 20.04, and we use those
deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /
# More Python versions: Deadsnakes
deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main
deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main
# LLVM/Clang repository
deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main
SOURCES