chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,26 @@
|
||||
#### Issue Description
|
||||
|
||||
Please describe our issue, along with:
|
||||
- expected behavior
|
||||
- encountered behavior
|
||||
|
||||
#### Version Information
|
||||
|
||||
Please indicate relevant versions, including, if relevant:
|
||||
|
||||
* Deeplearning4j version
|
||||
* Platform information (OS, etc)
|
||||
* CUDA version, if used
|
||||
* NVIDIA driver version, if in use
|
||||
|
||||
#### Additional Information
|
||||
|
||||
Where applicable, please also provide:
|
||||
|
||||
* Full log or exception stack trace (ideally in a Gist: gist.github.com)
|
||||
* pom.xml file or similar (also in a Gist)
|
||||
|
||||
#### Contributing
|
||||
|
||||
If you'd like to help us fix the issue by contributing some code, but would
|
||||
like guidance or help in doing so, please mention it!
|
||||
@@ -0,0 +1,16 @@
|
||||
## What changes were proposed in this pull request?
|
||||
|
||||
(Please fill in changes proposed in this fix)
|
||||
|
||||
## How was this patch tested?
|
||||
|
||||
(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
|
||||
|
||||
## Quick checklist
|
||||
|
||||
The following checklist helps ensure your PR is complete:
|
||||
|
||||
- [ ] Eclipse Contributor Agreement signed, and signed commits - see [IP Requirements](https://deeplearning4j.konduit.ai/multi-project/how-to-guides/contribute/eclipse-contributors) page for details
|
||||
- [ ] Reviewed the [Contributing Guidelines](https://github.com/eclipse/deeplearning4j/blob/master/CONTRIBUTING.md) and followed the steps within.
|
||||
- [ ] Created tests for any significant new code additions.
|
||||
- [ ] Relevant tests for your changes are passing.
|
||||
@@ -0,0 +1,5 @@
|
||||
FROM centos:7
|
||||
COPY ./build.sh /build.sh
|
||||
RUN chmod +x /build.sh
|
||||
ENV JAVA_HOME=/opt/zulu11.58.15-ca-jdk11.0.16-linux_x64
|
||||
ENTRYPOINT '/build.sh'
|
||||
@@ -0,0 +1,10 @@
|
||||
## Centos build for deeplearning4j
|
||||
Implements a centos 7 based build based on the following docs:
|
||||
https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainervolumes
|
||||
https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action
|
||||
https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions
|
||||
|
||||
This is for a -compat build for older glibcs mainly running on centos based systems.
|
||||
The action relies on volumes to just run an install workload followed by a deployment
|
||||
using the host's gpg signature infrastructure very similar to the existing linux-x86_64 builds
|
||||
based on ubuntu 16.04
|
||||
@@ -0,0 +1,6 @@
|
||||
name: Build deeplearning4j on centos 7
|
||||
runs:
|
||||
using: 'docker'
|
||||
image: 'Dockerfile'
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/bin/bash
|
||||
env
|
||||
## Based on the javacpp presets github actions centos 7 build at https://github.com/bytedeco/javacpp-presets/
|
||||
SCL_ENABLE="devtoolset-7"
|
||||
yum -y update && yum -y install wget unzip centos-release-scl-rh epel-release
|
||||
echo "Downloading java from azul"
|
||||
cd /opt && wget https://cdn.azul.com/zulu/bin/zulu11.58.15-ca-jdk11.0.16-linux_x64.zip
|
||||
echo "Downloaded azul java"
|
||||
ls /opt
|
||||
cd /opt && unzip zulu11.58.15-ca-jdk11.0.16-linux_x64.zip
|
||||
#zulu11.58.15-ca-jdk11.0.16-linux_x64
|
||||
yum -y install $SCL_ENABLE rh-java-common-ant boost-devel ccache clang gcc-c++ gcc-gfortran ant python python36-devel python36-pip swig file which wget unzip tar bzip2 gzip xz patch autoconf-archive automake make libtool bison flex perl nasm alsa-lib-devel freeglut-devel gtk2-devel libusb-devel libusb1-devel curl-devel expat-devel gettext-devel openssl-devel bzip2-devel zlib-devel SDL-devel libva-devel libxkbcommon-devel libxkbcommon-x11-devel xcb-util* fontconfig-devel libffi-devel ragel ocl-icd-devel GeoIP-devel pcre-devel ssdeep-devel yajl-devel
|
||||
sed -i 's/_mm512_abs_pd (__m512 __A)/_mm512_abs_pd (__m512d __A)/g' /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/include/avx512fintrin.h
|
||||
source scl_source enable $SCL_ENABLE || true
|
||||
curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.6/cmake-3.16.6-Linux-x86_64.tar.gz
|
||||
curl -LO https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
|
||||
curl -LO https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.18.3.tar.gz
|
||||
tar -xzf cmake-3.16.6-Linux-x86_64.tar.gz -C /opt/
|
||||
mv /opt/cmake-3.16.6-Linux-x86_64 /opt/cmake
|
||||
tar -xzf apache-maven-3.6.3-bin.tar.gz -C /opt/
|
||||
tar -xzf git-2.18.3.tar.gz
|
||||
pushd git-2.18.3; make -j2 prefix=/usr/local/; make -j2 prefix=/usr/local/ install; popd
|
||||
ln -sf /usr/bin/python3.6 /usr/bin/python3
|
||||
ln -sf /opt/cmake-3.16.6-Linux-x86_64/bin/* /usr/bin/
|
||||
ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn
|
||||
curl -fsSL https://github.com/google/protobuf/releases/download/v3.8.0/protobuf-cpp-3.8.0.tar.gz \
|
||||
| tar xz && \
|
||||
cd protobuf-3.8.0 && \
|
||||
./configure --prefix=/opt/protobuf && \
|
||||
make -j2 && \
|
||||
make install && \
|
||||
cd .. && \
|
||||
rm -rf protobuf-3.8.0
|
||||
echo "/opt/protobuf/bin" >> $GITHUB_PATH
|
||||
# need to hardcode due to conflicting java home being set
|
||||
export JAVA_HOME=/opt/zulu11.58.15-ca-jdk11.0.16-linux_x64
|
||||
echo "${JAVA_HOME}/bin" >> $GITHUB_PATH
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$JAVA_HOME/bin:$PATH
|
||||
echo "JAVA_HOME ${JAVA_HOME}"
|
||||
java -version
|
||||
which javac
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
pwd
|
||||
# The volume directory for the workspace
|
||||
cd "/github/workspace/"
|
||||
bash ./bootstrap-libnd4j-from-url.sh
|
||||
|
||||
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$OPENBLAS_PATH"
|
||||
echo "Running INSTALL COMMAND ${INSTALL_COMMAND}"
|
||||
eval "${INSTALL_COMMAND}"
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
name: Download dl4j test resources
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: bash
|
||||
run: |
|
||||
wget https://github.com/KonduitAI/dl4j-test-resources/archive/master.zip && unzip master.zip
|
||||
cd dl4j-test-resources-master
|
||||
mvn clean install -DskipTests
|
||||
echo "Extracted test resources"
|
||||
@@ -0,0 +1,12 @@
|
||||
name: Download dl4j test resources
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: cmd
|
||||
run: |
|
||||
set "PATH=C:\msys64\usr\bin;%PATH%"
|
||||
wget https://github.com/KonduitAI/dl4j-test-resources/archive/master.zip && unzip master.zip
|
||||
cd dl4j-test-resources-master
|
||||
mvn clean install -DskipTests
|
||||
echo "Extracted test resources"
|
||||
@@ -0,0 +1,12 @@
|
||||
name: Download dl4j test resources
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt install git gcc-8-aarch64-linux-gnu g++-8-aarch64-linux-gnu libc6-armel-cross libc6-dev-armel-cross binutils-arm-linux-gnueabi libncurses5-dev build-essential bison flex libssl-dev bc \
|
||||
gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf crossbuild-essential-arm64
|
||||
mkdir -p /opt/raspberrypi && \
|
||||
cd /opt/raspberrypi && \
|
||||
git clone git://github.com/raspberrypi/tools.git
|
||||
@@ -0,0 +1,11 @@
|
||||
name: Install protobuf linux
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install protobuf linux
|
||||
shell: bash
|
||||
run: |
|
||||
apt-get -yq update && apt-get install -y build-essential unzip libssl-dev
|
||||
curl -fsSL http://cmake.org/files/v3.28.3/cmake-3.28.3.tar.gz | tar xz && cd cmake-3.28.3
|
||||
./configure --prefix=/opt/cmake && make -j2 && make install && cd .. && rm -r cmake-3.28.3
|
||||
echo "/opt/cmake/bin" >> $GITHUB_PATH
|
||||
@@ -0,0 +1,16 @@
|
||||
name: Install protobuf linux
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install protobuf linux
|
||||
shell: bash
|
||||
run: |
|
||||
curl -fsSL https://github.com/google/protobuf/releases/download/v3.8.0/protobuf-cpp-3.8.0.tar.gz \
|
||||
| tar xz && \
|
||||
cd protobuf-3.8.0 && \
|
||||
./configure --prefix=/opt/protobuf && \
|
||||
make -j2 && \
|
||||
sudo make install && \
|
||||
cd .. && \
|
||||
rm -rf protobuf-3.8.0
|
||||
echo "/opt/protobuf/bin" >> $GITHUB_PATH
|
||||
@@ -0,0 +1,10 @@
|
||||
name: Setup for msys2
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: cmd
|
||||
run: |
|
||||
msys2do pacman -S --needed --noconfirm base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake patch
|
||||
msys2do pacman -S --needed mingw-w64-x86_64-make --noconfirm mingw-w64-x86_64-gnupg mingw-w64-x86_64-cmake mingw-w64-x86_64-nasm mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-fortran mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-x86_64-ragel mingw-w64-x86_64-sed
|
||||
echo '--yes --always-trust' >> ~/.gnupg/gpg.conf
|
||||
@@ -0,0 +1,9 @@
|
||||
name: Publish to github packages
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Publish to GitHub Packages
|
||||
run: mvn -Pgithub --batch-mode deploy
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Remove un needed packages for linux
|
||||
description:
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt remove -y mongodb-org *google-chrome* firefox apache2 kubectl esl-erlang hhvm nginx libpq-dev postgresql postgresql-client ruby-full powershell r-base mono-complete nuget
|
||||
sudo rm -rf /usr/share/java/selnium*
|
||||
sudo rm -rf /usr/local/share/phantomjs*
|
||||
sudo rm -rf /usr/local/share/gecko_driver*
|
||||
sudo rm -rf /usr/local/lib/lein*
|
||||
sudo rm -rf /usr/share/miniconda*
|
||||
sudo apt remove -y apache2-bin* azure-cli dotnet-* ghc-* google-cloud-sdk* libboost-* libmono-* libobjc-* moby-* mono-* mysql* postgresql* r-* ruby* sqlite3* swig*
|
||||
sudo apt-get autoclean
|
||||
sudo rm -rf "/usr/local/share/boost"
|
||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||
sudo swapoff -a
|
||||
sudo rm -f /swapfile
|
||||
sudo apt clean
|
||||
df -h
|
||||
@@ -0,0 +1,39 @@
|
||||
name: Set OS install version
|
||||
description:
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -f /etc/os-release ]; then
|
||||
. /etc/os-release
|
||||
OS="$NAME"
|
||||
VER="$VERSION_ID"
|
||||
elif [ type lsb_release >/dev/null 2>&1 ]; then
|
||||
# linuxbase.org
|
||||
OS="$(lsb_release -si)"
|
||||
VER="$(lsb_release -sr)"
|
||||
elif [ -f /etc/lsb-release ]; then
|
||||
/etc/lsb-release
|
||||
OS="$DISTRIB_ID"
|
||||
VER="$DISTRIB_RELEASE"
|
||||
elif [ -f /etc/debian_version ]; then
|
||||
OS="Debian"
|
||||
VER="$(cat /etc/debian_version)"
|
||||
elif [ -f /etc/SuSe-release ]; then
|
||||
OS="SUSE"
|
||||
VER="13.1"
|
||||
elif [ -f /etc/redhat-release ]; then
|
||||
# Older Red Hat, CentOS, etc.
|
||||
OS="Centos"
|
||||
VER="6"
|
||||
else
|
||||
# Fall back to uname, e.g. "Linux <version>", also works for BSD, etc.
|
||||
OS="$(uname -s)"
|
||||
VER="$(uname -r)"
|
||||
fi
|
||||
|
||||
echo "OS=$OS" >> "$GITHUB_ENV"
|
||||
echo "VER=$VER" >> "$GITHUB_ENV"
|
||||
echo "OS is $OS and VERSION is $VER"
|
||||
@@ -0,0 +1,13 @@
|
||||
name: Update dependencies linux
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Update dependencies linux
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get install build-essential make zlib1g-dev wget
|
||||
sudo apt-get install pinentry-curses
|
||||
sudo apt-get install ca-certificates libgomp1
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: '2'
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: '0'
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: '1.0.0-M3'
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: '1.0.0-SNAPSHOT'
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: 'central'
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set.'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: 'ubuntu-22.04-arm'
|
||||
|
||||
debug_enabled:
|
||||
description: 'Enable debug session'
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
jobs:
|
||||
android-arm64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [armcompute, ""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
timeout-minutes: 720
|
||||
steps:
|
||||
- name: Clean workspace
|
||||
uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space (Initial)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf-intel-v1
|
||||
|
||||
- name: Install protobuf
|
||||
uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Cache CMake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake-v1
|
||||
|
||||
- name: Install CMake
|
||||
uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Install system dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y \
|
||||
ninja-build \
|
||||
build-essential \
|
||||
clang \
|
||||
lld \
|
||||
llvm-dev \
|
||||
gcc-aarch64-linux-gnu \
|
||||
g++-aarch64-linux-gnu \
|
||||
libc6-dev-arm64-cross
|
||||
|
||||
- name: Set up Java for publishing
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
|
||||
|
||||
|
||||
- name: Download and Setup Android NDK
|
||||
shell: bash
|
||||
run: |
|
||||
NDK_HOME="${GITHUB_WORKSPACE}/android-ndk-r27d"
|
||||
if [ ! -d "${NDK_HOME}" ]; then
|
||||
echo "Downloading Android NDK r27d for Linux x86_64..."
|
||||
cd "${GITHUB_WORKSPACE}"
|
||||
wget --progress=dot:giga "https://dl.google.com/android/repository/android-ndk-r27d-linux.zip"
|
||||
unzip -q "android-ndk-r27d-linux.zip"
|
||||
rm "android-ndk-r27d-linux.zip"
|
||||
echo "NDK extracted to: ${NDK_HOME}"
|
||||
ls -la "${NDK_HOME}" | head -10
|
||||
else
|
||||
echo "Using cached Android NDK."
|
||||
fi
|
||||
echo "ANDROID_NDK_ROOT=${NDK_HOME}" >> $GITHUB_ENV
|
||||
echo "ANDROID_NDK=${NDK_HOME}" >> $GITHUB_ENV
|
||||
|
||||
|
||||
|
||||
- name: Setup OpenBLAS
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Setting up OpenBLAS for Android ARM64..."
|
||||
OPENBLAS_JAR="openblas-0.3.28-1.5.11-android-arm64.jar"
|
||||
OPENBLAS_HOME="${GITHUB_WORKSPACE}/openblas_home"
|
||||
OPENBLAS_DIR="${OPENBLAS_HOME}/lib/arm64-v8a"
|
||||
|
||||
mkdir -p "${OPENBLAS_HOME}"
|
||||
cd "${OPENBLAS_HOME}"
|
||||
|
||||
wget -q "https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/${OPENBLAS_JAR}"
|
||||
unzip -q "${OPENBLAS_JAR}"
|
||||
|
||||
if [ -f "${OPENBLAS_DIR}/libopenblas.so" ] && [ ! -f "${OPENBLAS_DIR}/libopenblas.so.0" ]; then
|
||||
ln -s "${OPENBLAS_DIR}/libopenblas.so" "${OPENBLAS_DIR}/libopenblas.so.0"
|
||||
fi
|
||||
|
||||
echo "OPENBLAS_PATH=${OPENBLAS_DIR}" >> "$GITHUB_ENV"
|
||||
echo "OPENBLAS_HOME=${OPENBLAS_HOME}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Setup ARM Compute Library
|
||||
if: matrix.helper == 'armcompute'
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Setting up ARM Compute Library for Android ARM64..."
|
||||
ARMCOMPUTE_VERSION="v25.04"
|
||||
ARMCOMPUTE_HOME="${GITHUB_WORKSPACE}/armcompute_home"
|
||||
ARMCOMPUTE_PACKAGE="arm_compute-${ARMCOMPUTE_VERSION}-android-aarch64-cpu-bin"
|
||||
|
||||
mkdir -p "${ARMCOMPUTE_HOME}"
|
||||
cd "${ARMCOMPUTE_HOME}"
|
||||
|
||||
wget -q "https://github.com/ARM-software/ComputeLibrary/releases/download/${ARMCOMPUTE_VERSION}/${ARMCOMPUTE_PACKAGE}.tar.gz"
|
||||
tar -xzf "${ARMCOMPUTE_PACKAGE}.tar.gz"
|
||||
|
||||
ARMCOMPUTE_ROOT="${ARMCOMPUTE_HOME}/${ARMCOMPUTE_PACKAGE}"
|
||||
echo "ARMCOMPUTE_ROOT=${ARMCOMPUTE_ROOT}" >> "$GITHUB_ENV"
|
||||
echo "ARMCOMPUTE_HOME=${ARMCOMPUTE_HOME}" >> "$GITHUB_ENV"
|
||||
|
||||
|
||||
- name: Build with Maven
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
MAVEN_OPTS: -Xmx3g
|
||||
PROTO_EXEC: /opt/protobuf/bin/protoc
|
||||
run: |
|
||||
export PATH="/opt/protobuf/bin:/opt/cmake/bin:/usr/bin:$PATH"
|
||||
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${OPENBLAS_PATH}"
|
||||
# JAVA_HOME is set by setup-java action or the runner environment
|
||||
find /opt -name java
|
||||
echo "✅ Using Android NDK at: ${ANDROID_NDK}"
|
||||
echo "✅ Cross-compiling from Intel x86_64 to ARM64"
|
||||
|
||||
LIBND4J_CLASSIFIER="android-arm64"
|
||||
|
||||
if [ "${{ github.event.inputs.libnd4jUrl }}" != '' ]; then
|
||||
MODULES=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
MODULES=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
|
||||
CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=${GITHUB_WORKSPACE}/libnd4j/cmake/android-arm64.cmake \
|
||||
-G Ninja \
|
||||
-DSD_ANDROID_BUILD=true \
|
||||
-DANDROID_ABI=arm64-v8a \
|
||||
-DANDROID_PLATFORM=android-21 \
|
||||
-DANDROID_NDK=${ANDROID_NDK} \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_MAKE_PROGRAM=/usr/bin/ninja \
|
||||
-DBLAS_LIBRARIES=${OPENBLAS_PATH}/libopenblas.so \
|
||||
-DLAPACK_LIBRARIES=${OPENBLAS_PATH}/libopenblas.so"
|
||||
|
||||
BASE_COMMAND="mvn ${{ matrix.mvn_ext }} \
|
||||
-Dlibnd4j.generate.flatc=ON \
|
||||
-Djavacpp.platform.compiler=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++ \
|
||||
--no-transfer-progress \
|
||||
-Posx-aarch64-protoc \
|
||||
-pl ${MODULES} \
|
||||
-Pcpu \
|
||||
-Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} \
|
||||
-Dhttp.keepAlive=false \
|
||||
-Dmaven.wagon.http.pool=false \
|
||||
-Dmaven.wagon.http.retryHandler.count=3 \
|
||||
-Possrh \
|
||||
-DskipTestResourceEnforcement=true \
|
||||
-Dmaven.javadoc.failOnError=false \
|
||||
-Djavacpp.platform=${LIBND4J_CLASSIFIER} \
|
||||
-Dlibnd4j.cmake=\"${CMAKE_ARGS}\" \
|
||||
--also-make \
|
||||
--batch-mode \
|
||||
deploy \
|
||||
-DskipTests"
|
||||
|
||||
if [ "${{ matrix.helper }}" == "armcompute" ]; then
|
||||
HELPER_FLAGS="-Dlibnd4j.helper=${{ matrix.helper }} \
|
||||
-Djavacpp.platform.extension=-${{ matrix.helper }} \
|
||||
-Dlibnd4j.classifier=${LIBND4J_CLASSIFIER}-${{ matrix.helper }}"
|
||||
else
|
||||
HELPER_FLAGS=""
|
||||
fi
|
||||
|
||||
MAVEN_COMMAND="${BASE_COMMAND} ${HELPER_FLAGS}"
|
||||
|
||||
if [ "${PERFORM_RELEASE}" == "1" ]; then
|
||||
bash "${GITHUB_WORKSPACE}/release-specified-component.sh" \
|
||||
"${RELEASE_VERSION}" \
|
||||
"${SNAPSHOT_VERSION}" \
|
||||
"${RELEASE_REPO_ID}" \
|
||||
"${MAVEN_COMMAND}"
|
||||
else
|
||||
eval "${MAVEN_COMMAND}"
|
||||
fi
|
||||
|
||||
- name: Setup tmate session
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
if: ${{ github.event.inputs.debug_enabled == 'true' }}
|
||||
with:
|
||||
limit-access-to-actor: true
|
||||
|
||||
@@ -0,0 +1,529 @@
|
||||
# GitHub Actions Workflow for android-x86_64
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-latest
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
android-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [onednn, ""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- name: Configure swap space
|
||||
shell: bash
|
||||
run: |
|
||||
echo "=== Initial system status ==="
|
||||
free -h
|
||||
df -h
|
||||
echo "Available disk space on root:"
|
||||
df -h /
|
||||
|
||||
# Check if swap is already enabled
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "Swap already configured:"
|
||||
swapon --show
|
||||
else
|
||||
echo "Configuring swap space..."
|
||||
|
||||
# Use fixed 4GB swap size
|
||||
SWAP_SIZE="12G"
|
||||
echo "Creating ${SWAP_SIZE} swap file..."
|
||||
|
||||
# Create swap file
|
||||
sudo fallocate -l ${SWAP_SIZE} /swapfile || {
|
||||
echo "fallocate failed, trying dd..."
|
||||
sudo dd if=/dev/zero of=/swapfile bs=1G count=12 status=progress
|
||||
}
|
||||
|
||||
# Set up swap
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
|
||||
# Verify swap is active
|
||||
echo "=== Swap configuration completed ==="
|
||||
free -h
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
# Tune swappiness for build workloads
|
||||
# Lower values (10-20) prefer RAM, higher values (60+) use swap more aggressively
|
||||
echo "Current swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.swappiness=80
|
||||
echo "Adjusted swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.overcommit_memory=2 # Allow overcommit
|
||||
sudo sysctl vm.overcommit_ratio=80
|
||||
# Show final memory status
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
|
||||
- name: Cache cmake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake-install
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cmake-
|
||||
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-protobuf-
|
||||
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
|
||||
- uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake-install.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- uses: nttld/setup-ndk@v1
|
||||
id: setup-ndk
|
||||
with:
|
||||
ndk-version: r26d
|
||||
|
||||
- name: Set mvn build command and environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
# Set constants for Android X86_64
|
||||
LIBND4J_CLASSIFIER="android-x86_64"
|
||||
|
||||
# Define the specific OpenBLAS JAR for android-x86_64
|
||||
OPENBLAS_VERSION="0.3.28-1.5.11"
|
||||
OPENBLAS_JAR="openblas-${OPENBLAS_VERSION}-${LIBND4J_CLASSIFIER}.jar"
|
||||
OPENBLAS_INSTALL_DIR="${GITHUB_WORKSPACE}/openblas_home"
|
||||
|
||||
NDK_VERSION_ENV="r26d"
|
||||
|
||||
echo "LIBND4J_CLASSIFIER=${LIBND4J_CLASSIFIER}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_VERSION=${OPENBLAS_VERSION}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_JAR=${OPENBLAS_JAR}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_INSTALL_DIR=${OPENBLAS_INSTALL_DIR}" >> $GITHUB_ENV
|
||||
echo "CURRENT_TARGET=android-x86_64" >> $GITHUB_ENV
|
||||
echo "NDK_VERSION=${NDK_VERSION_ENV}" >> $GITHUB_ENV
|
||||
echo "ANDROID_VERSION=21" >> $GITHUB_ENV
|
||||
|
||||
if [ "${{ github.event.inputs.libnd4jUrl }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
|
||||
# Base Maven command
|
||||
command="mvn ${{ matrix.mvn_ext }} -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=${LIBND4J_CLASSIFIER} -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
|
||||
# Updated conditional flags for the helper library
|
||||
if [ "${{ matrix.helper }}" != '' ]; then
|
||||
if [ "${{ matrix.helper }}" == "onednn" ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=onednn -Djavacpp.platform.extension=-onednn -Dlibnd4j.classifier=${LIBND4J_CLASSIFIER}-onednn"
|
||||
else
|
||||
echo "Warning: Unsupported helper '${{ matrix.helper }}' defined in matrix. Only 'onednn' or empty is handled for Maven flags."
|
||||
mvn_ext=""
|
||||
fi
|
||||
else
|
||||
mvn_ext=""
|
||||
fi
|
||||
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
|
||||
- name: Setup OpenBLAS Library
|
||||
shell: bash
|
||||
env:
|
||||
OPENBLAS_INSTALL_DIR: ${{ env.OPENBLAS_INSTALL_DIR }}
|
||||
OPENBLAS_JAR: ${{ env.OPENBLAS_JAR }}
|
||||
OPENBLAS_VERSION: ${{ env.OPENBLAS_VERSION }}
|
||||
LIBND4J_CLASSIFIER: ${{ env.LIBND4J_CLASSIFIER }}
|
||||
run: |
|
||||
echo "=== Setting up OpenBLAS ${OPENBLAS_VERSION} for ${LIBND4J_CLASSIFIER} ==="
|
||||
|
||||
# Create and navigate to install directory
|
||||
mkdir -p "${OPENBLAS_INSTALL_DIR}"
|
||||
cd "${OPENBLAS_INSTALL_DIR}"
|
||||
|
||||
# Download OpenBLAS JAR
|
||||
OPENBLAS_URL="https://repo1.maven.org/maven2/org/bytedeco/openblas/${OPENBLAS_VERSION}/${OPENBLAS_JAR}"
|
||||
echo "Downloading ${OPENBLAS_JAR} from ${OPENBLAS_URL}..."
|
||||
wget --quiet ${OPENBLAS_URL} || {
|
||||
echo "❌ Failed to download OpenBLAS JAR"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Extract JAR
|
||||
echo "Extracting ${OPENBLAS_JAR}..."
|
||||
unzip -q ${OPENBLAS_JAR} || {
|
||||
echo "❌ Failed to extract OpenBLAS JAR"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "=== OpenBLAS JAR extraction completed ==="
|
||||
|
||||
# The JAR contains: lib/x86_64/include/ and lib/x86_64/
|
||||
OPENBLAS_PLATFORM_PATH="lib/x86_64"
|
||||
|
||||
if [[ -d "$OPENBLAS_PLATFORM_PATH" ]]; then
|
||||
echo "✅ Found OpenBLAS platform directory: $OPENBLAS_PLATFORM_PATH"
|
||||
|
||||
# Verify the expected structure
|
||||
if [[ -d "$OPENBLAS_PLATFORM_PATH/include" && -f "$OPENBLAS_PLATFORM_PATH/include/cblas.h" ]]; then
|
||||
echo "✅ Found headers at: $OPENBLAS_PLATFORM_PATH/include/"
|
||||
else
|
||||
echo "❌ Headers not found at expected location: $OPENBLAS_PLATFORM_PATH/include/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -d "$OPENBLAS_PLATFORM_PATH/lib" ]] || [[ -f "$OPENBLAS_PLATFORM_PATH/libopenblas.so" ]]; then
|
||||
echo "✅ Found libraries in: $OPENBLAS_PLATFORM_PATH/"
|
||||
else
|
||||
echo "❌ Libraries not found in: $OPENBLAS_PLATFORM_PATH/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set the correct path - this should be the full path to lib/x86_64
|
||||
OPENBLAS_FINAL_PATH="$(realpath "$OPENBLAS_PLATFORM_PATH")"
|
||||
|
||||
else
|
||||
echo "❌ Could not find OpenBLAS platform directory: $OPENBLAS_PLATFORM_PATH"
|
||||
echo "Available directories:"
|
||||
find . -type d -maxdepth 3 | head -20
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set final environment variables - this is the key fix
|
||||
echo "=== Setting final environment variables ==="
|
||||
echo "OPENBLAS_PATH=${OPENBLAS_FINAL_PATH}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_INCLUDE_PATH=${OPENBLAS_FINAL_PATH}/include" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_LIB_PATH=${OPENBLAS_FINAL_PATH}/" >> $GITHUB_ENV
|
||||
|
||||
echo "=== OpenBLAS setup completed ==="
|
||||
echo "Final OpenBLAS path: ${OPENBLAS_FINAL_PATH}"
|
||||
echo "Include path: ${OPENBLAS_FINAL_PATH}/include"
|
||||
echo "Library path: ${OPENBLAS_FINAL_PATH}/"
|
||||
|
||||
# Verify the final setup
|
||||
echo "=== Final verification ==="
|
||||
ls -la "${OPENBLAS_FINAL_PATH}/include/" | head -10
|
||||
ls -la "${OPENBLAS_FINAL_PATH}/" | grep -E "\.(so|a)$" | head -5
|
||||
|
||||
- name: OpenBLAS Configuration Verification
|
||||
shell: bash
|
||||
run: |
|
||||
echo "=== OpenBLAS Configuration Verification ==="
|
||||
echo "OPENBLAS_PATH: ${OPENBLAS_PATH}"
|
||||
echo "OPENBLAS_INCLUDE_PATH: ${OPENBLAS_INCLUDE_PATH}"
|
||||
echo "OPENBLAS_LIB_PATH: ${OPENBLAS_LIB_PATH}"
|
||||
echo
|
||||
|
||||
# Test that paths are accessible
|
||||
if [[ -d "$OPENBLAS_PATH" ]]; then
|
||||
echo "✅ OPENBLAS_PATH is accessible"
|
||||
else
|
||||
echo "❌ OPENBLAS_PATH is not accessible: $OPENBLAS_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -d "$OPENBLAS_INCLUDE_PATH" ]]; then
|
||||
echo "✅ OPENBLAS_INCLUDE_PATH is accessible"
|
||||
else
|
||||
echo "❌ OPENBLAS_INCLUDE_PATH is not accessible: $OPENBLAS_INCLUDE_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test header file accessibility
|
||||
if [[ -f "$OPENBLAS_INCLUDE_PATH/cblas.h" ]]; then
|
||||
echo "✅ cblas.h is accessible"
|
||||
echo "First few lines of cblas.h:"
|
||||
head -5 "$OPENBLAS_INCLUDE_PATH/cblas.h"
|
||||
else
|
||||
echo "❌ cblas.h is not accessible"
|
||||
echo "Contents of include directory:"
|
||||
ls -la "$OPENBLAS_INCLUDE_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -f "$OPENBLAS_INCLUDE_PATH/openblas_config.h" ]]; then
|
||||
echo "✅ openblas_config.h is accessible"
|
||||
else
|
||||
echo "❌ openblas_config.h is not accessible"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Verification completed successfully ==="
|
||||
|
||||
- name: Debug Info
|
||||
shell: bash
|
||||
run: |
|
||||
echo "--- Workflow Inputs ---"
|
||||
echo "buildThreads: ${{ github.event.inputs.buildThreads }}"
|
||||
echo "deployToReleaseStaging: ${{ github.event.inputs.deployToReleaseStaging }}"
|
||||
echo "releaseVersion: ${{ github.event.inputs.releaseVersion }}"
|
||||
echo "snapshotVersion: ${{ github.event.inputs.snapshotVersion }}"
|
||||
echo "releaseRepoId: ${{ github.event.inputs.releaseRepoId }}"
|
||||
echo "serverId: ${{ github.event.inputs.serverId }}"
|
||||
echo "mvnFlags: ${{ github.event.inputs.mvnFlags }}"
|
||||
echo "libnd4jUrl: ${{ github.event.inputs.libnd4jUrl }}"
|
||||
echo "runsOn: ${{ github.event.inputs.runsOn }}"
|
||||
echo "debug_enabled: ${{ github.event.inputs.debug_enabled }}"
|
||||
echo "--- Matrix Values ---"
|
||||
echo "Helper: ${{ matrix.helper }}"
|
||||
echo "--- Environment Variables ---"
|
||||
echo "LIBND4J_CLASSIFIER: $LIBND4J_CLASSIFIER"
|
||||
echo "OPENBLAS_VERSION: $OPENBLAS_VERSION"
|
||||
echo "OPENBLAS_JAR: $OPENBLAS_JAR"
|
||||
echo "OPENBLAS_INSTALL_DIR: $OPENBLAS_INSTALL_DIR"
|
||||
echo "OPENBLAS_PATH: $OPENBLAS_PATH"
|
||||
echo "OPENBLAS_INCLUDE_PATH: $OPENBLAS_INCLUDE_PATH"
|
||||
echo "OPENBLAS_LIB_PATH: $OPENBLAS_LIB_PATH"
|
||||
echo "CURRENT_TARGET: $CURRENT_TARGET"
|
||||
echo "NDK_VERSION: $NDK_VERSION"
|
||||
echo "ANDROID_VERSION: $ANDROID_VERSION"
|
||||
echo "NDK Path (from step): ${{ steps.setup-ndk.outputs.ndk-path }}"
|
||||
echo "COMMAND: $COMMAND"
|
||||
echo "--- System Info ---"
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
|
||||
- name: Monitor memory during build
|
||||
shell: bash
|
||||
run: |
|
||||
# Start background memory monitoring
|
||||
(while true; do
|
||||
echo "$(date '+%H:%M:%S'): $(free -h | head -n 2 | tail -n 1)"
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "$(date '+%H:%M:%S'): Swap: $(swapon --show --noheadings)"
|
||||
fi
|
||||
sleep 120 # Check every 2 minutes
|
||||
done) > memory_monitor.log 2>&1 &
|
||||
MONITOR_PID=$!
|
||||
echo "MONITOR_PID=${MONITOR_PID}" >> $GITHUB_ENV
|
||||
echo "Started memory monitoring (PID: ${MONITOR_PID})"
|
||||
|
||||
- name: Build with Maven
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
LIBND4J_BUILD_THREADS: ${{ github.event.inputs.buildThreads }}
|
||||
ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }}
|
||||
OPENBLAS_PATH: ${{ env.OPENBLAS_PATH }}
|
||||
ANDROID_VERSION: ${{ env.ANDROID_VERSION }}
|
||||
NDK_VERSION: ${{ env.NDK_VERSION }}
|
||||
PROTO_EXEC: /opt/protobuf/bin/protoc
|
||||
LD_LIBRARY_PATH: $LD_LIBRARY_PATH:${{ env.OPENBLAS_LIB_PATH }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
DEPLOY: 1
|
||||
BUILD_USING_MAVEN: 1
|
||||
TARGET_OS: android
|
||||
PUBLISH_TO: central
|
||||
DEPLOY_TO: central
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
RLIMIT_AS: 6442450944 # 6GB virtual memory limit
|
||||
RLIMIT_DATA: 4294967296 # 4GB data segment limit
|
||||
run: |
|
||||
echo "=== Build Environment Summary ==="
|
||||
echo "NDK Path: ${ANDROID_NDK}"
|
||||
echo "OpenBLAS Path: ${OPENBLAS_PATH}"
|
||||
echo "OpenBLAS Include Path: ${OPENBLAS_INCLUDE_PATH}"
|
||||
echo "OpenBLAS Lib Path: ${OPENBLAS_LIB_PATH}"
|
||||
echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
|
||||
echo "Build Threads: ${LIBND4J_BUILD_THREADS}"
|
||||
echo "Helper: ${HELPER}"
|
||||
echo "================================="
|
||||
|
||||
# Show memory status before build
|
||||
echo "=== Memory status before build ==="
|
||||
free -h
|
||||
|
||||
# Pre-build verification
|
||||
echo "=== Pre-build OpenBLAS verification ==="
|
||||
if [[ -f "$OPENBLAS_PATH/include/cblas.h" ]]; then
|
||||
echo "✅ cblas.h found and accessible"
|
||||
else
|
||||
echo "❌ cblas.h not found at $OPENBLAS_PATH/include/cblas.h"
|
||||
echo "Attempting to locate cblas.h..."
|
||||
find "$OPENBLAS_PATH" -name "cblas.h" 2>/dev/null | head -5
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Execute build
|
||||
if [ "$PERFORM_RELEASE" == 1 ] || [ "$PERFORM_RELEASE" == "true" ]; then
|
||||
echo "=== Executing release build ==="
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
echo "=== Executing snapshot build ==="
|
||||
echo "Command: ${COMMAND}"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
|
||||
- name: Show memory usage summary
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
# Stop memory monitoring
|
||||
if [ -n "$MONITOR_PID" ]; then
|
||||
kill $MONITOR_PID 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "=== Swap usage ==="
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
echo "=== Memory usage during build ==="
|
||||
if [ -f memory_monitor.log ]; then
|
||||
cat memory_monitor.log
|
||||
else
|
||||
echo "No memory monitoring log found"
|
||||
fi
|
||||
|
||||
- name: Cleanup swap
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Cleaning up swap configuration..."
|
||||
sudo swapoff /swapfile 2>/dev/null || true
|
||||
sudo rm -f /swapfile 2>/dev/null || true
|
||||
echo "Swap cleanup completed"
|
||||
|
||||
- name: Upload build artifacts on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: build-logs-${{ matrix.helper }}-${{ github.run_id }}
|
||||
path: |
|
||||
memory_monitor.log
|
||||
**/cmake_install.log
|
||||
**/build.log
|
||||
**/CMakeFiles/CMakeError.log
|
||||
**/CMakeFiles/CMakeOutput.log
|
||||
retention-days: 3
|
||||
@@ -0,0 +1,139 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
jobs:
|
||||
linux-x86_64:
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
- name: Cache cmake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake
|
||||
restore-keys: ${{ runner.os }}-cmake
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf
|
||||
restore-keys: ${{ runner.os }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
- uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake.outputs.cache-hit != 'true'
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Build on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ github.event.inputs.buildThreads }}"
|
||||
echo "deploy to release staging repo or not ${{ github.event.inputs.deployToReleaseStaging }}"
|
||||
echo "release version ${{ github.event.inputs.releaseVersion }}"
|
||||
echo "snapshot version ${{ github.event.inputs.snapshotVersion }}"
|
||||
echo "debug enabled ${{ github.event.inputs.debug_enabled }}"
|
||||
echo "libnd4j url ${{ github.event.inputs.libnd4jUrl }}"
|
||||
echo "maven flags ${{ github.event.inputs.mvnFlags }}"
|
||||
echo "snapshot version ${{ github.event.inputs.snapshotVersion }}"
|
||||
echo "server id ${{ github.event.inputs.serverId }}"
|
||||
echo "release repo id ${{ github.event.inputs.releaseRepoId }}"
|
||||
|
||||
sudo sysctl vm.overcommit_memory=2
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
sudo apt-get autoremove
|
||||
sudo apt-get clean
|
||||
command="mvn -Possrh -pl !:blas-lapack-generator,!:libnd4j-gen -DskipTestResourceEnforcement=true -Djavacpp.platform=linux-x86_64 --batch-mode deploy -DskipTests"
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${command}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "$command"
|
||||
fi
|
||||
|
||||
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${command}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "$command"
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04-arm
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
swapSize:
|
||||
description: 'Swap file size in GB (e.g., 8 for 8GB)'
|
||||
required: false
|
||||
default: 8
|
||||
|
||||
jobs:
|
||||
linux-arm64:
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [armcompute, ""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
- swap_size: ${{ github.event.inputs.swapSize }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- name: Configure swap space
|
||||
shell: bash
|
||||
run: |
|
||||
sudo fallocate -l 16G /swapfile
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
sudo sysctl vm.swappiness=60
|
||||
free -h
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
LIBND4J_CLASSIFIER="linux-arm64"
|
||||
echo "LIBND4J_CLASSIFIER=${LIBND4J_CLASSIFIER}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_JAR=openblas-0.3.28-1.5.11-linux-arm64.jar" >> $GITHUB_ENV
|
||||
echo "ARMCOMPUTE_TARGET=arm64-v8a" >> $GITHUB_ENV
|
||||
if [ "${{ github.event.inputs.libnd4jUrl }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Posx-aarch64-protoc -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=${LIBND4J_CLASSIFIER} -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.classifier=${LIBND4J_CLASSIFIER}-${{ matrix.helper }}"
|
||||
else
|
||||
mvn_ext=""
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
- name: Setup Path
|
||||
shell: bash
|
||||
run: |
|
||||
echo "/opt/protobuf/bin" >> $GITHUB_PATH
|
||||
echo "/opt/cmake/bin" >> $GITHUB_PATH
|
||||
- name: Setup OpenBLAS and ARM Compute Library
|
||||
shell: bash
|
||||
run: |
|
||||
# Setup OpenBLAS
|
||||
mkdir -p "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/${OPENBLAS_JAR}
|
||||
unzip ${OPENBLAS_JAR}
|
||||
OPENBLAS_DIR="${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-arm64"
|
||||
echo "OPENBLAS_PATH=${OPENBLAS_DIR}" >> "$GITHUB_ENV"
|
||||
# Ensure the symlink exists for consistency
|
||||
ln -sf "${OPENBLAS_DIR}/libopenblas.so.0" "${OPENBLAS_DIR}/libopenblas.so"
|
||||
# Setup ARM Compute Library if selected
|
||||
if [ "${{ matrix.helper }}" == "armcompute" ]; then
|
||||
ARMCOMPUTE_HOME="${GITHUB_WORKSPACE}/armcompute_home"
|
||||
mkdir -p "${ARMCOMPUTE_HOME}"
|
||||
cd "${ARMCOMPUTE_HOME}"
|
||||
wget https://github.com/ARM-software/ComputeLibrary/releases/download/v25.04/arm_compute-v25.04-linux-aarch64-cpu-bin.tar.gz
|
||||
tar -xzf arm_compute-v25.04-linux-aarch64-cpu-bin.tar.gz
|
||||
# Set ARMCOMPUTE_ROOT to the explicit path of the extracted contents
|
||||
EXTRACTED_DIR="${ARMCOMPUTE_HOME}/arm_compute-v25.04-linux-aarch64-cpu-bin"
|
||||
echo "ARMCOMPUTE_ROOT=${EXTRACTED_DIR}" >> "$GITHUB_ENV"
|
||||
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${EXTRACTED_DIR}/lib/armv8a-neon" >> "$GITHUB_ENV"
|
||||
fi
|
||||
- name: Build with Maven
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
PROTO_EXEC: /opt/protobuf/bin/protoc
|
||||
run: |
|
||||
# Pre-build native libraries with CMake if not using a pre-packaged libnd4j
|
||||
if [ "${{ github.event.inputs.libnd4jUrl }}" == '' ]; then
|
||||
echo "Starting manual pre-build of libnd4j..."
|
||||
|
||||
# Define CMake arguments for building libnd4j
|
||||
cmake_args="-DOPENBLAS_PATH=${OPENBLAS_PATH}"
|
||||
|
||||
# Add armcompute helper flags if enabled
|
||||
if [ "${{ matrix.helper }}" == "armcompute" ]; then
|
||||
echo "Enabling armcompute helper for native build."
|
||||
cmake_args="${cmake_args} -DHELPERS_armcompute=true -DARMCOMPUTE_ROOT=${ARMCOMPUTE_ROOT}"
|
||||
fi
|
||||
|
||||
# Navigate to libnd4j build directory
|
||||
cd "${GITHUB_WORKSPACE}/libnd4j"
|
||||
mkdir -p blasbuild/linux-arm64
|
||||
cd blasbuild/linux-arm64
|
||||
|
||||
# Configure with CMake
|
||||
echo "Configuring with CMake args: ${cmake_args}"
|
||||
cmake ${cmake_args} ../..
|
||||
|
||||
# Build with specified number of threads
|
||||
echo "Building with make -j${{ github.event.inputs.buildThreads }}"
|
||||
make -j${{ github.event.inputs.buildThreads }}
|
||||
cd ${GITHUB_WORKSPACE}
|
||||
echo "Manual pre-build of libnd4j finished."
|
||||
fi
|
||||
|
||||
# Run the main Maven build
|
||||
echo "Running Maven command: ${COMMAND}"
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
- name: Cleanup swap
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
sudo swapoff /swapfile || true
|
||||
sudo rm -f /swapfile || true
|
||||
@@ -0,0 +1,311 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 4
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Whether to download libnd4j using https://github.com/KonduitAI/gh-actions-libnd4j-urls/ for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
jobs:
|
||||
linux-x86_64-cuda-12-3:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ cudnn, "" ]
|
||||
extension: [ "" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
env:
|
||||
CUDA_PATH: /usr/local/cuda-12.3
|
||||
CUDNN_ROOT_DIR: /usr/local/cuda-12.3
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-cuda,:nd4j-cuda-preset'
|
||||
elif [ "${{ matrix.helper }}" == '' ]; then
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-cuda,:nd4j-cuda-preset,:libnd4j,:nd4j-cuda-platform'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-cuda,:nd4j-cuda-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Pcuda -Dlibnd4j.generate.flatc=ON --no-transfer-progress -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda -Pcuda --also-make -pl ${modules} -Dlibnd4j.compute='8.6 9.0' -Dlibnd4j.cpu.compile.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cuda --also-make -Pcuda clean --batch-mode package deploy -DskipTests"
|
||||
libnd4j_download_file_url=""
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-cuda-12.3-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
libnd4j_download_file_url="linux-cuda-12.3-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.classifier=linux-x86_64-cuda-12.3-${{ matrix.helper }}"
|
||||
libnd4j_download_file_url="linux-cuda-12.3-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
else
|
||||
mvn_ext=" -Dlibnd4j.classifier=linux-x86_64-cuda-12.3"
|
||||
libnd4j_download_file_url="linux-cuda-12.3-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ matrix.runs_on }}-protobuf
|
||||
restore-keys: ${{ matrix.runs_on }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master
|
||||
env:
|
||||
cuda: 12.3.0
|
||||
GCC: 11
|
||||
if: steps.cache-cuda-123.outputs.cache-hit != 'true'
|
||||
|
||||
# Set up CUDA environment paths
|
||||
- name: Setup CUDA PATH
|
||||
run: |
|
||||
echo "PATH=/usr/local/cuda-12.3/bin:$PATH" >> $GITHUB_ENV
|
||||
echo "LD_LIBRARY_PATH=/usr/local/cuda-12.3/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||
|
||||
# Debug CUDA installation
|
||||
- name: Debug CUDA Installation
|
||||
run: |
|
||||
echo "Debugging CUDA installation"
|
||||
echo "Current PATH: $PATH"
|
||||
echo "Current LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
|
||||
echo "Contents of /usr/local/cuda-12.3:"
|
||||
ls -la /usr/local/cuda-12.3
|
||||
echo "Contents of /usr/local/cuda-12.3/bin:"
|
||||
ls -la /usr/local/cuda-12.3/bin
|
||||
echo "Finding nvcc:"
|
||||
find /usr/local -name nvcc 2>/dev/null
|
||||
echo "CUDA installation info:"
|
||||
if [ -f /usr/local/cuda-12.3/bin/nvcc ]; then
|
||||
/usr/local/cuda-12.3/bin/nvcc --version
|
||||
else
|
||||
echo "nvcc not found in expected location"
|
||||
# Try to find in alternative locations
|
||||
find /usr -name nvcc 2>/dev/null
|
||||
fi
|
||||
# Check symlinks
|
||||
echo "Checking CUDA symlinks:"
|
||||
ls -la /usr/local/cuda*
|
||||
# Create symlink if needed
|
||||
if [ ! -f /usr/local/cuda-12.3/bin/nvcc ]; then
|
||||
echo "Attempting to fix nvcc location"
|
||||
# Check if nvcc exists in alternative location
|
||||
nvcc_path=$(find /usr -name nvcc 2>/dev/null | head -1)
|
||||
if [ -n "$nvcc_path" ]; then
|
||||
echo "Found nvcc at $nvcc_path"
|
||||
sudo mkdir -p /usr/local/cuda-12.3/bin
|
||||
sudo ln -sf $nvcc_path /usr/local/cuda-12.3/bin/nvcc
|
||||
echo "Created symlink to nvcc"
|
||||
ls -la /usr/local/cuda-12.3/bin/nvcc
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify the CUDA installation and PATH setup
|
||||
- name: Verify CUDA Setup
|
||||
run: |
|
||||
echo "Verifying CUDA installation and PATH setup"
|
||||
echo $PATH
|
||||
ls -la /usr/local/cuda-12.3/bin
|
||||
which nvcc || echo "nvcc not found in PATH"
|
||||
nvcc --version || echo "nvcc command failed"
|
||||
# If still failing, try direct path
|
||||
if ! which nvcc; then
|
||||
echo "Trying direct path to nvcc"
|
||||
/usr/local/cuda-12.3/bin/nvcc --version || echo "Direct nvcc command failed"
|
||||
fi
|
||||
|
||||
- name: Run cuda compilation on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
LIBND4J_HOME_SUFFIX: cuda
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
# Explicitly set PATH and other environment variables for this step
|
||||
export PATH=/usr/local/cuda-12.3/bin:/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.3/lib64:$LD_LIBRARY_PATH
|
||||
|
||||
# Final check for nvcc before proceeding
|
||||
echo "Final check for nvcc:"
|
||||
which nvcc || echo "nvcc still not found in PATH"
|
||||
if ! which nvcc; then
|
||||
echo "WARNING: nvcc not in PATH, trying to diagnose and fix..."
|
||||
direct_nvcc=$(find /usr -name nvcc 2>/dev/null | head -1)
|
||||
if [ -n "$direct_nvcc" ]; then
|
||||
echo "Found nvcc at $direct_nvcc"
|
||||
sudo ln -sf $direct_nvcc /usr/bin/nvcc
|
||||
echo "Created global symlink to nvcc"
|
||||
which nvcc
|
||||
else
|
||||
echo "ERROR: Could not find nvcc anywhere. Build may fail!"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Test that nvcc is properly in the PATH
|
||||
echo "Checking nvcc availability:"
|
||||
which nvcc
|
||||
nvcc --version
|
||||
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
sudo apt-get autoremove
|
||||
sudo apt-get clean
|
||||
bash ./change-cuda-versions.sh 12.3
|
||||
|
||||
# Note: we need this for the cudnn helpers, our cmake can't find it otherwise.
|
||||
# See here: https://github.com/eclipse/deeplearning4j/blob/master/libnd4j/CMakeLists.txt#L298
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
@@ -0,0 +1,304 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 4
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Whether to download libnd4j using https://github.com/KonduitAI/gh-actions-libnd4j-urls/ for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
jobs:
|
||||
linux-x86_64-cuda-12-6:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ cudnn, "" ]
|
||||
extension: [ "" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
env:
|
||||
CUDA_PATH: /usr/local/cuda-12.6
|
||||
CUDNN_ROOT_DIR: /usr/local/cuda-12.6
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- name: Configure swap space
|
||||
shell: bash
|
||||
run: |
|
||||
echo "=== Initial system status ==="
|
||||
free -h
|
||||
df -h
|
||||
echo "Available disk space on root:"
|
||||
df -h /
|
||||
|
||||
# Check if swap is already enabled
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "Swap already configured:"
|
||||
swapon --show
|
||||
else
|
||||
echo "Configuring swap space..."
|
||||
|
||||
# Use input parameter for swap size, default to 12GB like Android workflow
|
||||
SWAP_SIZE="12G"
|
||||
echo "Creating ${SWAP_SIZE} swap file..."
|
||||
|
||||
# Create swap file
|
||||
sudo fallocate -l ${SWAP_SIZE} /swapfile || {
|
||||
sudo dd if=/dev/zero of=/swapfile bs=1G count=12 status=progress
|
||||
}
|
||||
|
||||
# Set up swap
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
|
||||
# Verify swap is active
|
||||
echo "=== Swap configuration completed ==="
|
||||
free -h
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
# Tune swappiness for build workloads - using Android workflow settings
|
||||
# Lower values (10-20) prefer RAM, higher values (60+) use swap more aggressively
|
||||
echo "Current swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.swappiness=80
|
||||
echo "Adjusted swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.overcommit_memory=2 # Allow overcommit
|
||||
sudo sysctl vm.overcommit_ratio=80
|
||||
sudo sysctl vm.vfs_cache_pressure=50
|
||||
sudo sysctl vm.dirty_ratio=10
|
||||
sudo sysctl vm.dirty_background_ratio=5
|
||||
sudo sysctl kernel.shmmax=68719476736
|
||||
# Show final memory status
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-cuda-12.6,:nd4j-cuda-12.6-preset'
|
||||
elif [ "${{ matrix.helper }}" == '' ]; then
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-cuda-12.6,:nd4j-cuda-12.6-preset,:libnd4j,:nd4j-cuda-12.6-platform'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-cuda-12.6,:nd4j-cuda-12.6-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Pcuda -Dlibnd4j.generate.flatc=ON --no-transfer-progress -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda -pl ${modules} -Dlibnd4j.compute='8.6 9.0' -Dlibnd4j.cpu.compile.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cuda --also-make clean --batch-mode package deploy -DskipTests"
|
||||
libnd4j_download_file_url=""
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-cuda-12.6-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
libnd4j_download_file_url="linux-cuda-12.6-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.classifier=linux-x86_64-cuda-12.6-${{ matrix.helper }}"
|
||||
libnd4j_download_file_url="linux-cuda-12.6-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
else
|
||||
mvn_ext=" -Dlibnd4j.classifier=linux-x86_64-cuda-12.6"
|
||||
libnd4j_download_file_url="linux-cuda-12.6-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ matrix.runs_on }}-protobuf
|
||||
restore-keys: ${{ matrix.runs_on }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master
|
||||
env:
|
||||
cuda: 12.6.0
|
||||
GCC: 11
|
||||
if: steps.cache-cuda-126.outputs.cache-hit != 'true'
|
||||
|
||||
# Set up CUDA environment paths
|
||||
- name: Setup CUDA PATH
|
||||
run: |
|
||||
echo "PATH=/usr/local/cuda-12.6/bin:$PATH" >> $GITHUB_ENV
|
||||
echo "LD_LIBRARY_PATH=/usr/local/cuda-12.6/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||
|
||||
# Verify the CUDA installation and PATH setup
|
||||
- name: Verify CUDA Setup
|
||||
run: |
|
||||
echo "Verifying CUDA installation and PATH setup"
|
||||
echo $PATH
|
||||
ls -la /usr/local/cuda-12.6/bin
|
||||
which nvcc || echo "nvcc not found in PATH"
|
||||
nvcc --version || echo "nvcc command failed"
|
||||
|
||||
- name: Run cuda compilation on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
LIBND4J_HOME_SUFFIX: cuda
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
# Explicitly set PATH and other environment variables for this step
|
||||
export PATH=/usr/local/cuda-12.6/bin:/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.6/lib64:$LD_LIBRARY_PATH
|
||||
|
||||
# Test that nvcc is properly in the PATH
|
||||
echo "Checking nvcc availability:"
|
||||
which nvcc
|
||||
nvcc --version
|
||||
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
sudo apt-get autoremove
|
||||
sudo apt-get clean
|
||||
bash ./change-cuda-versions.sh 12.6
|
||||
|
||||
# Note: we need this for the cudnn helpers, our cmake can't find it otherwise.
|
||||
# See here: https://github.com/eclipse/deeplearning4j/blob/master/libnd4j/CMakeLists.txt#L298
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
@@ -0,0 +1,210 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
jobs:
|
||||
#Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
|
||||
linux-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [""]
|
||||
extension: [compat]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space Before Build
|
||||
run: |
|
||||
echo "Disk space before cleanup:"
|
||||
df -h
|
||||
sudo rm -rf /usr/local/.ghcup
|
||||
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
||||
sudo rm -rf /usr/local/lib/android/sdk/ndk
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf /usr/local/share/boost
|
||||
sudo apt-get clean
|
||||
echo "Disk space after cleanup:"
|
||||
df -h
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
command="mvn -pl ':nd4j-native-preset,:libnd4j,:nd4j-native' ${{ matrix.mvn_ext }} -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=linux-x86_64 -Pcpu --also-make --batch-mode -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.extension }}"
|
||||
else
|
||||
mvn_ext=""
|
||||
fi
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
echo "Adding libnd4j download"
|
||||
echo "LIBND4J_FILE_NAME=${libnd4j_download_file_url}" >> $GITHUB_ENV
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "DEPLOY_COMMAND=${command} deploy -Dlibnd4j.cpu.compile.skip=true -Djavacpp.compiler.skip=true -Djavacpp.parser.skip=true" >> $GITHUB_ENV
|
||||
echo "INSTALL_COMMAND=${command} install" >> "$GITHUB_ENV"
|
||||
echo "COMMAND=${command}" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
|
||||
- name: Build on linux-x86_64
|
||||
uses: ./.github/actions/build-centos
|
||||
env:
|
||||
JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64/
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
|
||||
- name: Deploy
|
||||
shell: bash
|
||||
run: |
|
||||
# Docker builds are root owned
|
||||
sudo chown -R $USER $GITHUB_WORKSPACE/*
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
export COMMAND="${DEPLOY_COMMAND}"
|
||||
bash "$GITHUB_WORKSPACE/release-specified-component.sh" "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${DEPLOY_COMMAND}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${DEPLOY_COMMAND}"
|
||||
fi
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
@@ -0,0 +1,360 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
swapSize:
|
||||
description: 'Swap file size in GB (e.g., 4 for 4GB)'
|
||||
required: false
|
||||
default: 12
|
||||
|
||||
jobs:
|
||||
#Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
|
||||
linux-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [onednn,""]
|
||||
extension: [avx2,avx512,""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: libnd4j download URL
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: Release repository id
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
- swap_size: ${{ github.event.inputs.swapSize }}
|
||||
experimental: true
|
||||
name: Swap file size in GB
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- name: Configure swap space
|
||||
shell: bash
|
||||
run: |
|
||||
echo "=== Initial system status ==="
|
||||
free -h
|
||||
df -h
|
||||
echo "Available disk space on root:"
|
||||
df -h /
|
||||
|
||||
# Check if swap is already enabled
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "Swap already configured:"
|
||||
swapon --show
|
||||
else
|
||||
echo "Configuring swap space..."
|
||||
|
||||
# Use input parameter for swap size, default to 12GB like Android workflow
|
||||
SWAP_SIZE="${{ matrix.swap_size }}G"
|
||||
echo "Creating ${SWAP_SIZE} swap file..."
|
||||
|
||||
# Create swap file
|
||||
sudo fallocate -l ${SWAP_SIZE} /swapfile || {
|
||||
echo "fallocate failed, trying dd..."
|
||||
sudo dd if=/dev/zero of=/swapfile bs=1G count=${{ matrix.swap_size }} status=progress
|
||||
}
|
||||
|
||||
# Set up swap
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
|
||||
# Verify swap is active
|
||||
echo "=== Swap configuration completed ==="
|
||||
free -h
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
# Tune swappiness for build workloads - using Android workflow settings
|
||||
# Lower values (10-20) prefer RAM, higher values (60+) use swap more aggressively
|
||||
echo "Current swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.swappiness=80
|
||||
echo "Adjusted swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.overcommit_memory=2 # Allow overcommit
|
||||
sudo sysctl vm.overcommit_ratio=80
|
||||
|
||||
# Show final memory status
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=linux-x86_64 -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.extension }}"
|
||||
else
|
||||
mvn_ext=""
|
||||
fi
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
echo "Adding libnd4j download"
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
|
||||
- name: Cache cmake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cmake-
|
||||
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-protobuf-
|
||||
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
|
||||
- uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/openblas-0.3.28-1.5.11-linux-x86_64.jar
|
||||
unzip openblas-0.3.28-1.5.11-linux-x86_64.jar
|
||||
cd ..
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64" >> "$GITHUB_ENV"
|
||||
cp ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64/libopenblas.so.0 ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64/libopenblas.so
|
||||
if: ${{ github.event.inputs.libnd4jUrl != '' }}
|
||||
|
||||
- name: Monitor memory during build
|
||||
shell: bash
|
||||
run: |
|
||||
# Start background memory monitoring
|
||||
(while true; do
|
||||
echo "$(date '+%H:%M:%S'): $(free -h | head -n 2 | tail -n 1)"
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "$(date '+%H:%M:%S'): Swap: $(swapon --show --noheadings)"
|
||||
fi
|
||||
sleep 120 # Check every 2 minutes
|
||||
done) > memory_monitor.log 2>&1 &
|
||||
MONITOR_PID=$!
|
||||
echo "MONITOR_PID=${MONITOR_PID}" >> $GITHUB_ENV
|
||||
echo "Started memory monitoring (PID: ${MONITOR_PID})"
|
||||
|
||||
- name: Build on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
RLIMIT_AS: 6442450944 # 6GB virtual memory limit
|
||||
RLIMIT_DATA: 4294967296 # 4GB data segment limit
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
echo "swap size ${{ matrix.swap_size }}GB"
|
||||
|
||||
# Show memory status before build
|
||||
echo "=== Memory status before build ==="
|
||||
free -h
|
||||
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$OPENBLAS_PATH"
|
||||
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
|
||||
- name: Show memory usage summary
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
# Stop memory monitoring
|
||||
if [ -n "$MONITOR_PID" ]; then
|
||||
kill $MONITOR_PID 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "=== Swap usage ==="
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
echo "=== Memory usage during build ==="
|
||||
if [ -f memory_monitor.log ]; then
|
||||
cat memory_monitor.log
|
||||
else
|
||||
echo "No memory monitoring log found"
|
||||
fi
|
||||
|
||||
- name: Cleanup swap
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Cleaning up swap configuration..."
|
||||
sudo swapoff /swapfile 2>/dev/null || true
|
||||
sudo rm -f /swapfile 2>/dev/null || true
|
||||
echo "Swap cleanup completed"
|
||||
@@ -0,0 +1,222 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 3
|
||||
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: macos-14
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
mac-arm64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [""]
|
||||
extension: [""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Dlibnd4j.arch=armv8-a -Dlibnd4j.platform=macosx-arm64 -Djavacpp.platform=macosx-arm64 -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=macosx-arm64 -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext="-Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-arm64-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=${{ matrix.helper }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-arm64-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }}"
|
||||
else
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-arm64"
|
||||
fi
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
echo "Adding libnd4j download"
|
||||
echo "LIBND4J_FILE_NAME=${libnd4j_download_file_url}" >> $GITHUB_ENV
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/MacOSX10*
|
||||
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Update path for gnu sed
|
||||
shell: bash
|
||||
run: |
|
||||
brew install gnu-sed unzip ccache gcc swig autoconf-archive automake libomp libtool libusb ant maven nasm xz pkg-config sdl bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python
|
||||
echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/openblas-0.3.28-1.5.11-macosx-arm64.jar
|
||||
unzip openblas-0.3.28-1.5.11-macosx-arm64.jar
|
||||
cd ..
|
||||
cp ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-arm64/libopenblas.0.dylib ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-arm64/libopenblas.dylib
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-arm64/" >> "$GITHUB_ENV"
|
||||
|
||||
if: ${{ matrix.libnd4j_file_download != '' }}
|
||||
|
||||
- name: Build and install
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
|
||||
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
|
||||
brew list
|
||||
brew list --cask
|
||||
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release with command ${COMMAND}"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 3
|
||||
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: macos-10.15
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
mac-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ onednn,"" ]
|
||||
extension: [ avx2,avx512,"" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=macosx-x86_64 -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-x86_64-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=${{ matrix.helper }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-x86_64-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }}"
|
||||
else
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-x86_64"
|
||||
fi
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
echo "Adding libnd4j download"
|
||||
echo "LIBND4J_FILE_NAME=${{ matrix.libnd4j_file_download }}/${libnd4j_download_file_url}" >> $GITHUB_ENV
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
|
||||
|
||||
|
||||
- name: Import GPG Key
|
||||
uses: crazy-max/ghaction-import-gpg@v1
|
||||
env:
|
||||
GPG_PRIVATE_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Update path for gnu sed
|
||||
shell: bash
|
||||
run: |
|
||||
brew install gpg1 gnu-sed unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python
|
||||
echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.19-1.5.7/openblas-0.3.19-1.5.7-macosx-x86_64.jar
|
||||
unzip openblas-0.3.19-1.5.7-macosx-x86_64.jar
|
||||
cd ..
|
||||
cp ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-x86_64/libopenblas.0.dylib ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-x86_64/libopenblas.dylib
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-x86_64/" >> "$GITHUB_ENV"
|
||||
|
||||
if: ${{ matrix.libnd4j_file_download != '' }}
|
||||
|
||||
- name: Build and install
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
|
||||
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
gpg --version
|
||||
gpg1 --version
|
||||
brew list
|
||||
brew list --cask
|
||||
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release with command ${COMMAND}"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
# Workflow for CUDA 12.3 Windows build
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 4
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
libnd4jUrl:
|
||||
description: 'Whether to download libnd4j using https://github.com/KonduitAI/gh-actions-libnd4j-urls/ for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: windows-2019
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
windows-x86_64-cuda-12-3:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ cudnn,"" ]
|
||||
extension: [ "" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set NVCC Compiler Flags
|
||||
shell: powershell
|
||||
run: |
|
||||
echo "Setting NVCC environment variables to allow unsupported compiler..."
|
||||
echo "CUDAFLAGS=--allow-unsupported-compiler -D__NVCC_ALLOW_UNSUPPORTED_COMPILER__" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "NVCC_APPEND_FLAGS=--allow-unsupported-compiler -D__NVCC_ALLOW_UNSUPPORTED_COMPILER__" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "CMAKE_CUDA_FLAGS=--allow-unsupported-compiler -D__NVCC_ALLOW_UNSUPPORTED_COMPILER__" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
- name: Free Disk Space (Windows)
|
||||
shell: powershell
|
||||
run: |
|
||||
# Show initial disk space
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
# Remove Windows Defender scan history
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed Windows Defender scan history"
|
||||
|
||||
# Clear Windows temp folders
|
||||
Remove-Item -Path "$env:SystemRoot\Temp\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows temp folders"
|
||||
|
||||
# Clear Windows Update cache safely (without stopping/starting service)
|
||||
try {
|
||||
Remove-Item -Path "$env:SystemRoot\SoftwareDistribution\Download\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows Update download cache"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Could not clear Windows Update cache. Continuing..."
|
||||
}
|
||||
|
||||
# Clean package manager caches
|
||||
if (Test-Path -Path "C:\npm\cache") {
|
||||
Remove-Item -Path "C:\npm\cache\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared NPM cache"
|
||||
}
|
||||
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Chocolatey cache"
|
||||
|
||||
# Remove Docker images if Docker is installed
|
||||
try {
|
||||
if (Get-Command "docker" -ErrorAction SilentlyContinue) {
|
||||
docker image prune -a -f
|
||||
docker container prune -f
|
||||
docker volume prune -f
|
||||
Write-Host "Pruned Docker resources"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to prune Docker resources. Continuing..."
|
||||
}
|
||||
|
||||
# Remove .NET SDK/Runtime backup folders
|
||||
if (Test-Path -Path "$env:ProgramData\Microsoft\.NET\*.backup") {
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\.NET\*.backup" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed .NET backup folders"
|
||||
}
|
||||
|
||||
# Clear Azure artifacts cache
|
||||
if (Test-Path -Path "$env:LOCALAPPDATA\Microsoft\Azure\*") {
|
||||
Remove-Item -Path "$env:LOCALAPPDATA\Microsoft\Azure\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Azure artifacts cache"
|
||||
}
|
||||
|
||||
# Optimize Windows Component Store
|
||||
try {
|
||||
Start-Process -FilePath "dism.exe" -ArgumentList "/online /Cleanup-Image /StartComponentCleanup" -NoNewWindow -Wait
|
||||
Write-Host "Optimized Windows Component Store"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to optimize Windows Component Store. Continuing..."
|
||||
}
|
||||
|
||||
# Show final disk space
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: powershell
|
||||
run: |
|
||||
if ( "${{ matrix.libnd4j_file_download }}" -ne "" ) {
|
||||
$modules=" :nd4j-cuda-preset,:nd4j-cuda"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$modules=":nd4j-cuda-preset,:nd4j-cuda,libnd4j"
|
||||
} elseif ( "${{ matrix.helper }}" -eq "" ) {
|
||||
$modules=":nd4j-cuda-preset,:nd4j-cuda,libnd4j,:nd4j-cuda-platform"
|
||||
}
|
||||
|
||||
$command="mvn ${{ matrix.mvn_ext }} -Pcuda -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda -Pcuda --also-make -pl $modules -Dlibnd4j.compute=`"8.6 9.0`" -Dlibnd4j.cpu.compile.skip=true -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests"
|
||||
|
||||
if ( "${{ matrix.helper }}" -ne "" -And "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.chip=cuda -Dlibnd4j.classifier=windows-x86_64-cuda-12.3-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests"
|
||||
$libnd4j_download_file_url="windows-cuda-12.3-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-cuda-12.3-${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests"
|
||||
$libnd4j_download_file_url="windows-cuda-12.3-${{ matrix.helper }}"
|
||||
} else {
|
||||
$libnd4j_download_file_url="windows-cuda-12.3"
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-cuda-12.3"
|
||||
}
|
||||
|
||||
$command2= -join("$($command)","$($mvn_ext)");
|
||||
$to_write = -join("COMMAND=","$($command2)");
|
||||
if ( "${{ matrix.libnd4j_file_download }}" -ne "") {
|
||||
Write-Host "Adding libnd4j download URL to GITHUB_ENV"
|
||||
$libnd4j_url_to_write = -join("LIBND4J_FILE_NAME=","$(${{ matrix.libnd4j_file_download }}/$libnd4j_download_file_url)");
|
||||
echo $libnd4j_url_to_write | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
}
|
||||
Write-Host "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to $($command2) and the libnd4j bootstrap file name to $($libnd4j_download_file_url)"
|
||||
echo $command2 | Out-File -FilePath "$env:GITHUB_WORKSPACE/mvn-command.bat" -Encoding utf8
|
||||
echo $to_write | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
- name: Install MSYS2 and dependencies
|
||||
uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: MINGW64
|
||||
update: true
|
||||
install: >-
|
||||
base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake patch
|
||||
mingw-w64-x86_64-make mingw-w64-x86_64-gnupg mingw-w64-x86_64-cmake mingw-w64-x86_64-nasm
|
||||
mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-fortran
|
||||
mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-x86_64-ragel
|
||||
|
||||
|
||||
- name: Cache cuda install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-123
|
||||
with:
|
||||
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.3
|
||||
key: ${{ matrix.runs_on }}-cuda-12.3-${{ matrix.helper }}-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ matrix.runs_on }}-cuda-12.3-${{ matrix.helper }}-
|
||||
|
||||
- name: Install CUDA 12.3 (if not cached)
|
||||
if: steps.cache-cuda-123.outputs.cache-hit != 'true'
|
||||
shell: powershell
|
||||
env:
|
||||
CUDA_VERSION: "12.3"
|
||||
run: |
|
||||
$scriptUrl = "https://raw.githubusercontent.com/KonduitAI/cuda-install/master/.github/actions/install-cuda-windows/install_cuda_windows.ps1"
|
||||
$scriptPath = ".\install_cuda_windows.ps1"
|
||||
Write-Host "Downloading CUDA install script from $scriptUrl..."
|
||||
Invoke-WebRequest $scriptUrl -OutFile $scriptPath -UseBasicParsing
|
||||
if (Test-Path $scriptPath) {
|
||||
Write-Host "Download complete. Executing script with CUDA_VERSION=$($env:CUDA_VERSION)..."
|
||||
& $scriptPath
|
||||
} else {
|
||||
Write-Error "Failed to download CUDA install script!"
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
- name: Setup Windows PATH and Compiler Environment
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "Setting up Windows PATH and Compiler Environment..."
|
||||
|
||||
# Find Visual Studio installation
|
||||
$vsPaths = @(
|
||||
"${env:ProgramFiles}\Microsoft Visual Studio\2022\Enterprise",
|
||||
"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\Enterprise",
|
||||
"${env:ProgramFiles}\Microsoft Visual Studio\2019\Enterprise",
|
||||
"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2019\Enterprise"
|
||||
)
|
||||
|
||||
$vsPath = $null
|
||||
foreach ($path in $vsPaths) {
|
||||
if (Test-Path $path) {
|
||||
$vsPath = $path
|
||||
Write-Host "Found Visual Studio at: $vsPath"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $vsPath) {
|
||||
Write-Error "Visual Studio not found in any expected location"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Setup MSVC environment using vcvars64.bat
|
||||
$vcvarsPath = "$vsPath\VC\Auxiliary\Build\vcvars64.bat"
|
||||
if (-not (Test-Path $vcvarsPath)) {
|
||||
Write-Error "vcvars64.bat not found at $vcvarsPath"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "Setting up MSVC environment using: $vcvarsPath"
|
||||
|
||||
# Get environment after vcvars64.bat
|
||||
$tempFile = [System.IO.Path]::GetTempFileName()
|
||||
$cmd = "`"$vcvarsPath`" && set"
|
||||
Write-Host "Executing: cmd.exe /c $cmd"
|
||||
|
||||
try {
|
||||
cmd.exe /c $cmd | Out-File -FilePath $tempFile -Encoding ASCII
|
||||
|
||||
# Parse and set environment variables
|
||||
$envVarsSet = 0
|
||||
Get-Content $tempFile | ForEach-Object {
|
||||
if ($_ -match '^([^=]+)=(.*)$') {
|
||||
$name = $matches[1]
|
||||
$value = $matches[2]
|
||||
|
||||
# Skip problematic variables
|
||||
if ($name -notmatch '^(TEMP|TMP|RANDOM)$') {
|
||||
[Environment]::SetEnvironmentVariable($name, $value, 'Process')
|
||||
echo "$name=$value" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
$envVarsSet++
|
||||
}
|
||||
}
|
||||
}
|
||||
Write-Host "Set $envVarsSet environment variables from vcvars64.bat"
|
||||
|
||||
Remove-Item $tempFile -ErrorAction SilentlyContinue
|
||||
}
|
||||
catch {
|
||||
Write-Error "Failed to execute vcvars64.bat: $_"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Setup CUDA paths
|
||||
$cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.3"
|
||||
$msysPath = "C:\msys64\mingw64\bin;C:\msys64\usr\bin"
|
||||
|
||||
Write-Host "Setting CUDA paths..."
|
||||
echo "CUDA_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "CUDNN_ROOT_DIR=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# Update PATH with MSYS2 and CUDA
|
||||
$newPath = "$msysPath;$cudaPath\bin;$cudaPath\libnvvp;"
|
||||
Write-Host "Adding to PATH: $newPath"
|
||||
echo $newPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
|
||||
- name: Run cuda build
|
||||
shell: cmd
|
||||
env:
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
PUBLISH_TO: central
|
||||
LIBND4J_HOME_SUFFIX: cuda
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
|
||||
run: |
|
||||
echo "=== CUDA Build Started ==="
|
||||
echo "Libnd4j build threads: ${{ matrix.build_threads }}"
|
||||
echo "Deploy to release staging: %PERFORM_RELEASE%"
|
||||
echo "Release version: %RELEASE_VERSION%"
|
||||
echo "Snapshot version: %SNAPSHOT_VERSION%"
|
||||
echo "Debug enabled: ${{ matrix.debug_enabled }}"
|
||||
echo "Libnd4j URL input: %LIBND4J_FILE_NAME%"
|
||||
echo "Maven flags input: ${{ matrix.mvn_flags }}"
|
||||
echo "Server id: ${{ matrix.server_id }}"
|
||||
echo "Release repo id: %RELEASE_REPO_ID%"
|
||||
echo "CUDA Path from env: %CUDA_PATH%"
|
||||
echo "CUDNN Root Dir from env: %CUDNN_ROOT_DIR%"
|
||||
echo "=== Environment Check ==="
|
||||
|
||||
echo "Verifying compiler availability..."
|
||||
where cl.exe >nul 2>&1 || (
|
||||
echo "ERROR: cl.exe not found in PATH"
|
||||
echo "Current PATH: %PATH%"
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
where nvcc.exe >nul 2>&1 || (
|
||||
echo "ERROR: nvcc.exe not found in PATH"
|
||||
echo "Current PATH: %PATH%"
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo "Compiler verification successful:"
|
||||
echo "cl.exe location:"
|
||||
where cl.exe
|
||||
cl.exe 2>&1 | findstr "Microsoft"
|
||||
|
||||
echo "nvcc.exe location:"
|
||||
where nvcc.exe
|
||||
nvcc.exe --version | findstr "release"
|
||||
|
||||
echo "=== Build Process ==="
|
||||
set MSYSTEM=MINGW64
|
||||
echo "Running cuda build..."
|
||||
echo "Maven command from file:"
|
||||
type "%GITHUB_WORKSPACE%\mvn-command.bat"
|
||||
|
||||
bash ./change-cuda-versions.sh 12.3
|
||||
|
||||
Rem Ensure CUDNN_ROOT_DIR is set if needed by cmake; often CUDA_PATH is sufficient
|
||||
if not defined CUDNN_ROOT_DIR set CUDNN_ROOT_DIR=%CUDA_PATH%
|
||||
|
||||
if "%PERFORM_RELEASE%"=="1" (
|
||||
echo "Running release build..."
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.3 "%HELPER%" "%EXTENSION%"
|
||||
bash "./release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%"
|
||||
) else (
|
||||
echo "Running snapshot build..."
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.3 "%HELPER%" "%EXTENSION%"
|
||||
call "%GITHUB_WORKSPACE%\mvn-command.bat"
|
||||
)
|
||||
@@ -0,0 +1,631 @@
|
||||
# Enhanced CUDA 12.6 Windows Build Workflow with Clean Toolchain Management
|
||||
name: CUDA 12.6 Windows Build
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j (controls memory usage)'
|
||||
required: true
|
||||
default: '4'
|
||||
deployToReleaseStaging:
|
||||
description: 'Deploy to release staging'
|
||||
required: false
|
||||
default: '0'
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: '1.0.0-M3'
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: '1.0.0-SNAPSHOT'
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default: ''
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: 'central'
|
||||
mvnFlags:
|
||||
description: 'Extra maven flags'
|
||||
required: false
|
||||
default: ''
|
||||
libnd4jUrl:
|
||||
description: 'Download libnd4j from URL'
|
||||
required: false
|
||||
default: ''
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: 'windows-2022'
|
||||
debug_enabled:
|
||||
description: 'Enable tmate debugging'
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
jobs:
|
||||
windows-x86_64-cuda-12-6:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ 'cudnn', '' ]
|
||||
include:
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.12.1
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Free Disk Space
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Freeing Disk Space ==="
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
# Clean Windows temp and cache directories
|
||||
$cleanupPaths = @(
|
||||
"$env:TEMP\*",
|
||||
"$env:SystemRoot\Temp\*",
|
||||
"$env:ProgramData\Microsoft\Windows Defender\Scans\History\*",
|
||||
"$env:SystemRoot\SoftwareDistribution\Download\*"
|
||||
)
|
||||
|
||||
foreach ($path in $cleanupPaths) {
|
||||
try {
|
||||
Remove-Item -Path $path -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleaned: $path"
|
||||
} catch {
|
||||
Write-Host "Could not clean: $path"
|
||||
}
|
||||
}
|
||||
|
||||
# Clean package caches
|
||||
if (Get-Command choco -ErrorAction SilentlyContinue) {
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
# Windows component cleanup
|
||||
try {
|
||||
dism.exe /online /Cleanup-Image /StartComponentCleanup /ResetBase
|
||||
} catch {
|
||||
Write-Host "DISM cleanup skipped"
|
||||
}
|
||||
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- name: Clean Install Visual Studio Build Tools
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Visual Studio Build Tools Setup ==="
|
||||
|
||||
# Function to completely remove VS installations
|
||||
function Remove-VSInstallation {
|
||||
param([string]$Path)
|
||||
if (Test-Path $Path) {
|
||||
Write-Host "Removing VS installation: $Path"
|
||||
Remove-Item -Path $Path -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
}
|
||||
|
||||
# Stop VS processes
|
||||
Get-Process -Name "*vs_*", "vs_installer", "VSIXInstaller", "*devenv*", "*MSBuild*" -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 5
|
||||
|
||||
# Remove all VS installations
|
||||
$vsPaths = @(
|
||||
"${env:ProgramFiles}\Microsoft Visual Studio",
|
||||
"${env:ProgramFiles(x86)}\Microsoft Visual Studio"
|
||||
)
|
||||
|
||||
foreach ($vsPath in $vsPaths) {
|
||||
if (Test-Path $vsPath) {
|
||||
Remove-VSInstallation -Path $vsPath
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "=== Installing Fresh VS Build Tools 2022 (17.8 LTSC) ==="
|
||||
|
||||
# Download VS Build Tools installer
|
||||
$installerUrl = "https://aka.ms/vs/17/release.ltsc.17.8/vs_buildtools.exe"
|
||||
$installerPath = "$env:TEMP\vs_buildtools.exe"
|
||||
|
||||
try {
|
||||
Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing -TimeoutSec 600
|
||||
Write-Host "Downloaded VS installer: $installerPath"
|
||||
} catch {
|
||||
Write-Error "Failed to download VS installer: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Install with Windows SDK 10.0.22621 (compatible with CUDA 12.6)
|
||||
$installArgs = @(
|
||||
"--quiet", "--wait", "--norestart", "--nocache",
|
||||
"--installPath", "`"C:\BuildTools`"",
|
||||
"--add", "Microsoft.VisualStudio.Workload.VCTools",
|
||||
"--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
|
||||
"--add", "Microsoft.VisualStudio.Component.VC.CMake.Project",
|
||||
"--add", "Microsoft.VisualStudio.Component.Windows10SDK.22621",
|
||||
"--add", "Microsoft.VisualStudio.Component.VC.ATL",
|
||||
"--includeRecommended"
|
||||
)
|
||||
|
||||
Write-Host "Installing VS Build Tools..."
|
||||
$process = Start-Process -FilePath $installerPath -ArgumentList $installArgs -Wait -PassThru -NoNewWindow
|
||||
|
||||
if ($process.ExitCode -ne 0) {
|
||||
Write-Error "VS Build Tools installation failed with exit code: $($process.ExitCode)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Verify installation
|
||||
$vsPath = "C:\BuildTools"
|
||||
$vcvarsPath = "$vsPath\VC\Auxiliary\Build\vcvars64.bat"
|
||||
$msvcPath = "$vsPath\VC\Tools\MSVC"
|
||||
|
||||
if (-not (Test-Path $vcvarsPath)) {
|
||||
Write-Error "VS Build Tools installation verification failed - vcvars64.bat not found"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if (-not (Test-Path $msvcPath)) {
|
||||
Write-Error "VS Build Tools installation verification failed - MSVC tools not found"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get MSVC version
|
||||
$msvcVersions = Get-ChildItem $msvcPath -Directory | Sort-Object Name -Descending
|
||||
if ($msvcVersions.Count -eq 0) {
|
||||
Write-Error "No MSVC versions found"
|
||||
exit 1
|
||||
}
|
||||
|
||||
$latestMSVC = $msvcVersions[0]
|
||||
Write-Host "SUCCESS: VS Build Tools installed successfully"
|
||||
Write-Host " Installation Path: $vsPath"
|
||||
Write-Host " MSVC Version: $($latestMSVC.Name)"
|
||||
|
||||
# Export for later steps
|
||||
echo "VS_INSTALLATION_PATH=$vsPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "MSVC_VERSION=$($latestMSVC.Name)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "MSVC_ROOT=$($latestMSVC.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
- name: Install MSYS2
|
||||
uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: MINGW64
|
||||
update: true
|
||||
install: >-
|
||||
base-devel git tar pkg-config unzip p7zip zip
|
||||
mingw-w64-x86_64-cmake mingw-w64-x86_64-make
|
||||
mingw-w64-x86_64-toolchain mingw-w64-x86_64-gcc
|
||||
|
||||
- name: Cache CUDA Installation
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda
|
||||
with:
|
||||
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6
|
||||
key: cuda-12.6-update1-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
cuda-12.6-update1-${{ runner.os }}-
|
||||
|
||||
- name: Install CUDA 12.6 Update 1
|
||||
if: steps.cache-cuda.outputs.cache-hit != 'true'
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Installing CUDA 12.6 Update 1 ==="
|
||||
|
||||
# Use network installer but with fixes for the -522190823 error
|
||||
$cudaUrl = "https://developer.download.nvidia.com/compute/cuda/12.6.3/network_installers/cuda_12.6.3_windows_network.exe"
|
||||
$cudaInstaller = "$env:TEMP\cuda_network_installer.exe"
|
||||
|
||||
Write-Host "Downloading CUDA network installer..."
|
||||
try {
|
||||
$webClient = New-Object System.Net.WebClient
|
||||
$webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||
$webClient.DownloadFile($cudaUrl, $cudaInstaller)
|
||||
$webClient.Dispose()
|
||||
|
||||
if (-not (Test-Path $cudaInstaller) -or (Get-Item $cudaInstaller).Length -lt 10MB) {
|
||||
throw "Downloaded file is missing or too small"
|
||||
}
|
||||
|
||||
Write-Host "Downloaded CUDA installer: $cudaInstaller ($(([math]::Round((Get-Item $cudaInstaller).Length / 1MB, 2))) MB)"
|
||||
} catch {
|
||||
Write-Error "Failed to download CUDA installer: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Create a temporary directory for CUDA installer logs and temp files
|
||||
$cudaTempDir = "$env:TEMP\cuda_install_temp"
|
||||
if (Test-Path $cudaTempDir) {
|
||||
Remove-Item -Path $cudaTempDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
New-Item -Path $cudaTempDir -ItemType Directory -Force | Out-Null
|
||||
|
||||
# Set environment variables to control installer behavior
|
||||
$env:CUDA_SETUP_ARGS = "--silent"
|
||||
$env:NVTOOLSEXT_INSTALL_DIR = "$cudaTempDir"
|
||||
|
||||
# Essential CUDA components only for faster installation
|
||||
$cudaComponents = @(
|
||||
"nvcc_12.6",
|
||||
"cudart_12.6",
|
||||
"cublas_12.6",
|
||||
"cublas_dev_12.6",
|
||||
"cufft_12.6",
|
||||
"cufft_dev_12.6",
|
||||
"curand_12.6",
|
||||
"curand_dev_12.6",
|
||||
"cusolver_12.6",
|
||||
"cusolver_dev_12.6",
|
||||
"cusparse_12.6",
|
||||
"cusparse_dev_12.6",
|
||||
"thrust_12.6",
|
||||
"nvrtc_12.6",
|
||||
"nvrtc_dev_12.6"
|
||||
)
|
||||
|
||||
# Build installation arguments
|
||||
$installArgs = @("-s") + $cudaComponents
|
||||
|
||||
Write-Host "Installing CUDA components: $($cudaComponents -join ', ')"
|
||||
Write-Host "Installation command: $cudaInstaller $($installArgs -join ' ')"
|
||||
|
||||
try {
|
||||
# Set working directory to temp folder to avoid permission issues
|
||||
Set-Location $cudaTempDir
|
||||
|
||||
# Run installation with timeout and proper error handling
|
||||
$installProcess = Start-Process -FilePath $cudaInstaller -ArgumentList $installArgs -Wait -PassThru -NoNewWindow -WorkingDirectory $cudaTempDir
|
||||
|
||||
Write-Host "CUDA installer completed with exit code: $($installProcess.ExitCode)"
|
||||
|
||||
# CUDA installer exit codes:
|
||||
# 0 = success
|
||||
# 1 = success with reboot required
|
||||
# 2 = already installed (newer version)
|
||||
# Other negative values = various errors
|
||||
if ($installProcess.ExitCode -notin @(0, 1, 2)) {
|
||||
throw "Installation failed with exit code: $($installProcess.ExitCode)"
|
||||
}
|
||||
|
||||
} catch {
|
||||
Write-Error "CUDA installation failed: $($_.Exception.Message)"
|
||||
|
||||
# Check for installation logs
|
||||
$logFiles = Get-ChildItem -Path $cudaTempDir -Filter "*.log" -ErrorAction SilentlyContinue
|
||||
if ($logFiles) {
|
||||
Write-Host "Installation logs found:"
|
||||
foreach ($logFile in $logFiles) {
|
||||
Write-Host "=== $($logFile.Name) ==="
|
||||
Get-Content $logFile.FullName | Select-Object -Last 20
|
||||
}
|
||||
}
|
||||
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "SUCCESS: CUDA installation completed"
|
||||
|
||||
# Verify installation
|
||||
$cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6"
|
||||
$nvccPath = "$cudaPath\bin\nvcc.exe"
|
||||
|
||||
# Wait for filesystem to settle
|
||||
Start-Sleep -Seconds 10
|
||||
|
||||
if (-not (Test-Path $nvccPath)) {
|
||||
Write-Host "Primary CUDA path not found, checking for alternative locations..."
|
||||
|
||||
# Check for any CUDA installation
|
||||
$cudaRoot = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA"
|
||||
if (Test-Path $cudaRoot) {
|
||||
Write-Host "Available CUDA installations:"
|
||||
$cudaVersions = Get-ChildItem $cudaRoot -Directory
|
||||
foreach ($version in $cudaVersions) {
|
||||
Write-Host " Found: $($version.Name)"
|
||||
$altNvccPath = "$($version.FullName)\bin\nvcc.exe"
|
||||
if (Test-Path $altNvccPath) {
|
||||
Write-Host " SUCCESS: nvcc.exe found in $($version.Name)"
|
||||
# Update our expected paths to the found version
|
||||
$cudaPath = $version.FullName
|
||||
$nvccPath = $altNvccPath
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Final check
|
||||
if (-not (Test-Path $nvccPath)) {
|
||||
Write-Error "CUDA installation verification failed - nvcc.exe not found anywhere"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Test nvcc functionality
|
||||
try {
|
||||
$nvccResult = & $nvccPath --version 2>&1
|
||||
Write-Host "SUCCESS: CUDA installation verified - nvcc version:"
|
||||
Write-Host $nvccResult
|
||||
|
||||
# Update environment variable if we found CUDA in a different location
|
||||
if ($cudaPath -ne "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6") {
|
||||
Write-Host "Updating CUDA_PATH to actual installation location: $cudaPath"
|
||||
echo "CUDA_ACTUAL_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
}
|
||||
|
||||
} catch {
|
||||
Write-Error "nvcc found but functionality test failed: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Clean up installation files
|
||||
try {
|
||||
Set-Location $env:TEMP
|
||||
Remove-Item -Path $cudaTempDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path $cudaInstaller -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleaned up installation files"
|
||||
} catch {
|
||||
Write-Host "Could not clean up some installation files (not critical)"
|
||||
}
|
||||
|
||||
Write-Host "SUCCESS: CUDA 12.6 Update 1 network installation complete"
|
||||
|
||||
- name: Setup Java
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- name: Configure Build Environment
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Configuring Build Environment ==="
|
||||
|
||||
# Setup VS environment using clean vcvars64.bat
|
||||
$vsPath = $env:VS_INSTALLATION_PATH
|
||||
$vcvarsPath = "$vsPath\VC\Auxiliary\Build\vcvars64.bat"
|
||||
|
||||
Write-Host "Initializing MSVC environment from: $vcvarsPath"
|
||||
|
||||
# Create temporary batch file to capture environment
|
||||
$tempBatch = [System.IO.Path]::GetTempFileName() + ".bat"
|
||||
$tempEnv = [System.IO.Path]::GetTempFileName() + ".txt"
|
||||
|
||||
@"
|
||||
@echo off
|
||||
call "$vcvarsPath" >nul 2>&1
|
||||
if %ERRORLEVEL% NEQ 0 exit /b 1
|
||||
set > "$tempEnv"
|
||||
"@ | Out-File -FilePath $tempBatch -Encoding ASCII
|
||||
|
||||
$result = Start-Process -FilePath "cmd.exe" -ArgumentList "/c", "`"$tempBatch`"" -Wait -PassThru -NoNewWindow
|
||||
|
||||
if ($result.ExitCode -ne 0 -or -not (Test-Path $tempEnv)) {
|
||||
Write-Error "Failed to initialize MSVC environment"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse and set environment variables
|
||||
$envCount = 0
|
||||
Get-Content $tempEnv | ForEach-Object {
|
||||
if ($_ -match '^([^=]+)=(.*)$') {
|
||||
$name = $matches[1]
|
||||
$value = $matches[2]
|
||||
# Skip problematic variables
|
||||
if ($name -notmatch '^(TEMP|TMP|RANDOM|PROMPT|PATHEXT)$') {
|
||||
try {
|
||||
[Environment]::SetEnvironmentVariable($name, $value, 'Process')
|
||||
echo "$name=$value" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
$envCount++
|
||||
} catch {
|
||||
# Silently continue on errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Set $envCount environment variables from vcvars64.bat"
|
||||
|
||||
# Clean up
|
||||
Remove-Item $tempBatch, $tempEnv -ErrorAction SilentlyContinue
|
||||
|
||||
# Setup CUDA environment
|
||||
$cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6"
|
||||
|
||||
# Check if CUDA was installed in a different location
|
||||
if ($env:CUDA_ACTUAL_PATH) {
|
||||
$cudaPath = $env:CUDA_ACTUAL_PATH
|
||||
Write-Host "Using actual CUDA installation path: $cudaPath"
|
||||
}
|
||||
|
||||
echo "CUDA_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "CUDA_HOME=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# Setup cuDNN only if needed
|
||||
if ("${{ matrix.helper }}" -eq "cudnn") {
|
||||
echo "CUDNN_ROOT_DIR=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
Write-Host "SUCCESS: cuDNN environment configured (CUDNN_ROOT_DIR=$cudaPath)"
|
||||
} else {
|
||||
Write-Host "INFO: Skipping cuDNN configuration (helper: ${{ matrix.helper }})"
|
||||
}
|
||||
|
||||
# Setup clean PATH
|
||||
$cleanPath = @(
|
||||
"C:\msys64\mingw64\bin",
|
||||
"C:\msys64\usr\bin",
|
||||
"$cudaPath\bin",
|
||||
"$cudaPath\libnvvp",
|
||||
"$env:MSVC_ROOT\bin\Hostx64\x64",
|
||||
"$env:SystemRoot\system32",
|
||||
"$env:SystemRoot"
|
||||
) -join ";"
|
||||
|
||||
echo $cleanPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
|
||||
Write-Host "SUCCESS: Build environment configured"
|
||||
Write-Host " VS Path: $vsPath"
|
||||
Write-Host " MSVC Version: $env:MSVC_VERSION"
|
||||
Write-Host " CUDA Path: $cudaPath"
|
||||
Write-Host " Helper: ${{ matrix.helper }}"
|
||||
|
||||
- name: Verify Build Tools
|
||||
shell: cmd
|
||||
run: |
|
||||
echo "=== Verifying Build Tools ==="
|
||||
|
||||
echo "Checking cl.exe..."
|
||||
where cl.exe
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo "ERROR: cl.exe not found in PATH"
|
||||
exit /b 1
|
||||
)
|
||||
cl.exe 2>&1 | findstr "Microsoft"
|
||||
|
||||
echo "Checking nvcc.exe..."
|
||||
where nvcc.exe
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo "ERROR: nvcc.exe not found in PATH"
|
||||
exit /b 1
|
||||
)
|
||||
nvcc.exe --version | findstr "release"
|
||||
|
||||
echo "Environment Check:"
|
||||
echo "CUDA_PATH: %CUDA_PATH%"
|
||||
echo "MSVC_ROOT: %MSVC_ROOT%"
|
||||
echo "Helper: ${{ matrix.helper }}"
|
||||
|
||||
if "${{ matrix.helper }}"=="cudnn" (
|
||||
echo "CUDNN_ROOT_DIR: %CUDNN_ROOT_DIR%"
|
||||
if not defined CUDNN_ROOT_DIR (
|
||||
echo "WARNING: CUDNN_ROOT_DIR not set but cuDNN helper enabled"
|
||||
)
|
||||
)
|
||||
|
||||
echo "SUCCESS: All build tools verified"
|
||||
|
||||
- name: Configure Maven Command
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Configuring Maven Command ==="
|
||||
|
||||
# Base modules
|
||||
if ("${{ github.event.inputs.libnd4jUrl }}" -ne "") {
|
||||
$modules = ":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6"
|
||||
} elseif ("${{ matrix.helper }}" -ne "") {
|
||||
$modules = ":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6,libnd4j"
|
||||
} else {
|
||||
$modules = ":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6,libnd4j,:nd4j-cuda-12.6-platform"
|
||||
}
|
||||
|
||||
# Base command
|
||||
$baseCommand = "mvn ${{ github.event.inputs.mvnFlags }} -Pcuda -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda --also-make -pl $modules"
|
||||
$baseCommand += " -Dlibnd4j.compute=`"8.6 9.0`" -Dlibnd4j.cpu.compile.skip=true"
|
||||
$baseCommand += " -Dlibnd4j.buildthreads=${{ matrix.build_threads }}"
|
||||
$baseCommand += " -Djavacpp.platform=windows-x86_64 -Dlibnd4j.platform=windows-x86_64"
|
||||
$baseCommand += " -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3"
|
||||
$baseCommand += " -Possrh deploy -DskipTests"
|
||||
|
||||
# Helper-specific extensions
|
||||
if ("${{ matrix.helper }}" -ne "") {
|
||||
$helperExt = " -Dlibnd4j.classifier=windows-x86_64-cuda-12.6-${{ matrix.helper }}"
|
||||
$helperExt += " -Dlibnd4j.extension=${{ matrix.helper }}"
|
||||
$helperExt += " -Djavacpp.platform.extension=-${{ matrix.helper }}"
|
||||
$helperExt += " -Dlibnd4j.helper=${{ matrix.helper }}"
|
||||
$libnd4jFileName = "windows-cuda-12.6-${{ matrix.helper }}"
|
||||
} else {
|
||||
$helperExt = " -Dlibnd4j.classifier=windows-x86_64-cuda-12.6"
|
||||
$libnd4jFileName = "windows-cuda-12.6"
|
||||
}
|
||||
|
||||
$fullCommand = $baseCommand + $helperExt
|
||||
|
||||
# Save command
|
||||
echo $fullCommand | Out-File -FilePath "$env:GITHUB_WORKSPACE\mvn-command.bat" -Encoding utf8
|
||||
echo "MAVEN_COMMAND=$fullCommand" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
if ("${{ github.event.inputs.libnd4jUrl }}" -ne "") {
|
||||
echo "LIBND4J_FILE_NAME=${{ github.event.inputs.libnd4jUrl }}/$libnd4jFileName" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
}
|
||||
|
||||
Write-Host "SUCCESS: Maven command configured for helper: ${{ matrix.helper }}"
|
||||
Write-Host "Command: $fullCommand"
|
||||
|
||||
- name: Run CUDA Build
|
||||
shell: cmd
|
||||
env:
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
run: |
|
||||
echo "=== Starting CUDA Build ==="
|
||||
echo "Build threads: ${{ matrix.build_threads }}"
|
||||
echo "Helper: ${{ matrix.helper }}"
|
||||
echo "Deploy to release: %PERFORM_RELEASE%"
|
||||
echo "Release version: %RELEASE_VERSION%"
|
||||
echo "Snapshot version: %SNAPSHOT_VERSION%"
|
||||
|
||||
rem Verify environment one more time
|
||||
echo "CUDA_PATH: %CUDA_PATH%"
|
||||
echo "MSVC_ROOT: %MSVC_ROOT%"
|
||||
if "${{ matrix.helper }}"=="cudnn" (
|
||||
echo "CUDNN_ROOT_DIR: %CUDNN_ROOT_DIR%"
|
||||
)
|
||||
|
||||
rem Change CUDA versions
|
||||
bash ./change-cuda-versions.sh 12.6
|
||||
|
||||
if "%PERFORM_RELEASE%"=="1" (
|
||||
echo "Running release build..."
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.6 "${{ matrix.helper }}" ""
|
||||
bash "./release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%"
|
||||
) else (
|
||||
echo "Running snapshot build..."
|
||||
if "%LIBND4J_FILE_NAME%" NEQ "" (
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.6 "${{ matrix.helper }}" ""
|
||||
)
|
||||
call "%GITHUB_WORKSPACE%\mvn-command.bat"
|
||||
)
|
||||
|
||||
echo "SUCCESS: Build completed"
|
||||
|
||||
- name: Upload Build Artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: build-logs-${{ matrix.helper || 'no-helper' }}
|
||||
path: |
|
||||
target/
|
||||
libnd4j/blasbuild/
|
||||
**/*.log
|
||||
retention-days: 7
|
||||
|
||||
- name: Setup tmate session for debugging
|
||||
if: ${{ github.event.inputs.debug_enabled == 'true' && failure() }}
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
timeout-minutes: 30
|
||||
@@ -0,0 +1,419 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
runsOn:
|
||||
description: 'OS to run on'
|
||||
required: false
|
||||
default: windows-2022
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
windows-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ onednn,"" ]
|
||||
extension: [ avx2,avx512,"" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space (Windows)
|
||||
shell: powershell
|
||||
run: |
|
||||
# Show initial disk space
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
# The system must be told to stop managing the pagefile size automatically.
|
||||
wmic computersystem set AutomaticManagedPagefile=False
|
||||
|
||||
# Find and delete any existing pagefile configuration to ensure ours is the only one.
|
||||
$currentPagefile = Get-WmiObject -Query "SELECT * FROM Win32_PageFileSetting WHERE Name='C:\\pagefile.sys'"
|
||||
if ($currentPagefile) {
|
||||
$currentPagefile.Delete()
|
||||
}
|
||||
|
||||
# Create a new pagefile with a static size of 12 GB (12288 MB).
|
||||
wmic pagefileset create name="C:\\pagefile.sys"
|
||||
wmic pagefileset where "name='C:\\pagefile.sys'" set InitialSize=12288, MaximumSize=12288
|
||||
|
||||
echo "Pagefile configured to 12 GB for the duration of this job."
|
||||
# Remove Windows Defender scan history
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed Windows Defender scan history"
|
||||
|
||||
# Clear Windows temp folders
|
||||
Remove-Item -Path "$env:SystemRoot\Temp\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows temp folders"
|
||||
|
||||
# Clear Windows Update cache safely (without stopping/starting service)
|
||||
try {
|
||||
Remove-Item -Path "$env:SystemRoot\SoftwareDistribution\Download\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows Update download cache"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Could not clear Windows Update cache. Continuing..."
|
||||
}
|
||||
|
||||
# Clean package manager caches
|
||||
if (Test-Path -Path "C:\npm\cache") {
|
||||
Remove-Item -Path "C:\npm\cache\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared NPM cache"
|
||||
}
|
||||
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Chocolatey cache"
|
||||
|
||||
# Remove Docker images if Docker is installed
|
||||
try {
|
||||
if (Get-Command "docker" -ErrorAction SilentlyContinue) {
|
||||
docker image prune -a -f
|
||||
docker container prune -f
|
||||
docker volume prune -f
|
||||
Write-Host "Pruned Docker resources"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to prune Docker resources. Continuing..."
|
||||
}
|
||||
|
||||
# Remove .NET SDK/Runtime backup folders
|
||||
if (Test-Path -Path "$env:ProgramData\Microsoft\.NET\*.backup") {
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\.NET\*.backup" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed .NET backup folders"
|
||||
}
|
||||
|
||||
# Clear Azure artifacts cache
|
||||
if (Test-Path -Path "$env:LOCALAPPDATA\Microsoft\Azure\*") {
|
||||
Remove-Item -Path "$env:LOCALAPPDATA\Microsoft\Azure\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Azure artifacts cache"
|
||||
}
|
||||
|
||||
# Optimize Windows Component Store
|
||||
try {
|
||||
Start-Process -FilePath "dism.exe" -ArgumentList "/online /Cleanup-Image /StartComponentCleanup" -NoNewWindow -Wait
|
||||
Write-Host "Optimized Windows Component Store"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to optimize Windows Component Store. Continuing..."
|
||||
}
|
||||
|
||||
# Show final disk space
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- name: Install environment
|
||||
shell: cmd
|
||||
env:
|
||||
GITHUB_EVENT_HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
|
||||
run: |
|
||||
cd /d %USERPROFILE%
|
||||
|
||||
echo Installing MSYS2
|
||||
C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm pkg-config"
|
||||
C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm base-devel git tar unzip p7zip zip autoconf autoconf-archive automake libtool make patch gnupg"
|
||||
C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm mingw-w64-x86_64-nasm mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-i686-gcc mingw-w64-x86_64-gcc-fortran mingw-w64-i686-gcc-fortran mingw-w64-x86_64-libwinpthread-git mingw-w64-i686-libwinpthread-git mingw-w64-x86_64-SDL2 mingw-w64-i686-SDL2 mingw-w64-x86_64-ragel mingw-w64-x86_64-vulkan-headers mingw-w64-i686-vulkan-headers mingw-w64-x86_64-vulkan-loader mingw-w64-i686-vulkan-loader"
|
||||
set "PATH=C:\hostedtoolcache\windows\Python\3.10.11\x64;C:\msys64\usr\bin;%PATH%"
|
||||
|
||||
C:\msys64\usr\bin\bash -lc "pacman -Q"
|
||||
|
||||
echo Installing Windows SDK 8.1
|
||||
curl -Lo sdksetup.exe https://go.microsoft.com/fwlink/p/?LinkId=323507
|
||||
sdksetup.exe /features OptionId.WindowsDesktopSoftwareDevelopmentKit OptionId.NetFxSoftwareDevelopmentKit /quiet
|
||||
|
||||
echo Removing broken stuff from WSL, MSYS2, etc
|
||||
rm "C:/msys64/usr/bin/curl.exe" "C:/msys64/mingw32/bin/curl.exe" "C:/msys64/mingw64/bin/curl.exe"
|
||||
rm "C:/WINDOWS/system32/bash.EXE" "C:/msys64/usr/bin/link.exe" "C:/msys64/usr/bin/timeout.exe" "C:/msys64/usr/bin/python.exe" "C:/msys64/usr/bin/python3.exe"
|
||||
rm "C:/ProgramData/chocolatey/bin/gfortran.exe" "C:/msys64/mingw32/bin/gfortran.exe" "C:/msys64/mingw32/bin/python.exe" "C:/msys64/mingw32/bin/python3.exe"
|
||||
rm "C:/Strawberry/c/bin/gfortran.exe" "C:/msys64/mingw64/bin/gfortran.exe" "C:/msys64/mingw64/bin/python.exe" "C:/msys64/mingw64/bin/python3.exe"
|
||||
rm "C:/msys64/mingw32/bin/clang-cl.exe" "C:/msys64/mingw64/bin/clang-cl.exe" "C:/msys64/mingw32/bin/cmake.exe" "C:/msys64/mingw64/bin/cmake.exe"
|
||||
rm "C:/Strawberry/c/lib/libz.a" "C:/Strawberry/c/lib/libzlib.a" "C:/Strawberry/c/lib/libzdll.a" "C:/Strawberry/c/bin/cmake.exe"
|
||||
|
||||
curl -LO https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz || curl -LO https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
|
||||
bash -c "tar -xzf apache-maven-3.6.3-bin.tar.gz -C 'C:/Program Files/'"
|
||||
|
||||
python -m pip install gdown || python -m pip install gdown
|
||||
|
||||
echo Installing ccache
|
||||
curl -LO https://github.com/ccache/ccache/releases/download/v4.6/ccache-4.6-windows-64.zip
|
||||
unzip -j ccache-4.6-windows-64.zip -d C:/msys64/usr/bin/
|
||||
mkdir ccache
|
||||
echo max_size = 3.0G > ccache\ccache.conf
|
||||
echo hash_dir = false >> ccache\ccache.conf
|
||||
echo sloppiness = file_macro,include_file_ctime,include_file_mtime,pch_defines,time_macros >> ccache\ccache.conf
|
||||
|
||||
echo Installing an older less buggy version of GCC
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-ada-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-objc-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-libs-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-fortran-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-libgfortran-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-binutils-2.42-2-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-crt-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-headers-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-libmangle-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-libwinpthread-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-tools-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-winpthreads-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-winstorecompat-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-vulkan-headers-1.3.280.0-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-vulkan-loader-1.3.280.0-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-SDL2-2.30.12-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-python-3.11.9-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-ada-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-objc-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-libs-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-fortran-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-libgfortran-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-binutils-2.42-2-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-crt-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-headers-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libmangle-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libwinpthread-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-tools-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-winpthreads-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-winstorecompat-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-vulkan-headers-1.3.280.0-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-vulkan-loader-1.3.280.0-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-SDL2-2.30.12-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-python-3.11.9-1-any.pkg.tar.zst
|
||||
bash -c "pacman -U --noconfirm *.pkg.tar.zst"
|
||||
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: powershell
|
||||
run: |
|
||||
if ( "${{ matrix.libnd4j_file_download }}" -ne "" ) {
|
||||
$modules=" :nd4j-native-preset,:nd4j-native"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$modules=":nd4j-native-preset,:nd4j-native,libnd4j"
|
||||
} elseif ( "${{ matrix.extension }}" -ne "" ) {
|
||||
$modules=":nd4j-native-preset,:nd4j-native,libnd4j"
|
||||
} else {
|
||||
$modules=":nd4j-native-preset,:nd4j-native,libnd4j,:nd4j-native-platform"
|
||||
}
|
||||
|
||||
$command="mvn ${{ matrix.mvn_ext }} -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.platform=windows-x86_64 deploy -DskipTests --also-make"
|
||||
if ( "${{ matrix.helper }}" -ne "" -And "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.platform=windows-x86_64 deploy -DskipTests"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.platform=windows-x86_64 deploy -DskipTests"
|
||||
} elseif ( "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }}"
|
||||
} else {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64"
|
||||
}
|
||||
|
||||
if ( "${{ matrix.libnd4j_file_download }}" -ne "") {
|
||||
echo "Adding libnd4j download"
|
||||
$libnd4j_url_to_write = -join("LIBND4J_FILE_NAME=","$(${{ matrix.libnd4j_file_download }}/$libnd4j_download_file_url)");
|
||||
echo $libnd4j_url_to_write | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
|
||||
}
|
||||
|
||||
$command2 = -join("$($command)","$($mvn_ext)");
|
||||
$to_write = -join("COMMAND=","$($command2)");
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to $($command2)"
|
||||
echo $command2 | Out-File -FilePath "$env:GITHUB_WORKSPACE/mvn-command.bat" -Encoding utf8 -Append
|
||||
echo $to_write | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: konduitai/setup-java@main
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
|
||||
- name: Setup windows path
|
||||
shell: powershell
|
||||
run: echo "C:\msys64\mingw64\bin;C:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: powershell
|
||||
run: |
|
||||
mkdir "%GITHUB_WORKSPACE%/openblas_home"
|
||||
cd "%GITHUB_WORKSPACE%/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/openblas-0.3.28-1.5.11-windows-x86_64.jar
|
||||
unzip openblas-0.3.28-1.5.11-windows-x86_64.jar
|
||||
cd ..
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/windows-x86_64/" | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
|
||||
if: ${{ matrix.libnd4j_file_download != '' }}
|
||||
|
||||
- name: Run windows cpu build
|
||||
shell: cmd
|
||||
run: |
|
||||
call "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64
|
||||
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64
|
||||
set MSYSTEM=MINGW64
|
||||
|
||||
set "CCACHE_DIR=%USERPROFILE%\ccache"
|
||||
set "PATH=C:\hostedtoolcache\windows\Python\3.10.11\x64;C:\msys64\%MSYSTEM%\bin;C:\msys64\usr\bin;%ProgramFiles%\apache-maven-3.6.3\bin;%PATH%"
|
||||
|
||||
where bash
|
||||
where curl
|
||||
where git
|
||||
where cl
|
||||
where gcc
|
||||
where cmake
|
||||
where mvn
|
||||
where python
|
||||
where python3
|
||||
where ccache
|
||||
|
||||
bash --version
|
||||
git --version
|
||||
cl
|
||||
gcc --version
|
||||
cmake --version
|
||||
call mvn -version
|
||||
python --version
|
||||
ccache --version -sv
|
||||
df -h
|
||||
wmic pagefile list /format:list
|
||||
|
||||
set MAKEJ=%NUMBER_OF_PROCESSORS%
|
||||
echo Fetching %GITHUB_REPOSITORY%@%GITHUB_SHA%
|
||||
git init
|
||||
git fetch --depth 1 https://github.com/%GITHUB_REPOSITORY% %GITHUB_SHA%
|
||||
git checkout %GITHUB_SHA%
|
||||
git submodule update --init --recursive
|
||||
git submodule foreach --recursive "git reset --hard"
|
||||
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
if "%PERFORM_RELEASE%"=="1" (
|
||||
echo "Running release"
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows x86_64 "${{ matrix.helper }}" "${{ matrix.extension }}"
|
||||
bash "%GITHUB_WORKSPACE%/release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%" "%COMMAND%"
|
||||
) else (
|
||||
if "%PERFORM_RELEASE%"==1 (
|
||||
echo "Running release"
|
||||
bash "%GITHUB_WORKSPACE%/release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%" "%COMMAND%"
|
||||
) else (
|
||||
echo "Running snapshots"
|
||||
call "%GITHUB_WORKSPACE%\mvn-command.bat"
|
||||
)
|
||||
)
|
||||
|
||||
ccache --version -sv
|
||||
df -h
|
||||
wmic pagefile list /format:list
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
|
||||
- name: Clean up
|
||||
shell: cmd
|
||||
run: |
|
||||
cd /d %USERPROFILE%
|
||||
set "PATH=C:\hostedtoolcache\windows\Python\3.10.11\x64;C:\msys64\usr\bin;%PATH%"
|
||||
bash -c "rm -Rf $(find .m2/repository/ -name '*SNAPSHOT*')"
|
||||
@@ -0,0 +1,357 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M2
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
jobs:
|
||||
#Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
|
||||
linux-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [onednn,""]
|
||||
extension: [avx2,avx512,""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
|
||||
- runs_on: ubuntu-22.04
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
|
||||
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
- name: Cache cmake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake
|
||||
restore-keys: ${{ runner.os }}-cmake
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf
|
||||
restore-keys: ${{ runner.os }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
- uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake.outputs.cache-hit != 'true'
|
||||
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.19-1.5.7/openblas-0.3.19-1.5.7-linux-x86_64.jar
|
||||
unzip openblas-0.3.19-1.5.7-linux-x86_64.jar
|
||||
cd ..
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64" >> "$GITHUB_ENV"
|
||||
cp ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64/libopenblas.so.0 ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64/libopenblas.so
|
||||
if: ${{ github.event.inputs.libnd4jUrl != '' }}
|
||||
- name: Download dl4j-test-resources
|
||||
uses: ./.github/actions/download-dl4j-test-resources-linux
|
||||
- name: Build on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
run: |
|
||||
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
export LIBGOMP_PATH=/usr/lib/gcc/x86_64-linux-gnu/5.5.0/libgomp.so
|
||||
if [ -z "${EXTENSION}" ] || [ -n "${EXTENSION}" ]; then
|
||||
export LIBGOMP_PATH=/usr/lib/gcc/x86_64-linux-gnu/7.5.0/libgomp.so
|
||||
echo "Extensions specified. This needs a newer version of gcc."
|
||||
sudo apt-get install gcc-7 g++-7
|
||||
echo "Using newer version of libgomp."
|
||||
ls /usr/bin | grep gcc
|
||||
ls /usr/bin | grep g++
|
||||
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
|
||||
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
|
||||
gcc --version
|
||||
fi
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dplatform.classifier=linux-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext="-Dplatform.classifier=linux-x86_64-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dplatform.classifier=linux-x86_64-${{matrix.extension}}"
|
||||
else
|
||||
mvn_ext=" -Dplatform.classifier=linux-x86_64"
|
||||
fi
|
||||
cd ${GITHUB_WORKSPACE}/platform-tests && mvn ${mvn_ext} -Djavacpp.platform=linux-x86_64 clean test
|
||||
|
||||
|
||||
windows-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ onednn,"" ]
|
||||
extension: [ avx2,avx512,"" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
|
||||
|
||||
runs-on: windows-2019
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space (Windows)
|
||||
shell: powershell
|
||||
run: |
|
||||
# Show initial disk space
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
# Remove Windows Defender scan history
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed Windows Defender scan history"
|
||||
|
||||
# Clear Windows temp folders
|
||||
Remove-Item -Path "$env:SystemRoot\Temp\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows temp folders"
|
||||
|
||||
# Clear Windows Update cache safely (without stopping/starting service)
|
||||
try {
|
||||
Remove-Item -Path "$env:SystemRoot\SoftwareDistribution\Download\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows Update download cache"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Could not clear Windows Update cache. Continuing..."
|
||||
}
|
||||
|
||||
# Clean package manager caches
|
||||
if (Test-Path -Path "C:\npm\cache") {
|
||||
Remove-Item -Path "C:\npm\cache\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared NPM cache"
|
||||
}
|
||||
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Chocolatey cache"
|
||||
|
||||
# Remove Docker images if Docker is installed
|
||||
try {
|
||||
if (Get-Command "docker" -ErrorAction SilentlyContinue) {
|
||||
docker image prune -a -f
|
||||
docker container prune -f
|
||||
docker volume prune -f
|
||||
Write-Host "Pruned Docker resources"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to prune Docker resources. Continuing..."
|
||||
}
|
||||
|
||||
# Remove .NET SDK/Runtime backup folders
|
||||
if (Test-Path -Path "$env:ProgramData\Microsoft\.NET\*.backup") {
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\.NET\*.backup" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed .NET backup folders"
|
||||
}
|
||||
|
||||
# Clear Azure artifacts cache
|
||||
if (Test-Path -Path "$env:LOCALAPPDATA\Microsoft\Azure\*") {
|
||||
Remove-Item -Path "$env:LOCALAPPDATA\Microsoft\Azure\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Azure artifacts cache"
|
||||
}
|
||||
|
||||
# Optimize Windows Component Store
|
||||
try {
|
||||
Start-Process -FilePath "dism.exe" -ArgumentList "/online /Cleanup-Image /StartComponentCleanup" -NoNewWindow -Wait
|
||||
Write-Host "Optimized Windows Component Store"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to optimize Windows Component Store. Continuing..."
|
||||
}
|
||||
|
||||
# Show final disk space
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: konduitai/setup-java@main
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
|
||||
|
||||
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: MINGW64
|
||||
update: true
|
||||
install: base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake patch mingw-w64-x86_64-gnupg mingw-w64-x86_64-make --noconfirm mingw-w64-x86_64-cmake mingw-w64-x86_64-nasm mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-fortran mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-x86_64-ragel
|
||||
- name: Setup windows path
|
||||
shell: powershell
|
||||
run: echo "C:\msys64\mingw64\bin;C:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: powershell
|
||||
run: |
|
||||
mkdir "%GITHUB_WORKSPACE%/openblas_home"
|
||||
cd "%GITHUB_WORKSPACE%/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.19-1.5.7/openblas-0.3.19-1.5.7-windows-x86_64.jar
|
||||
unzip openblas-0.3.19-1.5.7-windows-x86_64.jar
|
||||
cd ..
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/windows-x86_64/" | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
|
||||
if: ${{ matrix.libnd4j_file_download != '' }}
|
||||
- name: Download dl4j-test-resources
|
||||
uses: ./.github/actions/download-dl4j-test-resources-windows
|
||||
- name: Run windows cpu build
|
||||
shell: cmd
|
||||
run: |
|
||||
if ( "${{ matrix.helper }}" -ne "" -And "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -platform.classifier=windows-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$mvn_ext=" -Dplatform.classifier=windows-x86_64-${{ matrix.helper }}"
|
||||
} elseif ( "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -Dplatform.classifier=windows-x86_64-${{matrix.extension}} "
|
||||
} else {
|
||||
$mvn_ext=" -Dplatform.classifier=windows-x86_64"
|
||||
}
|
||||
cd "%GITHUB_WORKSPACE%\platform-tests"
|
||||
mvn "%mvn_ext%" -Djavacpp.platform=windows-x86_64 clean test
|
||||
|
||||
env:
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
|
||||
|
||||
mac-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ onednn,"" ]
|
||||
extension: [ avx2,avx512,"" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
runs-on: macos-10.15
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: 'central'
|
||||
server-username: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
server-password: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
|
||||
|
||||
|
||||
- name: Update path for gnu sed
|
||||
shell: bash
|
||||
run: |
|
||||
brew install gpg1 gnu-sed unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python
|
||||
echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
|
||||
- name: Download dl4j-test-resources
|
||||
uses: ./.github/actions/download-dl4j-test-resources-linux
|
||||
- name: Build and install
|
||||
shell: bash
|
||||
env:
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
|
||||
|
||||
run: |
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dplatform.classifier=macosx-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext="-Dplatform.classifier=macosx-x86_64-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dplatform.classifier=macosx-x86_64-${{matrix.extension}}"
|
||||
else
|
||||
mvn_ext=" -Dlibnd4j.classifier=macosx-x86_64"
|
||||
fi
|
||||
cd ${GITHUB_WORKSPACE}/platform-tests && mvn -Djavacpp.platform=macosx-x86_64 ${mvn_ext} clean test
|
||||
@@ -0,0 +1,49 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 1
|
||||
|
||||
runsOn:
|
||||
description: 'The operating system to run on, defaults to self hosted'
|
||||
required: false
|
||||
default: self-hosted
|
||||
|
||||
jobs:
|
||||
linux-x86_64:
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- uses: ./.github/actions/download-dl4j-test-resources-linux
|
||||
- name: Run cpu tests
|
||||
shell: bash
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
run: |
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
export OMP_NUM_THREADS=1
|
||||
mkdir -p ${GITHUB_WORKSPACE}/resources
|
||||
mkdir -p ${GITHUB_WORKSPACE}/cache
|
||||
mvn -Djunit.jupiter.execution.parallel.enabled=true -Djunit.jupiter.execution.parallel.mode.default=concurrent -Djunit.jupiter.execution.parallel.config.strategy=fixed -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -DexcludedGroups="long-running-tests, large-resources, distributed-systems" -DskipTestResourceEnforcement=true clean test --fail-never
|
||||
mvn -Djunit.jupiter.execution.parallel.enabled=true -Djunit.jupiter.execution.parallel.mode.default=concurrent -Djunit.jupiter.execution.parallel.config.strategy=fixed -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -Dgroups="long-running-tests, large-resources, distributed-systems" -Dtest.offheap.size=14g -Dtest.heap.size=6g -Dsurefire.parallel.forcedTimeout=500 -Dsurefire.parallel.timeout=500 -Dsurefire.timeout=200 -Dsurefire.exitTimeout=500 test --fail-never -rf :nd4j
|
||||
@@ -0,0 +1,45 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 1
|
||||
|
||||
runsOn:
|
||||
description: 'The operating system to run on, defaults to self-hosted'
|
||||
required: false
|
||||
default: self-hosted
|
||||
|
||||
jobs:
|
||||
linux-x86_64:
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- uses: ./.github/actions/download-dl4j-test-resources-linux
|
||||
- name: Run cpu tests
|
||||
shell: bash
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
run: |
|
||||
mvn --version
|
||||
cmake --version
|
||||
export OMP_NUM_THREADS=2
|
||||
mvn -DskipTestResourceEnforcement=true -Dlibnd4j.build=debug -Dlibnd4j.sanitize=ON clean test
|
||||
@@ -0,0 +1,100 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
cache:
|
||||
runs-on: windows-2019
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space (Windows)
|
||||
shell: powershell
|
||||
run: |
|
||||
# Show initial disk space
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
# Remove Windows Defender scan history
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed Windows Defender scan history"
|
||||
|
||||
# Clear Windows temp folders
|
||||
Remove-Item -Path "$env:SystemRoot\Temp\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows temp folders"
|
||||
|
||||
# Clear Windows Update cache safely (without stopping/starting service)
|
||||
try {
|
||||
Remove-Item -Path "$env:SystemRoot\SoftwareDistribution\Download\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows Update download cache"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Could not clear Windows Update cache. Continuing..."
|
||||
}
|
||||
|
||||
# Clean package manager caches
|
||||
if (Test-Path -Path "C:\npm\cache") {
|
||||
Remove-Item -Path "C:\npm\cache\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared NPM cache"
|
||||
}
|
||||
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Chocolatey cache"
|
||||
|
||||
# Remove Docker images if Docker is installed
|
||||
try {
|
||||
if (Get-Command "docker" -ErrorAction SilentlyContinue) {
|
||||
docker image prune -a -f
|
||||
docker container prune -f
|
||||
docker volume prune -f
|
||||
Write-Host "Pruned Docker resources"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to prune Docker resources. Continuing..."
|
||||
}
|
||||
|
||||
# Remove .NET SDK/Runtime backup folders
|
||||
if (Test-Path -Path "$env:ProgramData\Microsoft\.NET\*.backup") {
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\.NET\*.backup" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed .NET backup folders"
|
||||
}
|
||||
|
||||
# Clear Azure artifacts cache
|
||||
if (Test-Path -Path "$env:LOCALAPPDATA\Microsoft\Azure\*") {
|
||||
Remove-Item -Path "$env:LOCALAPPDATA\Microsoft\Azure\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Azure artifacts cache"
|
||||
}
|
||||
|
||||
# Optimize Windows Component Store
|
||||
try {
|
||||
Start-Process -FilePath "dism.exe" -ArgumentList "/online /Cleanup-Image /StartComponentCleanup" -NoNewWindow -Wait
|
||||
Write-Host "Optimized Windows Component Store"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to optimize Windows Component Store. Continuing..."
|
||||
}
|
||||
|
||||
# Show final disk space
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master
|
||||
env:
|
||||
cuda: 11.6.0
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master
|
||||
env:
|
||||
cuda: 11.4.1
|
||||
- name: Cache cuda install windows cuda 11.4
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-114
|
||||
with:
|
||||
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4
|
||||
key: windows-2019-cuda-11.4
|
||||
restore-keys: windows-2019-cuda-11.4
|
||||
- name: Cache cuda install windows cuda 11.6
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-116
|
||||
with:
|
||||
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6
|
||||
key: windows-2019-cuda-11.6
|
||||
restore-keys: windows-2019-cuda-11.6
|
||||
@@ -0,0 +1,32 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
cache:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master
|
||||
env:
|
||||
cuda: 11.6.0
|
||||
GCC: 7
|
||||
- name: Cache cuda 11.6 install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-116
|
||||
with:
|
||||
path: /usr/local/cuda-11.6
|
||||
key: ubuntu-22.04-cuda-11.6
|
||||
restore-keys: ubuntu-22.04-cuda-11.6
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master
|
||||
env:
|
||||
cuda: 11.4
|
||||
GCC: 7
|
||||
- name: Cache cuda 11.4 install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-114
|
||||
with:
|
||||
path: /usr/local/cuda-11.4
|
||||
key: ubuntu-22.04-cuda-11.4
|
||||
restore-keys: ubuntu-22.04-cuda-11.4
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user