chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,26 @@
|
||||
#### Issue Description
|
||||
|
||||
Please describe our issue, along with:
|
||||
- expected behavior
|
||||
- encountered behavior
|
||||
|
||||
#### Version Information
|
||||
|
||||
Please indicate relevant versions, including, if relevant:
|
||||
|
||||
* Deeplearning4j version
|
||||
* Platform information (OS, etc)
|
||||
* CUDA version, if used
|
||||
* NVIDIA driver version, if in use
|
||||
|
||||
#### Additional Information
|
||||
|
||||
Where applicable, please also provide:
|
||||
|
||||
* Full log or exception stack trace (ideally in a Gist: gist.github.com)
|
||||
* pom.xml file or similar (also in a Gist)
|
||||
|
||||
#### Contributing
|
||||
|
||||
If you'd like to help us fix the issue by contributing some code, but would
|
||||
like guidance or help in doing so, please mention it!
|
||||
@@ -0,0 +1,16 @@
|
||||
## What changes were proposed in this pull request?
|
||||
|
||||
(Please fill in changes proposed in this fix)
|
||||
|
||||
## How was this patch tested?
|
||||
|
||||
(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
|
||||
|
||||
## Quick checklist
|
||||
|
||||
The following checklist helps ensure your PR is complete:
|
||||
|
||||
- [ ] Eclipse Contributor Agreement signed, and signed commits - see [IP Requirements](https://deeplearning4j.konduit.ai/multi-project/how-to-guides/contribute/eclipse-contributors) page for details
|
||||
- [ ] Reviewed the [Contributing Guidelines](https://github.com/eclipse/deeplearning4j/blob/master/CONTRIBUTING.md) and followed the steps within.
|
||||
- [ ] Created tests for any significant new code additions.
|
||||
- [ ] Relevant tests for your changes are passing.
|
||||
@@ -0,0 +1,5 @@
|
||||
FROM centos:7
|
||||
COPY ./build.sh /build.sh
|
||||
RUN chmod +x /build.sh
|
||||
ENV JAVA_HOME=/opt/zulu11.58.15-ca-jdk11.0.16-linux_x64
|
||||
ENTRYPOINT '/build.sh'
|
||||
@@ -0,0 +1,10 @@
|
||||
## Centos build for deeplearning4j
|
||||
Implements a centos 7 based build based on the following docs:
|
||||
https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainervolumes
|
||||
https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action
|
||||
https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions
|
||||
|
||||
This is for a -compat build for older glibcs mainly running on centos based systems.
|
||||
The action relies on volumes to just run an install workload followed by a deployment
|
||||
using the host's gpg signature infrastructure very similar to the existing linux-x86_64 builds
|
||||
based on ubuntu 16.04
|
||||
@@ -0,0 +1,6 @@
|
||||
name: Build deeplearning4j on centos 7
|
||||
runs:
|
||||
using: 'docker'
|
||||
image: 'Dockerfile'
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/bin/bash
|
||||
env
|
||||
## Based on the javacpp presets github actions centos 7 build at https://github.com/bytedeco/javacpp-presets/
|
||||
SCL_ENABLE="devtoolset-7"
|
||||
yum -y update && yum -y install wget unzip centos-release-scl-rh epel-release
|
||||
echo "Downloading java from azul"
|
||||
cd /opt && wget https://cdn.azul.com/zulu/bin/zulu11.58.15-ca-jdk11.0.16-linux_x64.zip
|
||||
echo "Downloaded azul java"
|
||||
ls /opt
|
||||
cd /opt && unzip zulu11.58.15-ca-jdk11.0.16-linux_x64.zip
|
||||
#zulu11.58.15-ca-jdk11.0.16-linux_x64
|
||||
yum -y install $SCL_ENABLE rh-java-common-ant boost-devel ccache clang gcc-c++ gcc-gfortran ant python python36-devel python36-pip swig file which wget unzip tar bzip2 gzip xz patch autoconf-archive automake make libtool bison flex perl nasm alsa-lib-devel freeglut-devel gtk2-devel libusb-devel libusb1-devel curl-devel expat-devel gettext-devel openssl-devel bzip2-devel zlib-devel SDL-devel libva-devel libxkbcommon-devel libxkbcommon-x11-devel xcb-util* fontconfig-devel libffi-devel ragel ocl-icd-devel GeoIP-devel pcre-devel ssdeep-devel yajl-devel
|
||||
sed -i 's/_mm512_abs_pd (__m512 __A)/_mm512_abs_pd (__m512d __A)/g' /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/include/avx512fintrin.h
|
||||
source scl_source enable $SCL_ENABLE || true
|
||||
curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.6/cmake-3.16.6-Linux-x86_64.tar.gz
|
||||
curl -LO https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
|
||||
curl -LO https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.18.3.tar.gz
|
||||
tar -xzf cmake-3.16.6-Linux-x86_64.tar.gz -C /opt/
|
||||
mv /opt/cmake-3.16.6-Linux-x86_64 /opt/cmake
|
||||
tar -xzf apache-maven-3.6.3-bin.tar.gz -C /opt/
|
||||
tar -xzf git-2.18.3.tar.gz
|
||||
pushd git-2.18.3; make -j2 prefix=/usr/local/; make -j2 prefix=/usr/local/ install; popd
|
||||
ln -sf /usr/bin/python3.6 /usr/bin/python3
|
||||
ln -sf /opt/cmake-3.16.6-Linux-x86_64/bin/* /usr/bin/
|
||||
ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn
|
||||
curl -fsSL https://github.com/google/protobuf/releases/download/v3.8.0/protobuf-cpp-3.8.0.tar.gz \
|
||||
| tar xz && \
|
||||
cd protobuf-3.8.0 && \
|
||||
./configure --prefix=/opt/protobuf && \
|
||||
make -j2 && \
|
||||
make install && \
|
||||
cd .. && \
|
||||
rm -rf protobuf-3.8.0
|
||||
echo "/opt/protobuf/bin" >> $GITHUB_PATH
|
||||
# need to hardcode due to conflicting java home being set
|
||||
export JAVA_HOME=/opt/zulu11.58.15-ca-jdk11.0.16-linux_x64
|
||||
echo "${JAVA_HOME}/bin" >> $GITHUB_PATH
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$JAVA_HOME/bin:$PATH
|
||||
echo "JAVA_HOME ${JAVA_HOME}"
|
||||
java -version
|
||||
which javac
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
pwd
|
||||
# The volume directory for the workspace
|
||||
cd "/github/workspace/"
|
||||
bash ./bootstrap-libnd4j-from-url.sh
|
||||
|
||||
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$OPENBLAS_PATH"
|
||||
echo "Running INSTALL COMMAND ${INSTALL_COMMAND}"
|
||||
eval "${INSTALL_COMMAND}"
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
name: Download dl4j test resources
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: bash
|
||||
run: |
|
||||
wget https://github.com/KonduitAI/dl4j-test-resources/archive/master.zip && unzip master.zip
|
||||
cd dl4j-test-resources-master
|
||||
mvn clean install -DskipTests
|
||||
echo "Extracted test resources"
|
||||
@@ -0,0 +1,12 @@
|
||||
name: Download dl4j test resources
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: cmd
|
||||
run: |
|
||||
set "PATH=C:\msys64\usr\bin;%PATH%"
|
||||
wget https://github.com/KonduitAI/dl4j-test-resources/archive/master.zip && unzip master.zip
|
||||
cd dl4j-test-resources-master
|
||||
mvn clean install -DskipTests
|
||||
echo "Extracted test resources"
|
||||
@@ -0,0 +1,12 @@
|
||||
name: Download dl4j test resources
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt install git gcc-8-aarch64-linux-gnu g++-8-aarch64-linux-gnu libc6-armel-cross libc6-dev-armel-cross binutils-arm-linux-gnueabi libncurses5-dev build-essential bison flex libssl-dev bc \
|
||||
gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf crossbuild-essential-arm64
|
||||
mkdir -p /opt/raspberrypi && \
|
||||
cd /opt/raspberrypi && \
|
||||
git clone git://github.com/raspberrypi/tools.git
|
||||
@@ -0,0 +1,11 @@
|
||||
name: Install protobuf linux
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install protobuf linux
|
||||
shell: bash
|
||||
run: |
|
||||
apt-get -yq update && apt-get install -y build-essential unzip libssl-dev
|
||||
curl -fsSL http://cmake.org/files/v3.28.3/cmake-3.28.3.tar.gz | tar xz && cd cmake-3.28.3
|
||||
./configure --prefix=/opt/cmake && make -j2 && make install && cd .. && rm -r cmake-3.28.3
|
||||
echo "/opt/cmake/bin" >> $GITHUB_PATH
|
||||
@@ -0,0 +1,16 @@
|
||||
name: Install protobuf linux
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install protobuf linux
|
||||
shell: bash
|
||||
run: |
|
||||
curl -fsSL https://github.com/google/protobuf/releases/download/v3.8.0/protobuf-cpp-3.8.0.tar.gz \
|
||||
| tar xz && \
|
||||
cd protobuf-3.8.0 && \
|
||||
./configure --prefix=/opt/protobuf && \
|
||||
make -j2 && \
|
||||
sudo make install && \
|
||||
cd .. && \
|
||||
rm -rf protobuf-3.8.0
|
||||
echo "/opt/protobuf/bin" >> $GITHUB_PATH
|
||||
@@ -0,0 +1,10 @@
|
||||
name: Setup for msys2
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: cmd
|
||||
run: |
|
||||
msys2do pacman -S --needed --noconfirm base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake patch
|
||||
msys2do pacman -S --needed mingw-w64-x86_64-make --noconfirm mingw-w64-x86_64-gnupg mingw-w64-x86_64-cmake mingw-w64-x86_64-nasm mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-fortran mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-x86_64-ragel mingw-w64-x86_64-sed
|
||||
echo '--yes --always-trust' >> ~/.gnupg/gpg.conf
|
||||
@@ -0,0 +1,9 @@
|
||||
name: Publish to github packages
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Publish to GitHub Packages
|
||||
run: mvn -Pgithub --batch-mode deploy
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Remove un needed packages for linux
|
||||
description:
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt remove -y mongodb-org *google-chrome* firefox apache2 kubectl esl-erlang hhvm nginx libpq-dev postgresql postgresql-client ruby-full powershell r-base mono-complete nuget
|
||||
sudo rm -rf /usr/share/java/selnium*
|
||||
sudo rm -rf /usr/local/share/phantomjs*
|
||||
sudo rm -rf /usr/local/share/gecko_driver*
|
||||
sudo rm -rf /usr/local/lib/lein*
|
||||
sudo rm -rf /usr/share/miniconda*
|
||||
sudo apt remove -y apache2-bin* azure-cli dotnet-* ghc-* google-cloud-sdk* libboost-* libmono-* libobjc-* moby-* mono-* mysql* postgresql* r-* ruby* sqlite3* swig*
|
||||
sudo apt-get autoclean
|
||||
sudo rm -rf "/usr/local/share/boost"
|
||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
||||
sudo swapoff -a
|
||||
sudo rm -f /swapfile
|
||||
sudo apt clean
|
||||
df -h
|
||||
@@ -0,0 +1,39 @@
|
||||
name: Set OS install version
|
||||
description:
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Initial install
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -f /etc/os-release ]; then
|
||||
. /etc/os-release
|
||||
OS="$NAME"
|
||||
VER="$VERSION_ID"
|
||||
elif [ type lsb_release >/dev/null 2>&1 ]; then
|
||||
# linuxbase.org
|
||||
OS="$(lsb_release -si)"
|
||||
VER="$(lsb_release -sr)"
|
||||
elif [ -f /etc/lsb-release ]; then
|
||||
/etc/lsb-release
|
||||
OS="$DISTRIB_ID"
|
||||
VER="$DISTRIB_RELEASE"
|
||||
elif [ -f /etc/debian_version ]; then
|
||||
OS="Debian"
|
||||
VER="$(cat /etc/debian_version)"
|
||||
elif [ -f /etc/SuSe-release ]; then
|
||||
OS="SUSE"
|
||||
VER="13.1"
|
||||
elif [ -f /etc/redhat-release ]; then
|
||||
# Older Red Hat, CentOS, etc.
|
||||
OS="Centos"
|
||||
VER="6"
|
||||
else
|
||||
# Fall back to uname, e.g. "Linux <version>", also works for BSD, etc.
|
||||
OS="$(uname -s)"
|
||||
VER="$(uname -r)"
|
||||
fi
|
||||
|
||||
echo "OS=$OS" >> "$GITHUB_ENV"
|
||||
echo "VER=$VER" >> "$GITHUB_ENV"
|
||||
echo "OS is $OS and VERSION is $VER"
|
||||
@@ -0,0 +1,13 @@
|
||||
name: Update dependencies linux
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Update dependencies linux
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get install build-essential make zlib1g-dev wget
|
||||
sudo apt-get install pinentry-curses
|
||||
sudo apt-get install ca-certificates libgomp1
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: '2'
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: '0'
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: '1.0.0-M3'
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: '1.0.0-SNAPSHOT'
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: 'central'
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set.'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: 'ubuntu-22.04-arm'
|
||||
|
||||
debug_enabled:
|
||||
description: 'Enable debug session'
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
jobs:
|
||||
android-arm64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [armcompute, ""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
timeout-minutes: 720
|
||||
steps:
|
||||
- name: Clean workspace
|
||||
uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space (Initial)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf-intel-v1
|
||||
|
||||
- name: Install protobuf
|
||||
uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Cache CMake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake-v1
|
||||
|
||||
- name: Install CMake
|
||||
uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Install system dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y \
|
||||
ninja-build \
|
||||
build-essential \
|
||||
clang \
|
||||
lld \
|
||||
llvm-dev \
|
||||
gcc-aarch64-linux-gnu \
|
||||
g++-aarch64-linux-gnu \
|
||||
libc6-dev-arm64-cross
|
||||
|
||||
- name: Set up Java for publishing
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
|
||||
|
||||
|
||||
- name: Download and Setup Android NDK
|
||||
shell: bash
|
||||
run: |
|
||||
NDK_HOME="${GITHUB_WORKSPACE}/android-ndk-r27d"
|
||||
if [ ! -d "${NDK_HOME}" ]; then
|
||||
echo "Downloading Android NDK r27d for Linux x86_64..."
|
||||
cd "${GITHUB_WORKSPACE}"
|
||||
wget --progress=dot:giga "https://dl.google.com/android/repository/android-ndk-r27d-linux.zip"
|
||||
unzip -q "android-ndk-r27d-linux.zip"
|
||||
rm "android-ndk-r27d-linux.zip"
|
||||
echo "NDK extracted to: ${NDK_HOME}"
|
||||
ls -la "${NDK_HOME}" | head -10
|
||||
else
|
||||
echo "Using cached Android NDK."
|
||||
fi
|
||||
echo "ANDROID_NDK_ROOT=${NDK_HOME}" >> $GITHUB_ENV
|
||||
echo "ANDROID_NDK=${NDK_HOME}" >> $GITHUB_ENV
|
||||
|
||||
|
||||
|
||||
- name: Setup OpenBLAS
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Setting up OpenBLAS for Android ARM64..."
|
||||
OPENBLAS_JAR="openblas-0.3.28-1.5.11-android-arm64.jar"
|
||||
OPENBLAS_HOME="${GITHUB_WORKSPACE}/openblas_home"
|
||||
OPENBLAS_DIR="${OPENBLAS_HOME}/lib/arm64-v8a"
|
||||
|
||||
mkdir -p "${OPENBLAS_HOME}"
|
||||
cd "${OPENBLAS_HOME}"
|
||||
|
||||
wget -q "https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/${OPENBLAS_JAR}"
|
||||
unzip -q "${OPENBLAS_JAR}"
|
||||
|
||||
if [ -f "${OPENBLAS_DIR}/libopenblas.so" ] && [ ! -f "${OPENBLAS_DIR}/libopenblas.so.0" ]; then
|
||||
ln -s "${OPENBLAS_DIR}/libopenblas.so" "${OPENBLAS_DIR}/libopenblas.so.0"
|
||||
fi
|
||||
|
||||
echo "OPENBLAS_PATH=${OPENBLAS_DIR}" >> "$GITHUB_ENV"
|
||||
echo "OPENBLAS_HOME=${OPENBLAS_HOME}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Setup ARM Compute Library
|
||||
if: matrix.helper == 'armcompute'
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Setting up ARM Compute Library for Android ARM64..."
|
||||
ARMCOMPUTE_VERSION="v25.04"
|
||||
ARMCOMPUTE_HOME="${GITHUB_WORKSPACE}/armcompute_home"
|
||||
ARMCOMPUTE_PACKAGE="arm_compute-${ARMCOMPUTE_VERSION}-android-aarch64-cpu-bin"
|
||||
|
||||
mkdir -p "${ARMCOMPUTE_HOME}"
|
||||
cd "${ARMCOMPUTE_HOME}"
|
||||
|
||||
wget -q "https://github.com/ARM-software/ComputeLibrary/releases/download/${ARMCOMPUTE_VERSION}/${ARMCOMPUTE_PACKAGE}.tar.gz"
|
||||
tar -xzf "${ARMCOMPUTE_PACKAGE}.tar.gz"
|
||||
|
||||
ARMCOMPUTE_ROOT="${ARMCOMPUTE_HOME}/${ARMCOMPUTE_PACKAGE}"
|
||||
echo "ARMCOMPUTE_ROOT=${ARMCOMPUTE_ROOT}" >> "$GITHUB_ENV"
|
||||
echo "ARMCOMPUTE_HOME=${ARMCOMPUTE_HOME}" >> "$GITHUB_ENV"
|
||||
|
||||
|
||||
- name: Build with Maven
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
MAVEN_OPTS: -Xmx3g
|
||||
PROTO_EXEC: /opt/protobuf/bin/protoc
|
||||
run: |
|
||||
export PATH="/opt/protobuf/bin:/opt/cmake/bin:/usr/bin:$PATH"
|
||||
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${OPENBLAS_PATH}"
|
||||
# JAVA_HOME is set by setup-java action or the runner environment
|
||||
find /opt -name java
|
||||
echo "✅ Using Android NDK at: ${ANDROID_NDK}"
|
||||
echo "✅ Cross-compiling from Intel x86_64 to ARM64"
|
||||
|
||||
LIBND4J_CLASSIFIER="android-arm64"
|
||||
|
||||
if [ "${{ github.event.inputs.libnd4jUrl }}" != '' ]; then
|
||||
MODULES=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
MODULES=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
|
||||
CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=${GITHUB_WORKSPACE}/libnd4j/cmake/android-arm64.cmake \
|
||||
-G Ninja \
|
||||
-DSD_ANDROID_BUILD=true \
|
||||
-DANDROID_ABI=arm64-v8a \
|
||||
-DANDROID_PLATFORM=android-21 \
|
||||
-DANDROID_NDK=${ANDROID_NDK} \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_MAKE_PROGRAM=/usr/bin/ninja \
|
||||
-DBLAS_LIBRARIES=${OPENBLAS_PATH}/libopenblas.so \
|
||||
-DLAPACK_LIBRARIES=${OPENBLAS_PATH}/libopenblas.so"
|
||||
|
||||
BASE_COMMAND="mvn ${{ matrix.mvn_ext }} \
|
||||
-Dlibnd4j.generate.flatc=ON \
|
||||
-Djavacpp.platform.compiler=${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++ \
|
||||
--no-transfer-progress \
|
||||
-Posx-aarch64-protoc \
|
||||
-pl ${MODULES} \
|
||||
-Pcpu \
|
||||
-Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} \
|
||||
-Dhttp.keepAlive=false \
|
||||
-Dmaven.wagon.http.pool=false \
|
||||
-Dmaven.wagon.http.retryHandler.count=3 \
|
||||
-Possrh \
|
||||
-DskipTestResourceEnforcement=true \
|
||||
-Dmaven.javadoc.failOnError=false \
|
||||
-Djavacpp.platform=${LIBND4J_CLASSIFIER} \
|
||||
-Dlibnd4j.cmake=\"${CMAKE_ARGS}\" \
|
||||
--also-make \
|
||||
--batch-mode \
|
||||
deploy \
|
||||
-DskipTests"
|
||||
|
||||
if [ "${{ matrix.helper }}" == "armcompute" ]; then
|
||||
HELPER_FLAGS="-Dlibnd4j.helper=${{ matrix.helper }} \
|
||||
-Djavacpp.platform.extension=-${{ matrix.helper }} \
|
||||
-Dlibnd4j.classifier=${LIBND4J_CLASSIFIER}-${{ matrix.helper }}"
|
||||
else
|
||||
HELPER_FLAGS=""
|
||||
fi
|
||||
|
||||
MAVEN_COMMAND="${BASE_COMMAND} ${HELPER_FLAGS}"
|
||||
|
||||
if [ "${PERFORM_RELEASE}" == "1" ]; then
|
||||
bash "${GITHUB_WORKSPACE}/release-specified-component.sh" \
|
||||
"${RELEASE_VERSION}" \
|
||||
"${SNAPSHOT_VERSION}" \
|
||||
"${RELEASE_REPO_ID}" \
|
||||
"${MAVEN_COMMAND}"
|
||||
else
|
||||
eval "${MAVEN_COMMAND}"
|
||||
fi
|
||||
|
||||
- name: Setup tmate session
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
if: ${{ github.event.inputs.debug_enabled == 'true' }}
|
||||
with:
|
||||
limit-access-to-actor: true
|
||||
|
||||
@@ -0,0 +1,529 @@
|
||||
# GitHub Actions Workflow for android-x86_64
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-latest
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
android-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [onednn, ""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- name: Configure swap space
|
||||
shell: bash
|
||||
run: |
|
||||
echo "=== Initial system status ==="
|
||||
free -h
|
||||
df -h
|
||||
echo "Available disk space on root:"
|
||||
df -h /
|
||||
|
||||
# Check if swap is already enabled
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "Swap already configured:"
|
||||
swapon --show
|
||||
else
|
||||
echo "Configuring swap space..."
|
||||
|
||||
# Use fixed 4GB swap size
|
||||
SWAP_SIZE="12G"
|
||||
echo "Creating ${SWAP_SIZE} swap file..."
|
||||
|
||||
# Create swap file
|
||||
sudo fallocate -l ${SWAP_SIZE} /swapfile || {
|
||||
echo "fallocate failed, trying dd..."
|
||||
sudo dd if=/dev/zero of=/swapfile bs=1G count=12 status=progress
|
||||
}
|
||||
|
||||
# Set up swap
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
|
||||
# Verify swap is active
|
||||
echo "=== Swap configuration completed ==="
|
||||
free -h
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
# Tune swappiness for build workloads
|
||||
# Lower values (10-20) prefer RAM, higher values (60+) use swap more aggressively
|
||||
echo "Current swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.swappiness=80
|
||||
echo "Adjusted swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.overcommit_memory=2 # Allow overcommit
|
||||
sudo sysctl vm.overcommit_ratio=80
|
||||
# Show final memory status
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
|
||||
- name: Cache cmake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake-install
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cmake-
|
||||
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-protobuf-
|
||||
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
|
||||
- uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake-install.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- uses: nttld/setup-ndk@v1
|
||||
id: setup-ndk
|
||||
with:
|
||||
ndk-version: r26d
|
||||
|
||||
- name: Set mvn build command and environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
# Set constants for Android X86_64
|
||||
LIBND4J_CLASSIFIER="android-x86_64"
|
||||
|
||||
# Define the specific OpenBLAS JAR for android-x86_64
|
||||
OPENBLAS_VERSION="0.3.28-1.5.11"
|
||||
OPENBLAS_JAR="openblas-${OPENBLAS_VERSION}-${LIBND4J_CLASSIFIER}.jar"
|
||||
OPENBLAS_INSTALL_DIR="${GITHUB_WORKSPACE}/openblas_home"
|
||||
|
||||
NDK_VERSION_ENV="r26d"
|
||||
|
||||
echo "LIBND4J_CLASSIFIER=${LIBND4J_CLASSIFIER}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_VERSION=${OPENBLAS_VERSION}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_JAR=${OPENBLAS_JAR}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_INSTALL_DIR=${OPENBLAS_INSTALL_DIR}" >> $GITHUB_ENV
|
||||
echo "CURRENT_TARGET=android-x86_64" >> $GITHUB_ENV
|
||||
echo "NDK_VERSION=${NDK_VERSION_ENV}" >> $GITHUB_ENV
|
||||
echo "ANDROID_VERSION=21" >> $GITHUB_ENV
|
||||
|
||||
if [ "${{ github.event.inputs.libnd4jUrl }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
|
||||
# Base Maven command
|
||||
command="mvn ${{ matrix.mvn_ext }} -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=${LIBND4J_CLASSIFIER} -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
|
||||
# Updated conditional flags for the helper library
|
||||
if [ "${{ matrix.helper }}" != '' ]; then
|
||||
if [ "${{ matrix.helper }}" == "onednn" ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=onednn -Djavacpp.platform.extension=-onednn -Dlibnd4j.classifier=${LIBND4J_CLASSIFIER}-onednn"
|
||||
else
|
||||
echo "Warning: Unsupported helper '${{ matrix.helper }}' defined in matrix. Only 'onednn' or empty is handled for Maven flags."
|
||||
mvn_ext=""
|
||||
fi
|
||||
else
|
||||
mvn_ext=""
|
||||
fi
|
||||
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
|
||||
- name: Setup OpenBLAS Library
|
||||
shell: bash
|
||||
env:
|
||||
OPENBLAS_INSTALL_DIR: ${{ env.OPENBLAS_INSTALL_DIR }}
|
||||
OPENBLAS_JAR: ${{ env.OPENBLAS_JAR }}
|
||||
OPENBLAS_VERSION: ${{ env.OPENBLAS_VERSION }}
|
||||
LIBND4J_CLASSIFIER: ${{ env.LIBND4J_CLASSIFIER }}
|
||||
run: |
|
||||
echo "=== Setting up OpenBLAS ${OPENBLAS_VERSION} for ${LIBND4J_CLASSIFIER} ==="
|
||||
|
||||
# Create and navigate to install directory
|
||||
mkdir -p "${OPENBLAS_INSTALL_DIR}"
|
||||
cd "${OPENBLAS_INSTALL_DIR}"
|
||||
|
||||
# Download OpenBLAS JAR
|
||||
OPENBLAS_URL="https://repo1.maven.org/maven2/org/bytedeco/openblas/${OPENBLAS_VERSION}/${OPENBLAS_JAR}"
|
||||
echo "Downloading ${OPENBLAS_JAR} from ${OPENBLAS_URL}..."
|
||||
wget --quiet ${OPENBLAS_URL} || {
|
||||
echo "❌ Failed to download OpenBLAS JAR"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Extract JAR
|
||||
echo "Extracting ${OPENBLAS_JAR}..."
|
||||
unzip -q ${OPENBLAS_JAR} || {
|
||||
echo "❌ Failed to extract OpenBLAS JAR"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "=== OpenBLAS JAR extraction completed ==="
|
||||
|
||||
# The JAR contains: lib/x86_64/include/ and lib/x86_64/
|
||||
OPENBLAS_PLATFORM_PATH="lib/x86_64"
|
||||
|
||||
if [[ -d "$OPENBLAS_PLATFORM_PATH" ]]; then
|
||||
echo "✅ Found OpenBLAS platform directory: $OPENBLAS_PLATFORM_PATH"
|
||||
|
||||
# Verify the expected structure
|
||||
if [[ -d "$OPENBLAS_PLATFORM_PATH/include" && -f "$OPENBLAS_PLATFORM_PATH/include/cblas.h" ]]; then
|
||||
echo "✅ Found headers at: $OPENBLAS_PLATFORM_PATH/include/"
|
||||
else
|
||||
echo "❌ Headers not found at expected location: $OPENBLAS_PLATFORM_PATH/include/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -d "$OPENBLAS_PLATFORM_PATH/lib" ]] || [[ -f "$OPENBLAS_PLATFORM_PATH/libopenblas.so" ]]; then
|
||||
echo "✅ Found libraries in: $OPENBLAS_PLATFORM_PATH/"
|
||||
else
|
||||
echo "❌ Libraries not found in: $OPENBLAS_PLATFORM_PATH/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set the correct path - this should be the full path to lib/x86_64
|
||||
OPENBLAS_FINAL_PATH="$(realpath "$OPENBLAS_PLATFORM_PATH")"
|
||||
|
||||
else
|
||||
echo "❌ Could not find OpenBLAS platform directory: $OPENBLAS_PLATFORM_PATH"
|
||||
echo "Available directories:"
|
||||
find . -type d -maxdepth 3 | head -20
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set final environment variables - this is the key fix
|
||||
echo "=== Setting final environment variables ==="
|
||||
echo "OPENBLAS_PATH=${OPENBLAS_FINAL_PATH}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_INCLUDE_PATH=${OPENBLAS_FINAL_PATH}/include" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_LIB_PATH=${OPENBLAS_FINAL_PATH}/" >> $GITHUB_ENV
|
||||
|
||||
echo "=== OpenBLAS setup completed ==="
|
||||
echo "Final OpenBLAS path: ${OPENBLAS_FINAL_PATH}"
|
||||
echo "Include path: ${OPENBLAS_FINAL_PATH}/include"
|
||||
echo "Library path: ${OPENBLAS_FINAL_PATH}/"
|
||||
|
||||
# Verify the final setup
|
||||
echo "=== Final verification ==="
|
||||
ls -la "${OPENBLAS_FINAL_PATH}/include/" | head -10
|
||||
ls -la "${OPENBLAS_FINAL_PATH}/" | grep -E "\.(so|a)$" | head -5
|
||||
|
||||
- name: OpenBLAS Configuration Verification
|
||||
shell: bash
|
||||
run: |
|
||||
echo "=== OpenBLAS Configuration Verification ==="
|
||||
echo "OPENBLAS_PATH: ${OPENBLAS_PATH}"
|
||||
echo "OPENBLAS_INCLUDE_PATH: ${OPENBLAS_INCLUDE_PATH}"
|
||||
echo "OPENBLAS_LIB_PATH: ${OPENBLAS_LIB_PATH}"
|
||||
echo
|
||||
|
||||
# Test that paths are accessible
|
||||
if [[ -d "$OPENBLAS_PATH" ]]; then
|
||||
echo "✅ OPENBLAS_PATH is accessible"
|
||||
else
|
||||
echo "❌ OPENBLAS_PATH is not accessible: $OPENBLAS_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -d "$OPENBLAS_INCLUDE_PATH" ]]; then
|
||||
echo "✅ OPENBLAS_INCLUDE_PATH is accessible"
|
||||
else
|
||||
echo "❌ OPENBLAS_INCLUDE_PATH is not accessible: $OPENBLAS_INCLUDE_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test header file accessibility
|
||||
if [[ -f "$OPENBLAS_INCLUDE_PATH/cblas.h" ]]; then
|
||||
echo "✅ cblas.h is accessible"
|
||||
echo "First few lines of cblas.h:"
|
||||
head -5 "$OPENBLAS_INCLUDE_PATH/cblas.h"
|
||||
else
|
||||
echo "❌ cblas.h is not accessible"
|
||||
echo "Contents of include directory:"
|
||||
ls -la "$OPENBLAS_INCLUDE_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -f "$OPENBLAS_INCLUDE_PATH/openblas_config.h" ]]; then
|
||||
echo "✅ openblas_config.h is accessible"
|
||||
else
|
||||
echo "❌ openblas_config.h is not accessible"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Verification completed successfully ==="
|
||||
|
||||
- name: Debug Info
|
||||
shell: bash
|
||||
run: |
|
||||
echo "--- Workflow Inputs ---"
|
||||
echo "buildThreads: ${{ github.event.inputs.buildThreads }}"
|
||||
echo "deployToReleaseStaging: ${{ github.event.inputs.deployToReleaseStaging }}"
|
||||
echo "releaseVersion: ${{ github.event.inputs.releaseVersion }}"
|
||||
echo "snapshotVersion: ${{ github.event.inputs.snapshotVersion }}"
|
||||
echo "releaseRepoId: ${{ github.event.inputs.releaseRepoId }}"
|
||||
echo "serverId: ${{ github.event.inputs.serverId }}"
|
||||
echo "mvnFlags: ${{ github.event.inputs.mvnFlags }}"
|
||||
echo "libnd4jUrl: ${{ github.event.inputs.libnd4jUrl }}"
|
||||
echo "runsOn: ${{ github.event.inputs.runsOn }}"
|
||||
echo "debug_enabled: ${{ github.event.inputs.debug_enabled }}"
|
||||
echo "--- Matrix Values ---"
|
||||
echo "Helper: ${{ matrix.helper }}"
|
||||
echo "--- Environment Variables ---"
|
||||
echo "LIBND4J_CLASSIFIER: $LIBND4J_CLASSIFIER"
|
||||
echo "OPENBLAS_VERSION: $OPENBLAS_VERSION"
|
||||
echo "OPENBLAS_JAR: $OPENBLAS_JAR"
|
||||
echo "OPENBLAS_INSTALL_DIR: $OPENBLAS_INSTALL_DIR"
|
||||
echo "OPENBLAS_PATH: $OPENBLAS_PATH"
|
||||
echo "OPENBLAS_INCLUDE_PATH: $OPENBLAS_INCLUDE_PATH"
|
||||
echo "OPENBLAS_LIB_PATH: $OPENBLAS_LIB_PATH"
|
||||
echo "CURRENT_TARGET: $CURRENT_TARGET"
|
||||
echo "NDK_VERSION: $NDK_VERSION"
|
||||
echo "ANDROID_VERSION: $ANDROID_VERSION"
|
||||
echo "NDK Path (from step): ${{ steps.setup-ndk.outputs.ndk-path }}"
|
||||
echo "COMMAND: $COMMAND"
|
||||
echo "--- System Info ---"
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
|
||||
- name: Monitor memory during build
|
||||
shell: bash
|
||||
run: |
|
||||
# Start background memory monitoring
|
||||
(while true; do
|
||||
echo "$(date '+%H:%M:%S'): $(free -h | head -n 2 | tail -n 1)"
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "$(date '+%H:%M:%S'): Swap: $(swapon --show --noheadings)"
|
||||
fi
|
||||
sleep 120 # Check every 2 minutes
|
||||
done) > memory_monitor.log 2>&1 &
|
||||
MONITOR_PID=$!
|
||||
echo "MONITOR_PID=${MONITOR_PID}" >> $GITHUB_ENV
|
||||
echo "Started memory monitoring (PID: ${MONITOR_PID})"
|
||||
|
||||
- name: Build with Maven
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
LIBND4J_BUILD_THREADS: ${{ github.event.inputs.buildThreads }}
|
||||
ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }}
|
||||
OPENBLAS_PATH: ${{ env.OPENBLAS_PATH }}
|
||||
ANDROID_VERSION: ${{ env.ANDROID_VERSION }}
|
||||
NDK_VERSION: ${{ env.NDK_VERSION }}
|
||||
PROTO_EXEC: /opt/protobuf/bin/protoc
|
||||
LD_LIBRARY_PATH: $LD_LIBRARY_PATH:${{ env.OPENBLAS_LIB_PATH }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
DEPLOY: 1
|
||||
BUILD_USING_MAVEN: 1
|
||||
TARGET_OS: android
|
||||
PUBLISH_TO: central
|
||||
DEPLOY_TO: central
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
RLIMIT_AS: 6442450944 # 6GB virtual memory limit
|
||||
RLIMIT_DATA: 4294967296 # 4GB data segment limit
|
||||
run: |
|
||||
echo "=== Build Environment Summary ==="
|
||||
echo "NDK Path: ${ANDROID_NDK}"
|
||||
echo "OpenBLAS Path: ${OPENBLAS_PATH}"
|
||||
echo "OpenBLAS Include Path: ${OPENBLAS_INCLUDE_PATH}"
|
||||
echo "OpenBLAS Lib Path: ${OPENBLAS_LIB_PATH}"
|
||||
echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
|
||||
echo "Build Threads: ${LIBND4J_BUILD_THREADS}"
|
||||
echo "Helper: ${HELPER}"
|
||||
echo "================================="
|
||||
|
||||
# Show memory status before build
|
||||
echo "=== Memory status before build ==="
|
||||
free -h
|
||||
|
||||
# Pre-build verification
|
||||
echo "=== Pre-build OpenBLAS verification ==="
|
||||
if [[ -f "$OPENBLAS_PATH/include/cblas.h" ]]; then
|
||||
echo "✅ cblas.h found and accessible"
|
||||
else
|
||||
echo "❌ cblas.h not found at $OPENBLAS_PATH/include/cblas.h"
|
||||
echo "Attempting to locate cblas.h..."
|
||||
find "$OPENBLAS_PATH" -name "cblas.h" 2>/dev/null | head -5
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Execute build
|
||||
if [ "$PERFORM_RELEASE" == 1 ] || [ "$PERFORM_RELEASE" == "true" ]; then
|
||||
echo "=== Executing release build ==="
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
echo "=== Executing snapshot build ==="
|
||||
echo "Command: ${COMMAND}"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
|
||||
- name: Show memory usage summary
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
# Stop memory monitoring
|
||||
if [ -n "$MONITOR_PID" ]; then
|
||||
kill $MONITOR_PID 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "=== Swap usage ==="
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
echo "=== Memory usage during build ==="
|
||||
if [ -f memory_monitor.log ]; then
|
||||
cat memory_monitor.log
|
||||
else
|
||||
echo "No memory monitoring log found"
|
||||
fi
|
||||
|
||||
- name: Cleanup swap
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Cleaning up swap configuration..."
|
||||
sudo swapoff /swapfile 2>/dev/null || true
|
||||
sudo rm -f /swapfile 2>/dev/null || true
|
||||
echo "Swap cleanup completed"
|
||||
|
||||
- name: Upload build artifacts on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: build-logs-${{ matrix.helper }}-${{ github.run_id }}
|
||||
path: |
|
||||
memory_monitor.log
|
||||
**/cmake_install.log
|
||||
**/build.log
|
||||
**/CMakeFiles/CMakeError.log
|
||||
**/CMakeFiles/CMakeOutput.log
|
||||
retention-days: 3
|
||||
@@ -0,0 +1,139 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
jobs:
|
||||
linux-x86_64:
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
- name: Cache cmake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake
|
||||
restore-keys: ${{ runner.os }}-cmake
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf
|
||||
restore-keys: ${{ runner.os }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
- uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake.outputs.cache-hit != 'true'
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Build on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ github.event.inputs.buildThreads }}"
|
||||
echo "deploy to release staging repo or not ${{ github.event.inputs.deployToReleaseStaging }}"
|
||||
echo "release version ${{ github.event.inputs.releaseVersion }}"
|
||||
echo "snapshot version ${{ github.event.inputs.snapshotVersion }}"
|
||||
echo "debug enabled ${{ github.event.inputs.debug_enabled }}"
|
||||
echo "libnd4j url ${{ github.event.inputs.libnd4jUrl }}"
|
||||
echo "maven flags ${{ github.event.inputs.mvnFlags }}"
|
||||
echo "snapshot version ${{ github.event.inputs.snapshotVersion }}"
|
||||
echo "server id ${{ github.event.inputs.serverId }}"
|
||||
echo "release repo id ${{ github.event.inputs.releaseRepoId }}"
|
||||
|
||||
sudo sysctl vm.overcommit_memory=2
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
sudo apt-get autoremove
|
||||
sudo apt-get clean
|
||||
command="mvn -Possrh -pl !:blas-lapack-generator,!:libnd4j-gen -DskipTestResourceEnforcement=true -Djavacpp.platform=linux-x86_64 --batch-mode deploy -DskipTests"
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${command}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "$command"
|
||||
fi
|
||||
|
||||
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${command}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "$command"
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04-arm
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
swapSize:
|
||||
description: 'Swap file size in GB (e.g., 8 for 8GB)'
|
||||
required: false
|
||||
default: 8
|
||||
|
||||
jobs:
|
||||
linux-arm64:
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [armcompute, ""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
- swap_size: ${{ github.event.inputs.swapSize }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- name: Configure swap space
|
||||
shell: bash
|
||||
run: |
|
||||
sudo fallocate -l 16G /swapfile
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
sudo sysctl vm.swappiness=60
|
||||
free -h
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
LIBND4J_CLASSIFIER="linux-arm64"
|
||||
echo "LIBND4J_CLASSIFIER=${LIBND4J_CLASSIFIER}" >> $GITHUB_ENV
|
||||
echo "OPENBLAS_JAR=openblas-0.3.28-1.5.11-linux-arm64.jar" >> $GITHUB_ENV
|
||||
echo "ARMCOMPUTE_TARGET=arm64-v8a" >> $GITHUB_ENV
|
||||
if [ "${{ github.event.inputs.libnd4jUrl }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Posx-aarch64-protoc -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=${LIBND4J_CLASSIFIER} -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.classifier=${LIBND4J_CLASSIFIER}-${{ matrix.helper }}"
|
||||
else
|
||||
mvn_ext=""
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
- name: Setup Path
|
||||
shell: bash
|
||||
run: |
|
||||
echo "/opt/protobuf/bin" >> $GITHUB_PATH
|
||||
echo "/opt/cmake/bin" >> $GITHUB_PATH
|
||||
- name: Setup OpenBLAS and ARM Compute Library
|
||||
shell: bash
|
||||
run: |
|
||||
# Setup OpenBLAS
|
||||
mkdir -p "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/${OPENBLAS_JAR}
|
||||
unzip ${OPENBLAS_JAR}
|
||||
OPENBLAS_DIR="${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-arm64"
|
||||
echo "OPENBLAS_PATH=${OPENBLAS_DIR}" >> "$GITHUB_ENV"
|
||||
# Ensure the symlink exists for consistency
|
||||
ln -sf "${OPENBLAS_DIR}/libopenblas.so.0" "${OPENBLAS_DIR}/libopenblas.so"
|
||||
# Setup ARM Compute Library if selected
|
||||
if [ "${{ matrix.helper }}" == "armcompute" ]; then
|
||||
ARMCOMPUTE_HOME="${GITHUB_WORKSPACE}/armcompute_home"
|
||||
mkdir -p "${ARMCOMPUTE_HOME}"
|
||||
cd "${ARMCOMPUTE_HOME}"
|
||||
wget https://github.com/ARM-software/ComputeLibrary/releases/download/v25.04/arm_compute-v25.04-linux-aarch64-cpu-bin.tar.gz
|
||||
tar -xzf arm_compute-v25.04-linux-aarch64-cpu-bin.tar.gz
|
||||
# Set ARMCOMPUTE_ROOT to the explicit path of the extracted contents
|
||||
EXTRACTED_DIR="${ARMCOMPUTE_HOME}/arm_compute-v25.04-linux-aarch64-cpu-bin"
|
||||
echo "ARMCOMPUTE_ROOT=${EXTRACTED_DIR}" >> "$GITHUB_ENV"
|
||||
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${EXTRACTED_DIR}/lib/armv8a-neon" >> "$GITHUB_ENV"
|
||||
fi
|
||||
- name: Build with Maven
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
PROTO_EXEC: /opt/protobuf/bin/protoc
|
||||
run: |
|
||||
# Pre-build native libraries with CMake if not using a pre-packaged libnd4j
|
||||
if [ "${{ github.event.inputs.libnd4jUrl }}" == '' ]; then
|
||||
echo "Starting manual pre-build of libnd4j..."
|
||||
|
||||
# Define CMake arguments for building libnd4j
|
||||
cmake_args="-DOPENBLAS_PATH=${OPENBLAS_PATH}"
|
||||
|
||||
# Add armcompute helper flags if enabled
|
||||
if [ "${{ matrix.helper }}" == "armcompute" ]; then
|
||||
echo "Enabling armcompute helper for native build."
|
||||
cmake_args="${cmake_args} -DHELPERS_armcompute=true -DARMCOMPUTE_ROOT=${ARMCOMPUTE_ROOT}"
|
||||
fi
|
||||
|
||||
# Navigate to libnd4j build directory
|
||||
cd "${GITHUB_WORKSPACE}/libnd4j"
|
||||
mkdir -p blasbuild/linux-arm64
|
||||
cd blasbuild/linux-arm64
|
||||
|
||||
# Configure with CMake
|
||||
echo "Configuring with CMake args: ${cmake_args}"
|
||||
cmake ${cmake_args} ../..
|
||||
|
||||
# Build with specified number of threads
|
||||
echo "Building with make -j${{ github.event.inputs.buildThreads }}"
|
||||
make -j${{ github.event.inputs.buildThreads }}
|
||||
cd ${GITHUB_WORKSPACE}
|
||||
echo "Manual pre-build of libnd4j finished."
|
||||
fi
|
||||
|
||||
# Run the main Maven build
|
||||
echo "Running Maven command: ${COMMAND}"
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
- name: Cleanup swap
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
sudo swapoff /swapfile || true
|
||||
sudo rm -f /swapfile || true
|
||||
@@ -0,0 +1,311 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 4
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Whether to download libnd4j using https://github.com/KonduitAI/gh-actions-libnd4j-urls/ for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
jobs:
|
||||
linux-x86_64-cuda-12-3:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ cudnn, "" ]
|
||||
extension: [ "" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
env:
|
||||
CUDA_PATH: /usr/local/cuda-12.3
|
||||
CUDNN_ROOT_DIR: /usr/local/cuda-12.3
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-cuda,:nd4j-cuda-preset'
|
||||
elif [ "${{ matrix.helper }}" == '' ]; then
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-cuda,:nd4j-cuda-preset,:libnd4j,:nd4j-cuda-platform'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-cuda,:nd4j-cuda-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Pcuda -Dlibnd4j.generate.flatc=ON --no-transfer-progress -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda -Pcuda --also-make -pl ${modules} -Dlibnd4j.compute='8.6 9.0' -Dlibnd4j.cpu.compile.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cuda --also-make -Pcuda clean --batch-mode package deploy -DskipTests"
|
||||
libnd4j_download_file_url=""
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-cuda-12.3-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
libnd4j_download_file_url="linux-cuda-12.3-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.classifier=linux-x86_64-cuda-12.3-${{ matrix.helper }}"
|
||||
libnd4j_download_file_url="linux-cuda-12.3-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
else
|
||||
mvn_ext=" -Dlibnd4j.classifier=linux-x86_64-cuda-12.3"
|
||||
libnd4j_download_file_url="linux-cuda-12.3-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ matrix.runs_on }}-protobuf
|
||||
restore-keys: ${{ matrix.runs_on }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master
|
||||
env:
|
||||
cuda: 12.3.0
|
||||
GCC: 11
|
||||
if: steps.cache-cuda-123.outputs.cache-hit != 'true'
|
||||
|
||||
# Set up CUDA environment paths
|
||||
- name: Setup CUDA PATH
|
||||
run: |
|
||||
echo "PATH=/usr/local/cuda-12.3/bin:$PATH" >> $GITHUB_ENV
|
||||
echo "LD_LIBRARY_PATH=/usr/local/cuda-12.3/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||
|
||||
# Debug CUDA installation
|
||||
- name: Debug CUDA Installation
|
||||
run: |
|
||||
echo "Debugging CUDA installation"
|
||||
echo "Current PATH: $PATH"
|
||||
echo "Current LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
|
||||
echo "Contents of /usr/local/cuda-12.3:"
|
||||
ls -la /usr/local/cuda-12.3
|
||||
echo "Contents of /usr/local/cuda-12.3/bin:"
|
||||
ls -la /usr/local/cuda-12.3/bin
|
||||
echo "Finding nvcc:"
|
||||
find /usr/local -name nvcc 2>/dev/null
|
||||
echo "CUDA installation info:"
|
||||
if [ -f /usr/local/cuda-12.3/bin/nvcc ]; then
|
||||
/usr/local/cuda-12.3/bin/nvcc --version
|
||||
else
|
||||
echo "nvcc not found in expected location"
|
||||
# Try to find in alternative locations
|
||||
find /usr -name nvcc 2>/dev/null
|
||||
fi
|
||||
# Check symlinks
|
||||
echo "Checking CUDA symlinks:"
|
||||
ls -la /usr/local/cuda*
|
||||
# Create symlink if needed
|
||||
if [ ! -f /usr/local/cuda-12.3/bin/nvcc ]; then
|
||||
echo "Attempting to fix nvcc location"
|
||||
# Check if nvcc exists in alternative location
|
||||
nvcc_path=$(find /usr -name nvcc 2>/dev/null | head -1)
|
||||
if [ -n "$nvcc_path" ]; then
|
||||
echo "Found nvcc at $nvcc_path"
|
||||
sudo mkdir -p /usr/local/cuda-12.3/bin
|
||||
sudo ln -sf $nvcc_path /usr/local/cuda-12.3/bin/nvcc
|
||||
echo "Created symlink to nvcc"
|
||||
ls -la /usr/local/cuda-12.3/bin/nvcc
|
||||
fi
|
||||
fi
|
||||
|
||||
# Verify the CUDA installation and PATH setup
|
||||
- name: Verify CUDA Setup
|
||||
run: |
|
||||
echo "Verifying CUDA installation and PATH setup"
|
||||
echo $PATH
|
||||
ls -la /usr/local/cuda-12.3/bin
|
||||
which nvcc || echo "nvcc not found in PATH"
|
||||
nvcc --version || echo "nvcc command failed"
|
||||
# If still failing, try direct path
|
||||
if ! which nvcc; then
|
||||
echo "Trying direct path to nvcc"
|
||||
/usr/local/cuda-12.3/bin/nvcc --version || echo "Direct nvcc command failed"
|
||||
fi
|
||||
|
||||
- name: Run cuda compilation on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
LIBND4J_HOME_SUFFIX: cuda
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
# Explicitly set PATH and other environment variables for this step
|
||||
export PATH=/usr/local/cuda-12.3/bin:/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.3/lib64:$LD_LIBRARY_PATH
|
||||
|
||||
# Final check for nvcc before proceeding
|
||||
echo "Final check for nvcc:"
|
||||
which nvcc || echo "nvcc still not found in PATH"
|
||||
if ! which nvcc; then
|
||||
echo "WARNING: nvcc not in PATH, trying to diagnose and fix..."
|
||||
direct_nvcc=$(find /usr -name nvcc 2>/dev/null | head -1)
|
||||
if [ -n "$direct_nvcc" ]; then
|
||||
echo "Found nvcc at $direct_nvcc"
|
||||
sudo ln -sf $direct_nvcc /usr/bin/nvcc
|
||||
echo "Created global symlink to nvcc"
|
||||
which nvcc
|
||||
else
|
||||
echo "ERROR: Could not find nvcc anywhere. Build may fail!"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Test that nvcc is properly in the PATH
|
||||
echo "Checking nvcc availability:"
|
||||
which nvcc
|
||||
nvcc --version
|
||||
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
sudo apt-get autoremove
|
||||
sudo apt-get clean
|
||||
bash ./change-cuda-versions.sh 12.3
|
||||
|
||||
# Note: we need this for the cudnn helpers, our cmake can't find it otherwise.
|
||||
# See here: https://github.com/eclipse/deeplearning4j/blob/master/libnd4j/CMakeLists.txt#L298
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
@@ -0,0 +1,304 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 4
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Whether to download libnd4j using https://github.com/KonduitAI/gh-actions-libnd4j-urls/ for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
jobs:
|
||||
linux-x86_64-cuda-12-6:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ cudnn, "" ]
|
||||
extension: [ "" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
env:
|
||||
CUDA_PATH: /usr/local/cuda-12.6
|
||||
CUDNN_ROOT_DIR: /usr/local/cuda-12.6
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- name: Configure swap space
|
||||
shell: bash
|
||||
run: |
|
||||
echo "=== Initial system status ==="
|
||||
free -h
|
||||
df -h
|
||||
echo "Available disk space on root:"
|
||||
df -h /
|
||||
|
||||
# Check if swap is already enabled
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "Swap already configured:"
|
||||
swapon --show
|
||||
else
|
||||
echo "Configuring swap space..."
|
||||
|
||||
# Use input parameter for swap size, default to 12GB like Android workflow
|
||||
SWAP_SIZE="12G"
|
||||
echo "Creating ${SWAP_SIZE} swap file..."
|
||||
|
||||
# Create swap file
|
||||
sudo fallocate -l ${SWAP_SIZE} /swapfile || {
|
||||
sudo dd if=/dev/zero of=/swapfile bs=1G count=12 status=progress
|
||||
}
|
||||
|
||||
# Set up swap
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
|
||||
# Verify swap is active
|
||||
echo "=== Swap configuration completed ==="
|
||||
free -h
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
# Tune swappiness for build workloads - using Android workflow settings
|
||||
# Lower values (10-20) prefer RAM, higher values (60+) use swap more aggressively
|
||||
echo "Current swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.swappiness=80
|
||||
echo "Adjusted swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.overcommit_memory=2 # Allow overcommit
|
||||
sudo sysctl vm.overcommit_ratio=80
|
||||
sudo sysctl vm.vfs_cache_pressure=50
|
||||
sudo sysctl vm.dirty_ratio=10
|
||||
sudo sysctl vm.dirty_background_ratio=5
|
||||
sudo sysctl kernel.shmmax=68719476736
|
||||
# Show final memory status
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-cuda-12.6,:nd4j-cuda-12.6-preset'
|
||||
elif [ "${{ matrix.helper }}" == '' ]; then
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-cuda-12.6,:nd4j-cuda-12.6-preset,:libnd4j,:nd4j-cuda-12.6-platform'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-cuda-12.6,:nd4j-cuda-12.6-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Pcuda -Dlibnd4j.generate.flatc=ON --no-transfer-progress -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda -pl ${modules} -Dlibnd4j.compute='8.6 9.0' -Dlibnd4j.cpu.compile.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Djavacpp.platform=linux-x86_64 -Dlibnd4j.chip=cuda --also-make clean --batch-mode package deploy -DskipTests"
|
||||
libnd4j_download_file_url=""
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-cuda-12.6-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
libnd4j_download_file_url="linux-cuda-12.6-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.classifier=linux-x86_64-cuda-12.6-${{ matrix.helper }}"
|
||||
libnd4j_download_file_url="linux-cuda-12.6-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
else
|
||||
mvn_ext=" -Dlibnd4j.classifier=linux-x86_64-cuda-12.6"
|
||||
libnd4j_download_file_url="linux-cuda-12.6-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ matrix.runs_on }}-protobuf
|
||||
restore-keys: ${{ matrix.runs_on }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master
|
||||
env:
|
||||
cuda: 12.6.0
|
||||
GCC: 11
|
||||
if: steps.cache-cuda-126.outputs.cache-hit != 'true'
|
||||
|
||||
# Set up CUDA environment paths
|
||||
- name: Setup CUDA PATH
|
||||
run: |
|
||||
echo "PATH=/usr/local/cuda-12.6/bin:$PATH" >> $GITHUB_ENV
|
||||
echo "LD_LIBRARY_PATH=/usr/local/cuda-12.6/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||
|
||||
# Verify the CUDA installation and PATH setup
|
||||
- name: Verify CUDA Setup
|
||||
run: |
|
||||
echo "Verifying CUDA installation and PATH setup"
|
||||
echo $PATH
|
||||
ls -la /usr/local/cuda-12.6/bin
|
||||
which nvcc || echo "nvcc not found in PATH"
|
||||
nvcc --version || echo "nvcc command failed"
|
||||
|
||||
- name: Run cuda compilation on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
LIBND4J_HOME_SUFFIX: cuda
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
# Explicitly set PATH and other environment variables for this step
|
||||
export PATH=/usr/local/cuda-12.6/bin:/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.6/lib64:$LD_LIBRARY_PATH
|
||||
|
||||
# Test that nvcc is properly in the PATH
|
||||
echo "Checking nvcc availability:"
|
||||
which nvcc
|
||||
nvcc --version
|
||||
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
sudo apt-get autoremove
|
||||
sudo apt-get clean
|
||||
bash ./change-cuda-versions.sh 12.6
|
||||
|
||||
# Note: we need this for the cudnn helpers, our cmake can't find it otherwise.
|
||||
# See here: https://github.com/eclipse/deeplearning4j/blob/master/libnd4j/CMakeLists.txt#L298
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
@@ -0,0 +1,210 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
jobs:
|
||||
#Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
|
||||
linux-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [""]
|
||||
extension: [compat]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space Before Build
|
||||
run: |
|
||||
echo "Disk space before cleanup:"
|
||||
df -h
|
||||
sudo rm -rf /usr/local/.ghcup
|
||||
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
||||
sudo rm -rf /usr/local/lib/android/sdk/ndk
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /opt/ghc
|
||||
sudo rm -rf /usr/local/share/boost
|
||||
sudo apt-get clean
|
||||
echo "Disk space after cleanup:"
|
||||
df -h
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
command="mvn -pl ':nd4j-native-preset,:libnd4j,:nd4j-native' ${{ matrix.mvn_ext }} -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=linux-x86_64 -Pcpu --also-make --batch-mode -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.extension }}"
|
||||
else
|
||||
mvn_ext=""
|
||||
fi
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
echo "Adding libnd4j download"
|
||||
echo "LIBND4J_FILE_NAME=${libnd4j_download_file_url}" >> $GITHUB_ENV
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "DEPLOY_COMMAND=${command} deploy -Dlibnd4j.cpu.compile.skip=true -Djavacpp.compiler.skip=true -Djavacpp.parser.skip=true" >> $GITHUB_ENV
|
||||
echo "INSTALL_COMMAND=${command} install" >> "$GITHUB_ENV"
|
||||
echo "COMMAND=${command}" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
|
||||
- name: Build on linux-x86_64
|
||||
uses: ./.github/actions/build-centos
|
||||
env:
|
||||
JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64/
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
|
||||
- name: Deploy
|
||||
shell: bash
|
||||
run: |
|
||||
# Docker builds are root owned
|
||||
sudo chown -R $USER $GITHUB_WORKSPACE/*
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
export COMMAND="${DEPLOY_COMMAND}"
|
||||
bash "$GITHUB_WORKSPACE/release-specified-component.sh" "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${DEPLOY_COMMAND}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${DEPLOY_COMMAND}"
|
||||
fi
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
@@ -0,0 +1,360 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: ubuntu-22.04
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
swapSize:
|
||||
description: 'Swap file size in GB (e.g., 4 for 4GB)'
|
||||
required: false
|
||||
default: 12
|
||||
|
||||
jobs:
|
||||
#Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
|
||||
linux-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [onednn,""]
|
||||
extension: [avx2,avx512,""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: libnd4j download URL
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: Release repository id
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
- swap_size: ${{ github.event.inputs.swapSize }}
|
||||
experimental: true
|
||||
name: Swap file size in GB
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- name: Configure swap space
|
||||
shell: bash
|
||||
run: |
|
||||
echo "=== Initial system status ==="
|
||||
free -h
|
||||
df -h
|
||||
echo "Available disk space on root:"
|
||||
df -h /
|
||||
|
||||
# Check if swap is already enabled
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "Swap already configured:"
|
||||
swapon --show
|
||||
else
|
||||
echo "Configuring swap space..."
|
||||
|
||||
# Use input parameter for swap size, default to 12GB like Android workflow
|
||||
SWAP_SIZE="${{ matrix.swap_size }}G"
|
||||
echo "Creating ${SWAP_SIZE} swap file..."
|
||||
|
||||
# Create swap file
|
||||
sudo fallocate -l ${SWAP_SIZE} /swapfile || {
|
||||
echo "fallocate failed, trying dd..."
|
||||
sudo dd if=/dev/zero of=/swapfile bs=1G count=${{ matrix.swap_size }} status=progress
|
||||
}
|
||||
|
||||
# Set up swap
|
||||
sudo chmod 600 /swapfile
|
||||
sudo mkswap /swapfile
|
||||
sudo swapon /swapfile
|
||||
|
||||
# Verify swap is active
|
||||
echo "=== Swap configuration completed ==="
|
||||
free -h
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
# Tune swappiness for build workloads - using Android workflow settings
|
||||
# Lower values (10-20) prefer RAM, higher values (60+) use swap more aggressively
|
||||
echo "Current swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.swappiness=80
|
||||
echo "Adjusted swappiness: $(cat /proc/sys/vm/swappiness)"
|
||||
sudo sysctl vm.overcommit_memory=2 # Allow overcommit
|
||||
sudo sysctl vm.overcommit_ratio=80
|
||||
|
||||
# Show final memory status
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=linux-x86_64 -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }} -Dlibnd4j.classifier=linux-x86_64-${{ matrix.extension }}"
|
||||
else
|
||||
mvn_ext=""
|
||||
fi
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
echo "Adding libnd4j download"
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
|
||||
- name: Cache cmake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cmake-
|
||||
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-protobuf-
|
||||
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
|
||||
- uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake.outputs.cache-hit != 'true'
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/openblas-0.3.28-1.5.11-linux-x86_64.jar
|
||||
unzip openblas-0.3.28-1.5.11-linux-x86_64.jar
|
||||
cd ..
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64" >> "$GITHUB_ENV"
|
||||
cp ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64/libopenblas.so.0 ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64/libopenblas.so
|
||||
if: ${{ github.event.inputs.libnd4jUrl != '' }}
|
||||
|
||||
- name: Monitor memory during build
|
||||
shell: bash
|
||||
run: |
|
||||
# Start background memory monitoring
|
||||
(while true; do
|
||||
echo "$(date '+%H:%M:%S'): $(free -h | head -n 2 | tail -n 1)"
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "$(date '+%H:%M:%S'): Swap: $(swapon --show --noheadings)"
|
||||
fi
|
||||
sleep 120 # Check every 2 minutes
|
||||
done) > memory_monitor.log 2>&1 &
|
||||
MONITOR_PID=$!
|
||||
echo "MONITOR_PID=${MONITOR_PID}" >> $GITHUB_ENV
|
||||
echo "Started memory monitoring (PID: ${MONITOR_PID})"
|
||||
|
||||
- name: Build on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
RLIMIT_AS: 6442450944 # 6GB virtual memory limit
|
||||
RLIMIT_DATA: 4294967296 # 4GB data segment limit
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
echo "swap size ${{ matrix.swap_size }}GB"
|
||||
|
||||
# Show memory status before build
|
||||
echo "=== Memory status before build ==="
|
||||
free -h
|
||||
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$OPENBLAS_PATH"
|
||||
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
|
||||
- name: Show memory usage summary
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
# Stop memory monitoring
|
||||
if [ -n "$MONITOR_PID" ]; then
|
||||
kill $MONITOR_PID 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "=== Final memory status ==="
|
||||
free -h
|
||||
if swapon --show | grep -q "/"; then
|
||||
echo "=== Swap usage ==="
|
||||
swapon --show
|
||||
fi
|
||||
|
||||
echo "=== Memory usage during build ==="
|
||||
if [ -f memory_monitor.log ]; then
|
||||
cat memory_monitor.log
|
||||
else
|
||||
echo "No memory monitoring log found"
|
||||
fi
|
||||
|
||||
- name: Cleanup swap
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Cleaning up swap configuration..."
|
||||
sudo swapoff /swapfile 2>/dev/null || true
|
||||
sudo rm -f /swapfile 2>/dev/null || true
|
||||
echo "Swap cleanup completed"
|
||||
@@ -0,0 +1,222 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 3
|
||||
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: macos-14
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
mac-arm64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [""]
|
||||
extension: [""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Dlibnd4j.arch=armv8-a -Dlibnd4j.platform=macosx-arm64 -Djavacpp.platform=macosx-arm64 -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=macosx-arm64 -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext="-Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-arm64-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=${{ matrix.helper }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-arm64-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }}"
|
||||
else
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-arm64"
|
||||
fi
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
echo "Adding libnd4j download"
|
||||
echo "LIBND4J_FILE_NAME=${libnd4j_download_file_url}" >> $GITHUB_ENV
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/MacOSX10*
|
||||
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Update path for gnu sed
|
||||
shell: bash
|
||||
run: |
|
||||
brew install gnu-sed unzip ccache gcc swig autoconf-archive automake libomp libtool libusb ant maven nasm xz pkg-config sdl bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python
|
||||
echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/openblas-0.3.28-1.5.11-macosx-arm64.jar
|
||||
unzip openblas-0.3.28-1.5.11-macosx-arm64.jar
|
||||
cd ..
|
||||
cp ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-arm64/libopenblas.0.dylib ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-arm64/libopenblas.dylib
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-arm64/" >> "$GITHUB_ENV"
|
||||
|
||||
if: ${{ matrix.libnd4j_file_download != '' }}
|
||||
|
||||
- name: Build and install
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
|
||||
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
|
||||
brew list
|
||||
brew list --cask
|
||||
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release with command ${COMMAND}"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 3
|
||||
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: macos-10.15
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
mac-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ onednn,"" ]
|
||||
extension: [ avx2,avx512,"" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
modules=':nd4j-native,:nd4j-native-preset'
|
||||
else
|
||||
echo "Building libnd4j from source"
|
||||
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
|
||||
fi
|
||||
command="mvn ${{ matrix.mvn_ext }} -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=macosx-x86_64 -Pcpu --also-make --batch-mode deploy -DskipTests"
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-x86_64-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=${{ matrix.helper }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-x86_64-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }}"
|
||||
else
|
||||
mvn_ext=" -Posx-aarch64-protoc -Dlibnd4j.classifier=macosx-x86_64"
|
||||
fi
|
||||
if [ "${{ matrix.libnd4j_file_download }}" != '' ]; then
|
||||
echo "Adding libnd4j download"
|
||||
echo "LIBND4J_FILE_NAME=${{ matrix.libnd4j_file_download }}/${libnd4j_download_file_url}" >> $GITHUB_ENV
|
||||
fi
|
||||
command="${command} ${mvn_ext}"
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to ${command}"
|
||||
echo "COMMAND=${command}" >> $GITHUB_ENV
|
||||
|
||||
|
||||
|
||||
- name: Import GPG Key
|
||||
uses: crazy-max/ghaction-import-gpg@v1
|
||||
env:
|
||||
GPG_PRIVATE_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
|
||||
|
||||
- name: Update path for gnu sed
|
||||
shell: bash
|
||||
run: |
|
||||
brew install gpg1 gnu-sed unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python
|
||||
echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.19-1.5.7/openblas-0.3.19-1.5.7-macosx-x86_64.jar
|
||||
unzip openblas-0.3.19-1.5.7-macosx-x86_64.jar
|
||||
cd ..
|
||||
cp ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-x86_64/libopenblas.0.dylib ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-x86_64/libopenblas.dylib
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/macosx-x86_64/" >> "$GITHUB_ENV"
|
||||
|
||||
if: ${{ matrix.libnd4j_file_download != '' }}
|
||||
|
||||
- name: Build and install
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
|
||||
|
||||
run: |
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
gpg --version
|
||||
gpg1 --version
|
||||
brew list
|
||||
brew list --cask
|
||||
|
||||
if [ "$PERFORM_RELEASE" == 1 ]; then
|
||||
echo "Performing release with command ${COMMAND}"
|
||||
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
|
||||
|
||||
else
|
||||
echo "Running build and deploying to snapshots"
|
||||
eval "${COMMAND}"
|
||||
fi
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
# Workflow for CUDA 12.3 Windows build
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 4
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
libnd4jUrl:
|
||||
description: 'Whether to download libnd4j using https://github.com/KonduitAI/gh-actions-libnd4j-urls/ for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: windows-2019
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
windows-x86_64-cuda-12-3:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ cudnn,"" ]
|
||||
extension: [ "" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set NVCC Compiler Flags
|
||||
shell: powershell
|
||||
run: |
|
||||
echo "Setting NVCC environment variables to allow unsupported compiler..."
|
||||
echo "CUDAFLAGS=--allow-unsupported-compiler -D__NVCC_ALLOW_UNSUPPORTED_COMPILER__" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "NVCC_APPEND_FLAGS=--allow-unsupported-compiler -D__NVCC_ALLOW_UNSUPPORTED_COMPILER__" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "CMAKE_CUDA_FLAGS=--allow-unsupported-compiler -D__NVCC_ALLOW_UNSUPPORTED_COMPILER__" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
- name: Free Disk Space (Windows)
|
||||
shell: powershell
|
||||
run: |
|
||||
# Show initial disk space
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
# Remove Windows Defender scan history
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed Windows Defender scan history"
|
||||
|
||||
# Clear Windows temp folders
|
||||
Remove-Item -Path "$env:SystemRoot\Temp\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows temp folders"
|
||||
|
||||
# Clear Windows Update cache safely (without stopping/starting service)
|
||||
try {
|
||||
Remove-Item -Path "$env:SystemRoot\SoftwareDistribution\Download\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows Update download cache"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Could not clear Windows Update cache. Continuing..."
|
||||
}
|
||||
|
||||
# Clean package manager caches
|
||||
if (Test-Path -Path "C:\npm\cache") {
|
||||
Remove-Item -Path "C:\npm\cache\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared NPM cache"
|
||||
}
|
||||
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Chocolatey cache"
|
||||
|
||||
# Remove Docker images if Docker is installed
|
||||
try {
|
||||
if (Get-Command "docker" -ErrorAction SilentlyContinue) {
|
||||
docker image prune -a -f
|
||||
docker container prune -f
|
||||
docker volume prune -f
|
||||
Write-Host "Pruned Docker resources"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to prune Docker resources. Continuing..."
|
||||
}
|
||||
|
||||
# Remove .NET SDK/Runtime backup folders
|
||||
if (Test-Path -Path "$env:ProgramData\Microsoft\.NET\*.backup") {
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\.NET\*.backup" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed .NET backup folders"
|
||||
}
|
||||
|
||||
# Clear Azure artifacts cache
|
||||
if (Test-Path -Path "$env:LOCALAPPDATA\Microsoft\Azure\*") {
|
||||
Remove-Item -Path "$env:LOCALAPPDATA\Microsoft\Azure\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Azure artifacts cache"
|
||||
}
|
||||
|
||||
# Optimize Windows Component Store
|
||||
try {
|
||||
Start-Process -FilePath "dism.exe" -ArgumentList "/online /Cleanup-Image /StartComponentCleanup" -NoNewWindow -Wait
|
||||
Write-Host "Optimized Windows Component Store"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to optimize Windows Component Store. Continuing..."
|
||||
}
|
||||
|
||||
# Show final disk space
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: powershell
|
||||
run: |
|
||||
if ( "${{ matrix.libnd4j_file_download }}" -ne "" ) {
|
||||
$modules=" :nd4j-cuda-preset,:nd4j-cuda"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$modules=":nd4j-cuda-preset,:nd4j-cuda,libnd4j"
|
||||
} elseif ( "${{ matrix.helper }}" -eq "" ) {
|
||||
$modules=":nd4j-cuda-preset,:nd4j-cuda,libnd4j,:nd4j-cuda-platform"
|
||||
}
|
||||
|
||||
$command="mvn ${{ matrix.mvn_ext }} -Pcuda -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda -Pcuda --also-make -pl $modules -Dlibnd4j.compute=`"8.6 9.0`" -Dlibnd4j.cpu.compile.skip=true -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests"
|
||||
|
||||
if ( "${{ matrix.helper }}" -ne "" -And "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.chip=cuda -Dlibnd4j.classifier=windows-x86_64-cuda-12.3-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests"
|
||||
$libnd4j_download_file_url="windows-cuda-12.3-${{ matrix.extension }}-${{ matrix.helper }}"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-cuda-12.3-${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests"
|
||||
$libnd4j_download_file_url="windows-cuda-12.3-${{ matrix.helper }}"
|
||||
} else {
|
||||
$libnd4j_download_file_url="windows-cuda-12.3"
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-cuda-12.3"
|
||||
}
|
||||
|
||||
$command2= -join("$($command)","$($mvn_ext)");
|
||||
$to_write = -join("COMMAND=","$($command2)");
|
||||
if ( "${{ matrix.libnd4j_file_download }}" -ne "") {
|
||||
Write-Host "Adding libnd4j download URL to GITHUB_ENV"
|
||||
$libnd4j_url_to_write = -join("LIBND4J_FILE_NAME=","$(${{ matrix.libnd4j_file_download }}/$libnd4j_download_file_url)");
|
||||
echo $libnd4j_url_to_write | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
}
|
||||
Write-Host "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to $($command2) and the libnd4j bootstrap file name to $($libnd4j_download_file_url)"
|
||||
echo $command2 | Out-File -FilePath "$env:GITHUB_WORKSPACE/mvn-command.bat" -Encoding utf8
|
||||
echo $to_write | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
- name: Install MSYS2 and dependencies
|
||||
uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: MINGW64
|
||||
update: true
|
||||
install: >-
|
||||
base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake patch
|
||||
mingw-w64-x86_64-make mingw-w64-x86_64-gnupg mingw-w64-x86_64-cmake mingw-w64-x86_64-nasm
|
||||
mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-fortran
|
||||
mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-x86_64-ragel
|
||||
|
||||
|
||||
- name: Cache cuda install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-123
|
||||
with:
|
||||
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.3
|
||||
key: ${{ matrix.runs_on }}-cuda-12.3-${{ matrix.helper }}-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ matrix.runs_on }}-cuda-12.3-${{ matrix.helper }}-
|
||||
|
||||
- name: Install CUDA 12.3 (if not cached)
|
||||
if: steps.cache-cuda-123.outputs.cache-hit != 'true'
|
||||
shell: powershell
|
||||
env:
|
||||
CUDA_VERSION: "12.3"
|
||||
run: |
|
||||
$scriptUrl = "https://raw.githubusercontent.com/KonduitAI/cuda-install/master/.github/actions/install-cuda-windows/install_cuda_windows.ps1"
|
||||
$scriptPath = ".\install_cuda_windows.ps1"
|
||||
Write-Host "Downloading CUDA install script from $scriptUrl..."
|
||||
Invoke-WebRequest $scriptUrl -OutFile $scriptPath -UseBasicParsing
|
||||
if (Test-Path $scriptPath) {
|
||||
Write-Host "Download complete. Executing script with CUDA_VERSION=$($env:CUDA_VERSION)..."
|
||||
& $scriptPath
|
||||
} else {
|
||||
Write-Error "Failed to download CUDA install script!"
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
|
||||
- name: Setup Windows PATH and Compiler Environment
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "Setting up Windows PATH and Compiler Environment..."
|
||||
|
||||
# Find Visual Studio installation
|
||||
$vsPaths = @(
|
||||
"${env:ProgramFiles}\Microsoft Visual Studio\2022\Enterprise",
|
||||
"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\Enterprise",
|
||||
"${env:ProgramFiles}\Microsoft Visual Studio\2019\Enterprise",
|
||||
"${env:ProgramFiles(x86)}\Microsoft Visual Studio\2019\Enterprise"
|
||||
)
|
||||
|
||||
$vsPath = $null
|
||||
foreach ($path in $vsPaths) {
|
||||
if (Test-Path $path) {
|
||||
$vsPath = $path
|
||||
Write-Host "Found Visual Studio at: $vsPath"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $vsPath) {
|
||||
Write-Error "Visual Studio not found in any expected location"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Setup MSVC environment using vcvars64.bat
|
||||
$vcvarsPath = "$vsPath\VC\Auxiliary\Build\vcvars64.bat"
|
||||
if (-not (Test-Path $vcvarsPath)) {
|
||||
Write-Error "vcvars64.bat not found at $vcvarsPath"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "Setting up MSVC environment using: $vcvarsPath"
|
||||
|
||||
# Get environment after vcvars64.bat
|
||||
$tempFile = [System.IO.Path]::GetTempFileName()
|
||||
$cmd = "`"$vcvarsPath`" && set"
|
||||
Write-Host "Executing: cmd.exe /c $cmd"
|
||||
|
||||
try {
|
||||
cmd.exe /c $cmd | Out-File -FilePath $tempFile -Encoding ASCII
|
||||
|
||||
# Parse and set environment variables
|
||||
$envVarsSet = 0
|
||||
Get-Content $tempFile | ForEach-Object {
|
||||
if ($_ -match '^([^=]+)=(.*)$') {
|
||||
$name = $matches[1]
|
||||
$value = $matches[2]
|
||||
|
||||
# Skip problematic variables
|
||||
if ($name -notmatch '^(TEMP|TMP|RANDOM)$') {
|
||||
[Environment]::SetEnvironmentVariable($name, $value, 'Process')
|
||||
echo "$name=$value" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
$envVarsSet++
|
||||
}
|
||||
}
|
||||
}
|
||||
Write-Host "Set $envVarsSet environment variables from vcvars64.bat"
|
||||
|
||||
Remove-Item $tempFile -ErrorAction SilentlyContinue
|
||||
}
|
||||
catch {
|
||||
Write-Error "Failed to execute vcvars64.bat: $_"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Setup CUDA paths
|
||||
$cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.3"
|
||||
$msysPath = "C:\msys64\mingw64\bin;C:\msys64\usr\bin"
|
||||
|
||||
Write-Host "Setting CUDA paths..."
|
||||
echo "CUDA_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "CUDNN_ROOT_DIR=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# Update PATH with MSYS2 and CUDA
|
||||
$newPath = "$msysPath;$cudaPath\bin;$cudaPath\libnvvp;"
|
||||
Write-Host "Adding to PATH: $newPath"
|
||||
echo $newPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
|
||||
- name: Run cuda build
|
||||
shell: cmd
|
||||
env:
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
PUBLISH_TO: central
|
||||
LIBND4J_HOME_SUFFIX: cuda
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
|
||||
run: |
|
||||
echo "=== CUDA Build Started ==="
|
||||
echo "Libnd4j build threads: ${{ matrix.build_threads }}"
|
||||
echo "Deploy to release staging: %PERFORM_RELEASE%"
|
||||
echo "Release version: %RELEASE_VERSION%"
|
||||
echo "Snapshot version: %SNAPSHOT_VERSION%"
|
||||
echo "Debug enabled: ${{ matrix.debug_enabled }}"
|
||||
echo "Libnd4j URL input: %LIBND4J_FILE_NAME%"
|
||||
echo "Maven flags input: ${{ matrix.mvn_flags }}"
|
||||
echo "Server id: ${{ matrix.server_id }}"
|
||||
echo "Release repo id: %RELEASE_REPO_ID%"
|
||||
echo "CUDA Path from env: %CUDA_PATH%"
|
||||
echo "CUDNN Root Dir from env: %CUDNN_ROOT_DIR%"
|
||||
echo "=== Environment Check ==="
|
||||
|
||||
echo "Verifying compiler availability..."
|
||||
where cl.exe >nul 2>&1 || (
|
||||
echo "ERROR: cl.exe not found in PATH"
|
||||
echo "Current PATH: %PATH%"
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
where nvcc.exe >nul 2>&1 || (
|
||||
echo "ERROR: nvcc.exe not found in PATH"
|
||||
echo "Current PATH: %PATH%"
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo "Compiler verification successful:"
|
||||
echo "cl.exe location:"
|
||||
where cl.exe
|
||||
cl.exe 2>&1 | findstr "Microsoft"
|
||||
|
||||
echo "nvcc.exe location:"
|
||||
where nvcc.exe
|
||||
nvcc.exe --version | findstr "release"
|
||||
|
||||
echo "=== Build Process ==="
|
||||
set MSYSTEM=MINGW64
|
||||
echo "Running cuda build..."
|
||||
echo "Maven command from file:"
|
||||
type "%GITHUB_WORKSPACE%\mvn-command.bat"
|
||||
|
||||
bash ./change-cuda-versions.sh 12.3
|
||||
|
||||
Rem Ensure CUDNN_ROOT_DIR is set if needed by cmake; often CUDA_PATH is sufficient
|
||||
if not defined CUDNN_ROOT_DIR set CUDNN_ROOT_DIR=%CUDA_PATH%
|
||||
|
||||
if "%PERFORM_RELEASE%"=="1" (
|
||||
echo "Running release build..."
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.3 "%HELPER%" "%EXTENSION%"
|
||||
bash "./release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%"
|
||||
) else (
|
||||
echo "Running snapshot build..."
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.3 "%HELPER%" "%EXTENSION%"
|
||||
call "%GITHUB_WORKSPACE%\mvn-command.bat"
|
||||
)
|
||||
@@ -0,0 +1,631 @@
|
||||
# Enhanced CUDA 12.6 Windows Build Workflow with Clean Toolchain Management
|
||||
name: CUDA 12.6 Windows Build
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j (controls memory usage)'
|
||||
required: true
|
||||
default: '4'
|
||||
deployToReleaseStaging:
|
||||
description: 'Deploy to release staging'
|
||||
required: false
|
||||
default: '0'
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: '1.0.0-M3'
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: '1.0.0-SNAPSHOT'
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default: ''
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: 'central'
|
||||
mvnFlags:
|
||||
description: 'Extra maven flags'
|
||||
required: false
|
||||
default: ''
|
||||
libnd4jUrl:
|
||||
description: 'Download libnd4j from URL'
|
||||
required: false
|
||||
default: ''
|
||||
runsOn:
|
||||
description: 'System to run on'
|
||||
required: false
|
||||
default: 'windows-2022'
|
||||
debug_enabled:
|
||||
description: 'Enable tmate debugging'
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
jobs:
|
||||
windows-x86_64-cuda-12-6:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ 'cudnn', '' ]
|
||||
include:
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.12.1
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Free Disk Space
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Freeing Disk Space ==="
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
# Clean Windows temp and cache directories
|
||||
$cleanupPaths = @(
|
||||
"$env:TEMP\*",
|
||||
"$env:SystemRoot\Temp\*",
|
||||
"$env:ProgramData\Microsoft\Windows Defender\Scans\History\*",
|
||||
"$env:SystemRoot\SoftwareDistribution\Download\*"
|
||||
)
|
||||
|
||||
foreach ($path in $cleanupPaths) {
|
||||
try {
|
||||
Remove-Item -Path $path -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleaned: $path"
|
||||
} catch {
|
||||
Write-Host "Could not clean: $path"
|
||||
}
|
||||
}
|
||||
|
||||
# Clean package caches
|
||||
if (Get-Command choco -ErrorAction SilentlyContinue) {
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
# Windows component cleanup
|
||||
try {
|
||||
dism.exe /online /Cleanup-Image /StartComponentCleanup /ResetBase
|
||||
} catch {
|
||||
Write-Host "DISM cleanup skipped"
|
||||
}
|
||||
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- name: Clean Install Visual Studio Build Tools
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Visual Studio Build Tools Setup ==="
|
||||
|
||||
# Function to completely remove VS installations
|
||||
function Remove-VSInstallation {
|
||||
param([string]$Path)
|
||||
if (Test-Path $Path) {
|
||||
Write-Host "Removing VS installation: $Path"
|
||||
Remove-Item -Path $Path -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
}
|
||||
|
||||
# Stop VS processes
|
||||
Get-Process -Name "*vs_*", "vs_installer", "VSIXInstaller", "*devenv*", "*MSBuild*" -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 5
|
||||
|
||||
# Remove all VS installations
|
||||
$vsPaths = @(
|
||||
"${env:ProgramFiles}\Microsoft Visual Studio",
|
||||
"${env:ProgramFiles(x86)}\Microsoft Visual Studio"
|
||||
)
|
||||
|
||||
foreach ($vsPath in $vsPaths) {
|
||||
if (Test-Path $vsPath) {
|
||||
Remove-VSInstallation -Path $vsPath
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "=== Installing Fresh VS Build Tools 2022 (17.8 LTSC) ==="
|
||||
|
||||
# Download VS Build Tools installer
|
||||
$installerUrl = "https://aka.ms/vs/17/release.ltsc.17.8/vs_buildtools.exe"
|
||||
$installerPath = "$env:TEMP\vs_buildtools.exe"
|
||||
|
||||
try {
|
||||
Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing -TimeoutSec 600
|
||||
Write-Host "Downloaded VS installer: $installerPath"
|
||||
} catch {
|
||||
Write-Error "Failed to download VS installer: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Install with Windows SDK 10.0.22621 (compatible with CUDA 12.6)
|
||||
$installArgs = @(
|
||||
"--quiet", "--wait", "--norestart", "--nocache",
|
||||
"--installPath", "`"C:\BuildTools`"",
|
||||
"--add", "Microsoft.VisualStudio.Workload.VCTools",
|
||||
"--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
|
||||
"--add", "Microsoft.VisualStudio.Component.VC.CMake.Project",
|
||||
"--add", "Microsoft.VisualStudio.Component.Windows10SDK.22621",
|
||||
"--add", "Microsoft.VisualStudio.Component.VC.ATL",
|
||||
"--includeRecommended"
|
||||
)
|
||||
|
||||
Write-Host "Installing VS Build Tools..."
|
||||
$process = Start-Process -FilePath $installerPath -ArgumentList $installArgs -Wait -PassThru -NoNewWindow
|
||||
|
||||
if ($process.ExitCode -ne 0) {
|
||||
Write-Error "VS Build Tools installation failed with exit code: $($process.ExitCode)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Verify installation
|
||||
$vsPath = "C:\BuildTools"
|
||||
$vcvarsPath = "$vsPath\VC\Auxiliary\Build\vcvars64.bat"
|
||||
$msvcPath = "$vsPath\VC\Tools\MSVC"
|
||||
|
||||
if (-not (Test-Path $vcvarsPath)) {
|
||||
Write-Error "VS Build Tools installation verification failed - vcvars64.bat not found"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if (-not (Test-Path $msvcPath)) {
|
||||
Write-Error "VS Build Tools installation verification failed - MSVC tools not found"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Get MSVC version
|
||||
$msvcVersions = Get-ChildItem $msvcPath -Directory | Sort-Object Name -Descending
|
||||
if ($msvcVersions.Count -eq 0) {
|
||||
Write-Error "No MSVC versions found"
|
||||
exit 1
|
||||
}
|
||||
|
||||
$latestMSVC = $msvcVersions[0]
|
||||
Write-Host "SUCCESS: VS Build Tools installed successfully"
|
||||
Write-Host " Installation Path: $vsPath"
|
||||
Write-Host " MSVC Version: $($latestMSVC.Name)"
|
||||
|
||||
# Export for later steps
|
||||
echo "VS_INSTALLATION_PATH=$vsPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "MSVC_VERSION=$($latestMSVC.Name)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "MSVC_ROOT=$($latestMSVC.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
- name: Install MSYS2
|
||||
uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: MINGW64
|
||||
update: true
|
||||
install: >-
|
||||
base-devel git tar pkg-config unzip p7zip zip
|
||||
mingw-w64-x86_64-cmake mingw-w64-x86_64-make
|
||||
mingw-w64-x86_64-toolchain mingw-w64-x86_64-gcc
|
||||
|
||||
- name: Cache CUDA Installation
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda
|
||||
with:
|
||||
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6
|
||||
key: cuda-12.6-update1-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
cuda-12.6-update1-${{ runner.os }}-
|
||||
|
||||
- name: Install CUDA 12.6 Update 1
|
||||
if: steps.cache-cuda.outputs.cache-hit != 'true'
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Installing CUDA 12.6 Update 1 ==="
|
||||
|
||||
# Use network installer but with fixes for the -522190823 error
|
||||
$cudaUrl = "https://developer.download.nvidia.com/compute/cuda/12.6.3/network_installers/cuda_12.6.3_windows_network.exe"
|
||||
$cudaInstaller = "$env:TEMP\cuda_network_installer.exe"
|
||||
|
||||
Write-Host "Downloading CUDA network installer..."
|
||||
try {
|
||||
$webClient = New-Object System.Net.WebClient
|
||||
$webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||
$webClient.DownloadFile($cudaUrl, $cudaInstaller)
|
||||
$webClient.Dispose()
|
||||
|
||||
if (-not (Test-Path $cudaInstaller) -or (Get-Item $cudaInstaller).Length -lt 10MB) {
|
||||
throw "Downloaded file is missing or too small"
|
||||
}
|
||||
|
||||
Write-Host "Downloaded CUDA installer: $cudaInstaller ($(([math]::Round((Get-Item $cudaInstaller).Length / 1MB, 2))) MB)"
|
||||
} catch {
|
||||
Write-Error "Failed to download CUDA installer: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Create a temporary directory for CUDA installer logs and temp files
|
||||
$cudaTempDir = "$env:TEMP\cuda_install_temp"
|
||||
if (Test-Path $cudaTempDir) {
|
||||
Remove-Item -Path $cudaTempDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
New-Item -Path $cudaTempDir -ItemType Directory -Force | Out-Null
|
||||
|
||||
# Set environment variables to control installer behavior
|
||||
$env:CUDA_SETUP_ARGS = "--silent"
|
||||
$env:NVTOOLSEXT_INSTALL_DIR = "$cudaTempDir"
|
||||
|
||||
# Essential CUDA components only for faster installation
|
||||
$cudaComponents = @(
|
||||
"nvcc_12.6",
|
||||
"cudart_12.6",
|
||||
"cublas_12.6",
|
||||
"cublas_dev_12.6",
|
||||
"cufft_12.6",
|
||||
"cufft_dev_12.6",
|
||||
"curand_12.6",
|
||||
"curand_dev_12.6",
|
||||
"cusolver_12.6",
|
||||
"cusolver_dev_12.6",
|
||||
"cusparse_12.6",
|
||||
"cusparse_dev_12.6",
|
||||
"thrust_12.6",
|
||||
"nvrtc_12.6",
|
||||
"nvrtc_dev_12.6"
|
||||
)
|
||||
|
||||
# Build installation arguments
|
||||
$installArgs = @("-s") + $cudaComponents
|
||||
|
||||
Write-Host "Installing CUDA components: $($cudaComponents -join ', ')"
|
||||
Write-Host "Installation command: $cudaInstaller $($installArgs -join ' ')"
|
||||
|
||||
try {
|
||||
# Set working directory to temp folder to avoid permission issues
|
||||
Set-Location $cudaTempDir
|
||||
|
||||
# Run installation with timeout and proper error handling
|
||||
$installProcess = Start-Process -FilePath $cudaInstaller -ArgumentList $installArgs -Wait -PassThru -NoNewWindow -WorkingDirectory $cudaTempDir
|
||||
|
||||
Write-Host "CUDA installer completed with exit code: $($installProcess.ExitCode)"
|
||||
|
||||
# CUDA installer exit codes:
|
||||
# 0 = success
|
||||
# 1 = success with reboot required
|
||||
# 2 = already installed (newer version)
|
||||
# Other negative values = various errors
|
||||
if ($installProcess.ExitCode -notin @(0, 1, 2)) {
|
||||
throw "Installation failed with exit code: $($installProcess.ExitCode)"
|
||||
}
|
||||
|
||||
} catch {
|
||||
Write-Error "CUDA installation failed: $($_.Exception.Message)"
|
||||
|
||||
# Check for installation logs
|
||||
$logFiles = Get-ChildItem -Path $cudaTempDir -Filter "*.log" -ErrorAction SilentlyContinue
|
||||
if ($logFiles) {
|
||||
Write-Host "Installation logs found:"
|
||||
foreach ($logFile in $logFiles) {
|
||||
Write-Host "=== $($logFile.Name) ==="
|
||||
Get-Content $logFile.FullName | Select-Object -Last 20
|
||||
}
|
||||
}
|
||||
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "SUCCESS: CUDA installation completed"
|
||||
|
||||
# Verify installation
|
||||
$cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6"
|
||||
$nvccPath = "$cudaPath\bin\nvcc.exe"
|
||||
|
||||
# Wait for filesystem to settle
|
||||
Start-Sleep -Seconds 10
|
||||
|
||||
if (-not (Test-Path $nvccPath)) {
|
||||
Write-Host "Primary CUDA path not found, checking for alternative locations..."
|
||||
|
||||
# Check for any CUDA installation
|
||||
$cudaRoot = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA"
|
||||
if (Test-Path $cudaRoot) {
|
||||
Write-Host "Available CUDA installations:"
|
||||
$cudaVersions = Get-ChildItem $cudaRoot -Directory
|
||||
foreach ($version in $cudaVersions) {
|
||||
Write-Host " Found: $($version.Name)"
|
||||
$altNvccPath = "$($version.FullName)\bin\nvcc.exe"
|
||||
if (Test-Path $altNvccPath) {
|
||||
Write-Host " SUCCESS: nvcc.exe found in $($version.Name)"
|
||||
# Update our expected paths to the found version
|
||||
$cudaPath = $version.FullName
|
||||
$nvccPath = $altNvccPath
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Final check
|
||||
if (-not (Test-Path $nvccPath)) {
|
||||
Write-Error "CUDA installation verification failed - nvcc.exe not found anywhere"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Test nvcc functionality
|
||||
try {
|
||||
$nvccResult = & $nvccPath --version 2>&1
|
||||
Write-Host "SUCCESS: CUDA installation verified - nvcc version:"
|
||||
Write-Host $nvccResult
|
||||
|
||||
# Update environment variable if we found CUDA in a different location
|
||||
if ($cudaPath -ne "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6") {
|
||||
Write-Host "Updating CUDA_PATH to actual installation location: $cudaPath"
|
||||
echo "CUDA_ACTUAL_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
}
|
||||
|
||||
} catch {
|
||||
Write-Error "nvcc found but functionality test failed: $($_.Exception.Message)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Clean up installation files
|
||||
try {
|
||||
Set-Location $env:TEMP
|
||||
Remove-Item -Path $cudaTempDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path $cudaInstaller -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleaned up installation files"
|
||||
} catch {
|
||||
Write-Host "Could not clean up some installation files (not critical)"
|
||||
}
|
||||
|
||||
Write-Host "SUCCESS: CUDA 12.6 Update 1 network installation complete"
|
||||
|
||||
- name: Setup Java
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
cache: 'maven'
|
||||
|
||||
- name: Configure Build Environment
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Configuring Build Environment ==="
|
||||
|
||||
# Setup VS environment using clean vcvars64.bat
|
||||
$vsPath = $env:VS_INSTALLATION_PATH
|
||||
$vcvarsPath = "$vsPath\VC\Auxiliary\Build\vcvars64.bat"
|
||||
|
||||
Write-Host "Initializing MSVC environment from: $vcvarsPath"
|
||||
|
||||
# Create temporary batch file to capture environment
|
||||
$tempBatch = [System.IO.Path]::GetTempFileName() + ".bat"
|
||||
$tempEnv = [System.IO.Path]::GetTempFileName() + ".txt"
|
||||
|
||||
@"
|
||||
@echo off
|
||||
call "$vcvarsPath" >nul 2>&1
|
||||
if %ERRORLEVEL% NEQ 0 exit /b 1
|
||||
set > "$tempEnv"
|
||||
"@ | Out-File -FilePath $tempBatch -Encoding ASCII
|
||||
|
||||
$result = Start-Process -FilePath "cmd.exe" -ArgumentList "/c", "`"$tempBatch`"" -Wait -PassThru -NoNewWindow
|
||||
|
||||
if ($result.ExitCode -ne 0 -or -not (Test-Path $tempEnv)) {
|
||||
Write-Error "Failed to initialize MSVC environment"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse and set environment variables
|
||||
$envCount = 0
|
||||
Get-Content $tempEnv | ForEach-Object {
|
||||
if ($_ -match '^([^=]+)=(.*)$') {
|
||||
$name = $matches[1]
|
||||
$value = $matches[2]
|
||||
# Skip problematic variables
|
||||
if ($name -notmatch '^(TEMP|TMP|RANDOM|PROMPT|PATHEXT)$') {
|
||||
try {
|
||||
[Environment]::SetEnvironmentVariable($name, $value, 'Process')
|
||||
echo "$name=$value" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
$envCount++
|
||||
} catch {
|
||||
# Silently continue on errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Set $envCount environment variables from vcvars64.bat"
|
||||
|
||||
# Clean up
|
||||
Remove-Item $tempBatch, $tempEnv -ErrorAction SilentlyContinue
|
||||
|
||||
# Setup CUDA environment
|
||||
$cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6"
|
||||
|
||||
# Check if CUDA was installed in a different location
|
||||
if ($env:CUDA_ACTUAL_PATH) {
|
||||
$cudaPath = $env:CUDA_ACTUAL_PATH
|
||||
Write-Host "Using actual CUDA installation path: $cudaPath"
|
||||
}
|
||||
|
||||
echo "CUDA_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
echo "CUDA_HOME=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# Setup cuDNN only if needed
|
||||
if ("${{ matrix.helper }}" -eq "cudnn") {
|
||||
echo "CUDNN_ROOT_DIR=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
Write-Host "SUCCESS: cuDNN environment configured (CUDNN_ROOT_DIR=$cudaPath)"
|
||||
} else {
|
||||
Write-Host "INFO: Skipping cuDNN configuration (helper: ${{ matrix.helper }})"
|
||||
}
|
||||
|
||||
# Setup clean PATH
|
||||
$cleanPath = @(
|
||||
"C:\msys64\mingw64\bin",
|
||||
"C:\msys64\usr\bin",
|
||||
"$cudaPath\bin",
|
||||
"$cudaPath\libnvvp",
|
||||
"$env:MSVC_ROOT\bin\Hostx64\x64",
|
||||
"$env:SystemRoot\system32",
|
||||
"$env:SystemRoot"
|
||||
) -join ";"
|
||||
|
||||
echo $cleanPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
|
||||
Write-Host "SUCCESS: Build environment configured"
|
||||
Write-Host " VS Path: $vsPath"
|
||||
Write-Host " MSVC Version: $env:MSVC_VERSION"
|
||||
Write-Host " CUDA Path: $cudaPath"
|
||||
Write-Host " Helper: ${{ matrix.helper }}"
|
||||
|
||||
- name: Verify Build Tools
|
||||
shell: cmd
|
||||
run: |
|
||||
echo "=== Verifying Build Tools ==="
|
||||
|
||||
echo "Checking cl.exe..."
|
||||
where cl.exe
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo "ERROR: cl.exe not found in PATH"
|
||||
exit /b 1
|
||||
)
|
||||
cl.exe 2>&1 | findstr "Microsoft"
|
||||
|
||||
echo "Checking nvcc.exe..."
|
||||
where nvcc.exe
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo "ERROR: nvcc.exe not found in PATH"
|
||||
exit /b 1
|
||||
)
|
||||
nvcc.exe --version | findstr "release"
|
||||
|
||||
echo "Environment Check:"
|
||||
echo "CUDA_PATH: %CUDA_PATH%"
|
||||
echo "MSVC_ROOT: %MSVC_ROOT%"
|
||||
echo "Helper: ${{ matrix.helper }}"
|
||||
|
||||
if "${{ matrix.helper }}"=="cudnn" (
|
||||
echo "CUDNN_ROOT_DIR: %CUDNN_ROOT_DIR%"
|
||||
if not defined CUDNN_ROOT_DIR (
|
||||
echo "WARNING: CUDNN_ROOT_DIR not set but cuDNN helper enabled"
|
||||
)
|
||||
)
|
||||
|
||||
echo "SUCCESS: All build tools verified"
|
||||
|
||||
- name: Configure Maven Command
|
||||
shell: powershell
|
||||
run: |
|
||||
Write-Host "=== Configuring Maven Command ==="
|
||||
|
||||
# Base modules
|
||||
if ("${{ github.event.inputs.libnd4jUrl }}" -ne "") {
|
||||
$modules = ":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6"
|
||||
} elseif ("${{ matrix.helper }}" -ne "") {
|
||||
$modules = ":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6,libnd4j"
|
||||
} else {
|
||||
$modules = ":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6,libnd4j,:nd4j-cuda-12.6-platform"
|
||||
}
|
||||
|
||||
# Base command
|
||||
$baseCommand = "mvn ${{ github.event.inputs.mvnFlags }} -Pcuda -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda --also-make -pl $modules"
|
||||
$baseCommand += " -Dlibnd4j.compute=`"8.6 9.0`" -Dlibnd4j.cpu.compile.skip=true"
|
||||
$baseCommand += " -Dlibnd4j.buildthreads=${{ matrix.build_threads }}"
|
||||
$baseCommand += " -Djavacpp.platform=windows-x86_64 -Dlibnd4j.platform=windows-x86_64"
|
||||
$baseCommand += " -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3"
|
||||
$baseCommand += " -Possrh deploy -DskipTests"
|
||||
|
||||
# Helper-specific extensions
|
||||
if ("${{ matrix.helper }}" -ne "") {
|
||||
$helperExt = " -Dlibnd4j.classifier=windows-x86_64-cuda-12.6-${{ matrix.helper }}"
|
||||
$helperExt += " -Dlibnd4j.extension=${{ matrix.helper }}"
|
||||
$helperExt += " -Djavacpp.platform.extension=-${{ matrix.helper }}"
|
||||
$helperExt += " -Dlibnd4j.helper=${{ matrix.helper }}"
|
||||
$libnd4jFileName = "windows-cuda-12.6-${{ matrix.helper }}"
|
||||
} else {
|
||||
$helperExt = " -Dlibnd4j.classifier=windows-x86_64-cuda-12.6"
|
||||
$libnd4jFileName = "windows-cuda-12.6"
|
||||
}
|
||||
|
||||
$fullCommand = $baseCommand + $helperExt
|
||||
|
||||
# Save command
|
||||
echo $fullCommand | Out-File -FilePath "$env:GITHUB_WORKSPACE\mvn-command.bat" -Encoding utf8
|
||||
echo "MAVEN_COMMAND=$fullCommand" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
if ("${{ github.event.inputs.libnd4jUrl }}" -ne "") {
|
||||
echo "LIBND4J_FILE_NAME=${{ github.event.inputs.libnd4jUrl }}/$libnd4jFileName" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
}
|
||||
|
||||
Write-Host "SUCCESS: Maven command configured for helper: ${{ matrix.helper }}"
|
||||
Write-Host "Command: $fullCommand"
|
||||
|
||||
- name: Run CUDA Build
|
||||
shell: cmd
|
||||
env:
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
|
||||
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
|
||||
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
run: |
|
||||
echo "=== Starting CUDA Build ==="
|
||||
echo "Build threads: ${{ matrix.build_threads }}"
|
||||
echo "Helper: ${{ matrix.helper }}"
|
||||
echo "Deploy to release: %PERFORM_RELEASE%"
|
||||
echo "Release version: %RELEASE_VERSION%"
|
||||
echo "Snapshot version: %SNAPSHOT_VERSION%"
|
||||
|
||||
rem Verify environment one more time
|
||||
echo "CUDA_PATH: %CUDA_PATH%"
|
||||
echo "MSVC_ROOT: %MSVC_ROOT%"
|
||||
if "${{ matrix.helper }}"=="cudnn" (
|
||||
echo "CUDNN_ROOT_DIR: %CUDNN_ROOT_DIR%"
|
||||
)
|
||||
|
||||
rem Change CUDA versions
|
||||
bash ./change-cuda-versions.sh 12.6
|
||||
|
||||
if "%PERFORM_RELEASE%"=="1" (
|
||||
echo "Running release build..."
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.6 "${{ matrix.helper }}" ""
|
||||
bash "./release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%"
|
||||
) else (
|
||||
echo "Running snapshot build..."
|
||||
if "%LIBND4J_FILE_NAME%" NEQ "" (
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.6 "${{ matrix.helper }}" ""
|
||||
)
|
||||
call "%GITHUB_WORKSPACE%\mvn-command.bat"
|
||||
)
|
||||
|
||||
echo "SUCCESS: Build completed"
|
||||
|
||||
- name: Upload Build Artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: build-logs-${{ matrix.helper || 'no-helper' }}
|
||||
path: |
|
||||
target/
|
||||
libnd4j/blasbuild/
|
||||
**/*.log
|
||||
retention-days: 7
|
||||
|
||||
- name: Setup tmate session for debugging
|
||||
if: ${{ github.event.inputs.debug_enabled == 'true' && failure() }}
|
||||
uses: mxschmitt/action-tmate@v3
|
||||
timeout-minutes: 30
|
||||
@@ -0,0 +1,419 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 2
|
||||
|
||||
deployToReleaseStaging:
|
||||
description: 'Whether to deploy to release staging or not.'
|
||||
required: false
|
||||
default: 0
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M3
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
serverId:
|
||||
description: 'Server id to publish to'
|
||||
required: false
|
||||
default: central
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
libnd4jUrl:
|
||||
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
|
||||
required: false
|
||||
default:
|
||||
|
||||
runsOn:
|
||||
description: 'OS to run on'
|
||||
required: false
|
||||
default: windows-2022
|
||||
|
||||
debug_enabled:
|
||||
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
windows-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ onednn,"" ]
|
||||
extension: [ avx2,avx512,"" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
- libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
|
||||
experimental: true
|
||||
name: Whether to deploy to release staging or not
|
||||
- release_version: ${{ github.event.inputs.releaseVersion }}
|
||||
experimental: true
|
||||
name: Release version
|
||||
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
|
||||
experimental: true
|
||||
name: Snapshot version
|
||||
- server_id: ${{ github.event.inputs.serverId }}
|
||||
experimental: true
|
||||
name: Server id
|
||||
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
|
||||
experimental: true
|
||||
name: The release repository to run on
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
- build_threads: ${{ github.event.inputs.buildThreads }}
|
||||
experimental: true
|
||||
name: The number of threads to build libnd4j with
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space (Windows)
|
||||
shell: powershell
|
||||
run: |
|
||||
# Show initial disk space
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
# The system must be told to stop managing the pagefile size automatically.
|
||||
wmic computersystem set AutomaticManagedPagefile=False
|
||||
|
||||
# Find and delete any existing pagefile configuration to ensure ours is the only one.
|
||||
$currentPagefile = Get-WmiObject -Query "SELECT * FROM Win32_PageFileSetting WHERE Name='C:\\pagefile.sys'"
|
||||
if ($currentPagefile) {
|
||||
$currentPagefile.Delete()
|
||||
}
|
||||
|
||||
# Create a new pagefile with a static size of 12 GB (12288 MB).
|
||||
wmic pagefileset create name="C:\\pagefile.sys"
|
||||
wmic pagefileset where "name='C:\\pagefile.sys'" set InitialSize=12288, MaximumSize=12288
|
||||
|
||||
echo "Pagefile configured to 12 GB for the duration of this job."
|
||||
# Remove Windows Defender scan history
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed Windows Defender scan history"
|
||||
|
||||
# Clear Windows temp folders
|
||||
Remove-Item -Path "$env:SystemRoot\Temp\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows temp folders"
|
||||
|
||||
# Clear Windows Update cache safely (without stopping/starting service)
|
||||
try {
|
||||
Remove-Item -Path "$env:SystemRoot\SoftwareDistribution\Download\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows Update download cache"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Could not clear Windows Update cache. Continuing..."
|
||||
}
|
||||
|
||||
# Clean package manager caches
|
||||
if (Test-Path -Path "C:\npm\cache") {
|
||||
Remove-Item -Path "C:\npm\cache\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared NPM cache"
|
||||
}
|
||||
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Chocolatey cache"
|
||||
|
||||
# Remove Docker images if Docker is installed
|
||||
try {
|
||||
if (Get-Command "docker" -ErrorAction SilentlyContinue) {
|
||||
docker image prune -a -f
|
||||
docker container prune -f
|
||||
docker volume prune -f
|
||||
Write-Host "Pruned Docker resources"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to prune Docker resources. Continuing..."
|
||||
}
|
||||
|
||||
# Remove .NET SDK/Runtime backup folders
|
||||
if (Test-Path -Path "$env:ProgramData\Microsoft\.NET\*.backup") {
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\.NET\*.backup" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed .NET backup folders"
|
||||
}
|
||||
|
||||
# Clear Azure artifacts cache
|
||||
if (Test-Path -Path "$env:LOCALAPPDATA\Microsoft\Azure\*") {
|
||||
Remove-Item -Path "$env:LOCALAPPDATA\Microsoft\Azure\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Azure artifacts cache"
|
||||
}
|
||||
|
||||
# Optimize Windows Component Store
|
||||
try {
|
||||
Start-Process -FilePath "dism.exe" -ArgumentList "/online /Cleanup-Image /StartComponentCleanup" -NoNewWindow -Wait
|
||||
Write-Host "Optimized Windows Component Store"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to optimize Windows Component Store. Continuing..."
|
||||
}
|
||||
|
||||
# Show final disk space
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- name: Install environment
|
||||
shell: cmd
|
||||
env:
|
||||
GITHUB_EVENT_HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
|
||||
run: |
|
||||
cd /d %USERPROFILE%
|
||||
|
||||
echo Installing MSYS2
|
||||
C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm pkg-config"
|
||||
C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm base-devel git tar unzip p7zip zip autoconf autoconf-archive automake libtool make patch gnupg"
|
||||
C:\msys64\usr\bin\bash -lc "pacman -S --needed --noconfirm mingw-w64-x86_64-nasm mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-i686-gcc mingw-w64-x86_64-gcc-fortran mingw-w64-i686-gcc-fortran mingw-w64-x86_64-libwinpthread-git mingw-w64-i686-libwinpthread-git mingw-w64-x86_64-SDL2 mingw-w64-i686-SDL2 mingw-w64-x86_64-ragel mingw-w64-x86_64-vulkan-headers mingw-w64-i686-vulkan-headers mingw-w64-x86_64-vulkan-loader mingw-w64-i686-vulkan-loader"
|
||||
set "PATH=C:\hostedtoolcache\windows\Python\3.10.11\x64;C:\msys64\usr\bin;%PATH%"
|
||||
|
||||
C:\msys64\usr\bin\bash -lc "pacman -Q"
|
||||
|
||||
echo Installing Windows SDK 8.1
|
||||
curl -Lo sdksetup.exe https://go.microsoft.com/fwlink/p/?LinkId=323507
|
||||
sdksetup.exe /features OptionId.WindowsDesktopSoftwareDevelopmentKit OptionId.NetFxSoftwareDevelopmentKit /quiet
|
||||
|
||||
echo Removing broken stuff from WSL, MSYS2, etc
|
||||
rm "C:/msys64/usr/bin/curl.exe" "C:/msys64/mingw32/bin/curl.exe" "C:/msys64/mingw64/bin/curl.exe"
|
||||
rm "C:/WINDOWS/system32/bash.EXE" "C:/msys64/usr/bin/link.exe" "C:/msys64/usr/bin/timeout.exe" "C:/msys64/usr/bin/python.exe" "C:/msys64/usr/bin/python3.exe"
|
||||
rm "C:/ProgramData/chocolatey/bin/gfortran.exe" "C:/msys64/mingw32/bin/gfortran.exe" "C:/msys64/mingw32/bin/python.exe" "C:/msys64/mingw32/bin/python3.exe"
|
||||
rm "C:/Strawberry/c/bin/gfortran.exe" "C:/msys64/mingw64/bin/gfortran.exe" "C:/msys64/mingw64/bin/python.exe" "C:/msys64/mingw64/bin/python3.exe"
|
||||
rm "C:/msys64/mingw32/bin/clang-cl.exe" "C:/msys64/mingw64/bin/clang-cl.exe" "C:/msys64/mingw32/bin/cmake.exe" "C:/msys64/mingw64/bin/cmake.exe"
|
||||
rm "C:/Strawberry/c/lib/libz.a" "C:/Strawberry/c/lib/libzlib.a" "C:/Strawberry/c/lib/libzdll.a" "C:/Strawberry/c/bin/cmake.exe"
|
||||
|
||||
curl -LO https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz || curl -LO https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
|
||||
bash -c "tar -xzf apache-maven-3.6.3-bin.tar.gz -C 'C:/Program Files/'"
|
||||
|
||||
python -m pip install gdown || python -m pip install gdown
|
||||
|
||||
echo Installing ccache
|
||||
curl -LO https://github.com/ccache/ccache/releases/download/v4.6/ccache-4.6-windows-64.zip
|
||||
unzip -j ccache-4.6-windows-64.zip -d C:/msys64/usr/bin/
|
||||
mkdir ccache
|
||||
echo max_size = 3.0G > ccache\ccache.conf
|
||||
echo hash_dir = false >> ccache\ccache.conf
|
||||
echo sloppiness = file_macro,include_file_ctime,include_file_mtime,pch_defines,time_macros >> ccache\ccache.conf
|
||||
|
||||
echo Installing an older less buggy version of GCC
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-ada-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-objc-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-libs-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-fortran-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gcc-libgfortran-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-binutils-2.42-2-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-crt-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-headers-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-libmangle-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-libwinpthread-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-tools-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-winpthreads-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-winstorecompat-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-vulkan-headers-1.3.280.0-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-vulkan-loader-1.3.280.0-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-SDL2-2.30.12-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-python-3.11.9-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-ada-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-objc-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-libs-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-fortran-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-gcc-libgfortran-13.2.0-6-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-binutils-2.42-2-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-crt-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-headers-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libmangle-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-libwinpthread-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-tools-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-winpthreads-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-winstorecompat-git-11.0.0.r750.g05598db99-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-vulkan-headers-1.3.280.0-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-vulkan-loader-1.3.280.0-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-SDL2-2.30.12-1-any.pkg.tar.zst
|
||||
curl -LO http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-python-3.11.9-1-any.pkg.tar.zst
|
||||
bash -c "pacman -U --noconfirm *.pkg.tar.zst"
|
||||
|
||||
- name: Set mvn build command based on matrix
|
||||
shell: powershell
|
||||
run: |
|
||||
if ( "${{ matrix.libnd4j_file_download }}" -ne "" ) {
|
||||
$modules=" :nd4j-native-preset,:nd4j-native"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$modules=":nd4j-native-preset,:nd4j-native,libnd4j"
|
||||
} elseif ( "${{ matrix.extension }}" -ne "" ) {
|
||||
$modules=":nd4j-native-preset,:nd4j-native,libnd4j"
|
||||
} else {
|
||||
$modules=":nd4j-native-preset,:nd4j-native,libnd4j,:nd4j-native-platform"
|
||||
}
|
||||
|
||||
$command="mvn ${{ matrix.mvn_ext }} -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.platform=windows-x86_64 deploy -DskipTests --also-make"
|
||||
if ( "${{ matrix.helper }}" -ne "" -And "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.platform=windows-x86_64 deploy -DskipTests"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.platform=windows-x86_64 deploy -DskipTests"
|
||||
} elseif ( "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.extension }}"
|
||||
} else {
|
||||
$mvn_ext=" -Dlibnd4j.classifier=windows-x86_64"
|
||||
}
|
||||
|
||||
if ( "${{ matrix.libnd4j_file_download }}" -ne "") {
|
||||
echo "Adding libnd4j download"
|
||||
$libnd4j_url_to_write = -join("LIBND4J_FILE_NAME=","$(${{ matrix.libnd4j_file_download }}/$libnd4j_download_file_url)");
|
||||
echo $libnd4j_url_to_write | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
|
||||
}
|
||||
|
||||
$command2 = -join("$($command)","$($mvn_ext)");
|
||||
$to_write = -join("COMMAND=","$($command2)");
|
||||
echo "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to $($command2)"
|
||||
echo $command2 | Out-File -FilePath "$env:GITHUB_WORKSPACE/mvn-command.bat" -Encoding utf8 -Append
|
||||
echo $to_write | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: konduitai/setup-java@main
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: ${{ github.event.inputs.serverId }}
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
|
||||
- name: Setup windows path
|
||||
shell: powershell
|
||||
run: echo "C:\msys64\mingw64\bin;C:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: powershell
|
||||
run: |
|
||||
mkdir "%GITHUB_WORKSPACE%/openblas_home"
|
||||
cd "%GITHUB_WORKSPACE%/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/openblas-0.3.28-1.5.11-windows-x86_64.jar
|
||||
unzip openblas-0.3.28-1.5.11-windows-x86_64.jar
|
||||
cd ..
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/windows-x86_64/" | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
|
||||
if: ${{ matrix.libnd4j_file_download != '' }}
|
||||
|
||||
- name: Run windows cpu build
|
||||
shell: cmd
|
||||
run: |
|
||||
call "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64
|
||||
call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64
|
||||
set MSYSTEM=MINGW64
|
||||
|
||||
set "CCACHE_DIR=%USERPROFILE%\ccache"
|
||||
set "PATH=C:\hostedtoolcache\windows\Python\3.10.11\x64;C:\msys64\%MSYSTEM%\bin;C:\msys64\usr\bin;%ProgramFiles%\apache-maven-3.6.3\bin;%PATH%"
|
||||
|
||||
where bash
|
||||
where curl
|
||||
where git
|
||||
where cl
|
||||
where gcc
|
||||
where cmake
|
||||
where mvn
|
||||
where python
|
||||
where python3
|
||||
where ccache
|
||||
|
||||
bash --version
|
||||
git --version
|
||||
cl
|
||||
gcc --version
|
||||
cmake --version
|
||||
call mvn -version
|
||||
python --version
|
||||
ccache --version -sv
|
||||
df -h
|
||||
wmic pagefile list /format:list
|
||||
|
||||
set MAKEJ=%NUMBER_OF_PROCESSORS%
|
||||
echo Fetching %GITHUB_REPOSITORY%@%GITHUB_SHA%
|
||||
git init
|
||||
git fetch --depth 1 https://github.com/%GITHUB_REPOSITORY% %GITHUB_SHA%
|
||||
git checkout %GITHUB_SHA%
|
||||
git submodule update --init --recursive
|
||||
git submodule foreach --recursive "git reset --hard"
|
||||
|
||||
echo "libnd4j build threads ${{ matrix.build_threads }}"
|
||||
echo "deploy to release staging repo or not ${{ matrix.deploy_to_release_staging }}"
|
||||
echo "release version ${{ matrix.release_version }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "debug enabled ${{ matrix.debug_enabled }}"
|
||||
echo "libnd4j url ${{ matrix.libnd4j_file_download }}"
|
||||
echo "maven flags ${{ matrix.mvn_flags }}"
|
||||
echo "snapshot version ${{ matrix.snapshot_version }}"
|
||||
echo "server id ${{ matrix.server_id }}"
|
||||
echo "release repo id ${{ matrix.release_repo_id }}"
|
||||
|
||||
if "%PERFORM_RELEASE%"=="1" (
|
||||
echo "Running release"
|
||||
bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows x86_64 "${{ matrix.helper }}" "${{ matrix.extension }}"
|
||||
bash "%GITHUB_WORKSPACE%/release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%" "%COMMAND%"
|
||||
) else (
|
||||
if "%PERFORM_RELEASE%"==1 (
|
||||
echo "Running release"
|
||||
bash "%GITHUB_WORKSPACE%/release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%" "%COMMAND%"
|
||||
) else (
|
||||
echo "Running snapshots"
|
||||
call "%GITHUB_WORKSPACE%\mvn-command.bat"
|
||||
)
|
||||
)
|
||||
|
||||
ccache --version -sv
|
||||
df -h
|
||||
wmic pagefile list /format:list
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PUBLISH_TO: central
|
||||
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
|
||||
PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }}
|
||||
RELEASE_VERSION: ${{ matrix.release_version }}
|
||||
SNAPSHOT_VERSION: ${{ matrix.snapshot_version }}
|
||||
RELEASE_REPO_ID: ${{ matrix.release_repo_id }}
|
||||
GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
LIBND4J_FILE_NAME: ${{ matrix.libnd4j_file_download }}
|
||||
|
||||
- name: Clean up
|
||||
shell: cmd
|
||||
run: |
|
||||
cd /d %USERPROFILE%
|
||||
set "PATH=C:\hostedtoolcache\windows\Python\3.10.11\x64;C:\msys64\usr\bin;%PATH%"
|
||||
bash -c "rm -Rf $(find .m2/repository/ -name '*SNAPSHOT*')"
|
||||
@@ -0,0 +1,357 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
|
||||
releaseVersion:
|
||||
description: 'Release version target'
|
||||
required: false
|
||||
default: 1.0.0-M2
|
||||
|
||||
snapshotVersion:
|
||||
description: 'Snapshot version target'
|
||||
required: false
|
||||
default: 1.0.0-SNAPSHOT
|
||||
|
||||
releaseRepoId:
|
||||
description: 'Release repository id'
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
mvnFlags:
|
||||
description: "Extra maven flags (must escape input yourself if used)"
|
||||
required: false
|
||||
default:
|
||||
|
||||
|
||||
jobs:
|
||||
#Note: no -pl here because we publish everything from this branch and use this as the basis for all uploads.
|
||||
linux-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [onednn,""]
|
||||
extension: [avx2,avx512,""]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
|
||||
- runs_on: ubuntu-22.04
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
|
||||
|
||||
|
||||
runs-on: ${{ matrix.runs_on }}
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
- uses: ./.github/actions/set-linux-distro-version
|
||||
- uses: ./.github/actions/update-deps-linux
|
||||
- name: Cache cmake install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cmake
|
||||
with:
|
||||
path: /opt/cmake
|
||||
key: ${{ runner.os }}-cmake
|
||||
restore-keys: ${{ runner.os }}-cmake
|
||||
- name: Cache protobuf install
|
||||
uses: actions/cache@v4
|
||||
id: cache-protobuf
|
||||
with:
|
||||
path: /opt/protobuf
|
||||
key: ${{ runner.os }}-protobuf
|
||||
restore-keys: ${{ runner.os }}-protobuf
|
||||
- uses: ./.github/actions/install-protobuf-linux
|
||||
if: steps.cache-protobuf.outputs.cache-hit != 'true'
|
||||
- uses: ./.github/actions/install-cmake-linux
|
||||
if: steps.cache-cmake.outputs.cache-hit != 'true'
|
||||
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
|
||||
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir "${GITHUB_WORKSPACE}/openblas_home"
|
||||
cd "${GITHUB_WORKSPACE}/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.19-1.5.7/openblas-0.3.19-1.5.7-linux-x86_64.jar
|
||||
unzip openblas-0.3.19-1.5.7-linux-x86_64.jar
|
||||
cd ..
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64" >> "$GITHUB_ENV"
|
||||
cp ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64/libopenblas.so.0 ${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-x86_64/libopenblas.so
|
||||
if: ${{ github.event.inputs.libnd4jUrl != '' }}
|
||||
- name: Download dl4j-test-resources
|
||||
uses: ./.github/actions/download-dl4j-test-resources-linux
|
||||
- name: Build on linux-x86_64
|
||||
shell: bash
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
MAVEN_OPTS: -Xmx2g
|
||||
run: |
|
||||
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
|
||||
export LIBGOMP_PATH=/usr/lib/gcc/x86_64-linux-gnu/5.5.0/libgomp.so
|
||||
if [ -z "${EXTENSION}" ] || [ -n "${EXTENSION}" ]; then
|
||||
export LIBGOMP_PATH=/usr/lib/gcc/x86_64-linux-gnu/7.5.0/libgomp.so
|
||||
echo "Extensions specified. This needs a newer version of gcc."
|
||||
sudo apt-get install gcc-7 g++-7
|
||||
echo "Using newer version of libgomp."
|
||||
ls /usr/bin | grep gcc
|
||||
ls /usr/bin | grep g++
|
||||
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 90
|
||||
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 90
|
||||
gcc --version
|
||||
fi
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dplatform.classifier=linux-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext="-Dplatform.classifier=linux-x86_64-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dplatform.classifier=linux-x86_64-${{matrix.extension}}"
|
||||
else
|
||||
mvn_ext=" -Dplatform.classifier=linux-x86_64"
|
||||
fi
|
||||
cd ${GITHUB_WORKSPACE}/platform-tests && mvn ${mvn_ext} -Djavacpp.platform=linux-x86_64 clean test
|
||||
|
||||
|
||||
windows-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ onednn,"" ]
|
||||
extension: [ avx2,avx512,"" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
|
||||
|
||||
runs-on: windows-2019
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space (Windows)
|
||||
shell: powershell
|
||||
run: |
|
||||
# Show initial disk space
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
# Remove Windows Defender scan history
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed Windows Defender scan history"
|
||||
|
||||
# Clear Windows temp folders
|
||||
Remove-Item -Path "$env:SystemRoot\Temp\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows temp folders"
|
||||
|
||||
# Clear Windows Update cache safely (without stopping/starting service)
|
||||
try {
|
||||
Remove-Item -Path "$env:SystemRoot\SoftwareDistribution\Download\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows Update download cache"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Could not clear Windows Update cache. Continuing..."
|
||||
}
|
||||
|
||||
# Clean package manager caches
|
||||
if (Test-Path -Path "C:\npm\cache") {
|
||||
Remove-Item -Path "C:\npm\cache\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared NPM cache"
|
||||
}
|
||||
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Chocolatey cache"
|
||||
|
||||
# Remove Docker images if Docker is installed
|
||||
try {
|
||||
if (Get-Command "docker" -ErrorAction SilentlyContinue) {
|
||||
docker image prune -a -f
|
||||
docker container prune -f
|
||||
docker volume prune -f
|
||||
Write-Host "Pruned Docker resources"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to prune Docker resources. Continuing..."
|
||||
}
|
||||
|
||||
# Remove .NET SDK/Runtime backup folders
|
||||
if (Test-Path -Path "$env:ProgramData\Microsoft\.NET\*.backup") {
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\.NET\*.backup" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed .NET backup folders"
|
||||
}
|
||||
|
||||
# Clear Azure artifacts cache
|
||||
if (Test-Path -Path "$env:LOCALAPPDATA\Microsoft\Azure\*") {
|
||||
Remove-Item -Path "$env:LOCALAPPDATA\Microsoft\Azure\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Azure artifacts cache"
|
||||
}
|
||||
|
||||
# Optimize Windows Component Store
|
||||
try {
|
||||
Start-Process -FilePath "dism.exe" -ArgumentList "/online /Cleanup-Image /StartComponentCleanup" -NoNewWindow -Wait
|
||||
Write-Host "Optimized Windows Component Store"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to optimize Windows Component Store. Continuing..."
|
||||
}
|
||||
|
||||
# Show final disk space
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- name: Set up Java for publishing to GitHub Packages
|
||||
uses: konduitai/setup-java@main
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
|
||||
|
||||
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: MINGW64
|
||||
update: true
|
||||
install: base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake patch mingw-w64-x86_64-gnupg mingw-w64-x86_64-make --noconfirm mingw-w64-x86_64-cmake mingw-w64-x86_64-nasm mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-fortran mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-x86_64-ragel
|
||||
- name: Setup windows path
|
||||
shell: powershell
|
||||
run: echo "C:\msys64\mingw64\bin;C:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
- name: Setup libnd4j home if a download url is specified
|
||||
shell: powershell
|
||||
run: |
|
||||
mkdir "%GITHUB_WORKSPACE%/openblas_home"
|
||||
cd "%GITHUB_WORKSPACE%/openblas_home"
|
||||
wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.19-1.5.7/openblas-0.3.19-1.5.7-windows-x86_64.jar
|
||||
unzip openblas-0.3.19-1.5.7-windows-x86_64.jar
|
||||
cd ..
|
||||
echo "OPENBLAS_PATH=${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/windows-x86_64/" | Out-File -FilePath "$env:GITHUB_ENV" -Encoding utf8 -Append
|
||||
if: ${{ matrix.libnd4j_file_download != '' }}
|
||||
- name: Download dl4j-test-resources
|
||||
uses: ./.github/actions/download-dl4j-test-resources-windows
|
||||
- name: Run windows cpu build
|
||||
shell: cmd
|
||||
run: |
|
||||
if ( "${{ matrix.helper }}" -ne "" -And "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -platform.classifier=windows-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
} elseif ( "${{ matrix.helper }}" -ne "" ) {
|
||||
$mvn_ext=" -Dplatform.classifier=windows-x86_64-${{ matrix.helper }}"
|
||||
} elseif ( "${{ matrix.extension }}" -ne "" ) {
|
||||
$mvn_ext=" -Dplatform.classifier=windows-x86_64-${{matrix.extension}} "
|
||||
} else {
|
||||
$mvn_ext=" -Dplatform.classifier=windows-x86_64"
|
||||
}
|
||||
cd "%GITHUB_WORKSPACE%\platform-tests"
|
||||
mvn "%mvn_ext%" -Djavacpp.platform=windows-x86_64 clean test
|
||||
|
||||
env:
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
|
||||
|
||||
mac-x86_64:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
helper: [ onednn,"" ]
|
||||
extension: [ avx2,avx512,"" ]
|
||||
include:
|
||||
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags
|
||||
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
|
||||
experimental: true
|
||||
name: Debug enabled
|
||||
|
||||
- runs_on: ${{ github.event.inputs.runsOn }}
|
||||
experimental: true
|
||||
name: OS to run on
|
||||
|
||||
- mvn_flags: ${{ github.event.inputs.mvnFlags }}
|
||||
experimental: true
|
||||
name: Extra maven flags to use as part of the build
|
||||
|
||||
runs-on: macos-10.15
|
||||
steps:
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Java for publishing to OSSRH
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: 'temurin'
|
||||
server-id: 'central'
|
||||
server-username: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
|
||||
server-password: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
|
||||
|
||||
|
||||
|
||||
- name: Update path for gnu sed
|
||||
shell: bash
|
||||
run: |
|
||||
brew install gpg1 gnu-sed unzip ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl gpg bison flex perl ragel binutils gradle gmp isl libmpc mpfr wget python
|
||||
echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
|
||||
- name: Download dl4j-test-resources
|
||||
uses: ./.github/actions/download-dl4j-test-resources-linux
|
||||
- name: Build and install
|
||||
shell: bash
|
||||
env:
|
||||
MODULES: ${{ matrix.mvn_flags }}
|
||||
MAVEN_OPTS: "-Xmx2g"
|
||||
HELPER: ${{ matrix.helper }}
|
||||
EXTENSION: ${{ matrix.extension }}
|
||||
|
||||
|
||||
run: |
|
||||
if [ "${{ matrix.helper }}" != '' ] && [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dplatform.classifier=macosx-x86_64-${{ matrix.helper }}-${{matrix.extension}}"
|
||||
elif [ "${{ matrix.helper }}" != '' ]; then
|
||||
mvn_ext="-Dplatform.classifier=macosx-x86_64-${{ matrix.helper }}"
|
||||
elif [ "${{ matrix.extension }}" != '' ]; then
|
||||
mvn_ext=" -Dplatform.classifier=macosx-x86_64-${{matrix.extension}}"
|
||||
else
|
||||
mvn_ext=" -Dlibnd4j.classifier=macosx-x86_64"
|
||||
fi
|
||||
cd ${GITHUB_WORKSPACE}/platform-tests && mvn -Djavacpp.platform=macosx-x86_64 ${mvn_ext} clean test
|
||||
@@ -0,0 +1,49 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 1
|
||||
|
||||
runsOn:
|
||||
description: 'The operating system to run on, defaults to self hosted'
|
||||
required: false
|
||||
default: self-hosted
|
||||
|
||||
jobs:
|
||||
linux-x86_64:
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- uses: ./.github/actions/download-dl4j-test-resources-linux
|
||||
- name: Run cpu tests
|
||||
shell: bash
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
run: |
|
||||
mvn --version
|
||||
cmake --version
|
||||
protoc --version
|
||||
export OMP_NUM_THREADS=1
|
||||
mkdir -p ${GITHUB_WORKSPACE}/resources
|
||||
mkdir -p ${GITHUB_WORKSPACE}/cache
|
||||
mvn -Djunit.jupiter.execution.parallel.enabled=true -Djunit.jupiter.execution.parallel.mode.default=concurrent -Djunit.jupiter.execution.parallel.config.strategy=fixed -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -DexcludedGroups="long-running-tests, large-resources, distributed-systems" -DskipTestResourceEnforcement=true clean test --fail-never
|
||||
mvn -Djunit.jupiter.execution.parallel.enabled=true -Djunit.jupiter.execution.parallel.mode.default=concurrent -Djunit.jupiter.execution.parallel.config.strategy=fixed -Dorg.nd4j.strumpf.resource.dirs=${GITHUB_WORKSPACE}/resources -Dorg.nd4j.test.resources.cache.dir=${GITHUB_WORKSPACE}/cache -Dgroups="long-running-tests, large-resources, distributed-systems" -Dtest.offheap.size=14g -Dtest.heap.size=6g -Dsurefire.parallel.forcedTimeout=500 -Dsurefire.parallel.timeout=500 -Dsurefire.timeout=200 -Dsurefire.exitTimeout=500 test --fail-never -rf :nd4j
|
||||
@@ -0,0 +1,45 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
buildThreads:
|
||||
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
|
||||
required: true
|
||||
default: 1
|
||||
|
||||
runsOn:
|
||||
description: 'The operating system to run on, defaults to self-hosted'
|
||||
required: false
|
||||
default: self-hosted
|
||||
|
||||
jobs:
|
||||
linux-x86_64:
|
||||
runs-on: ${{ github.event.inputs.runsOn }}
|
||||
steps:
|
||||
- uses: AutoModality/action-clean@v1
|
||||
- name: Cancel Previous Runs
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
with:
|
||||
access_token: ${{ github.token }}
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
|
||||
- uses: ./.github/actions/download-dl4j-test-resources-linux
|
||||
- name: Run cpu tests
|
||||
shell: bash
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
run: |
|
||||
mvn --version
|
||||
cmake --version
|
||||
export OMP_NUM_THREADS=2
|
||||
mvn -DskipTestResourceEnforcement=true -Dlibnd4j.build=debug -Dlibnd4j.sanitize=ON clean test
|
||||
@@ -0,0 +1,100 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
cache:
|
||||
runs-on: windows-2019
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Free Disk Space (Windows)
|
||||
shell: powershell
|
||||
run: |
|
||||
# Show initial disk space
|
||||
Write-Host "Initial disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
# Remove Windows Defender scan history
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed Windows Defender scan history"
|
||||
|
||||
# Clear Windows temp folders
|
||||
Remove-Item -Path "$env:SystemRoot\Temp\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows temp folders"
|
||||
|
||||
# Clear Windows Update cache safely (without stopping/starting service)
|
||||
try {
|
||||
Remove-Item -Path "$env:SystemRoot\SoftwareDistribution\Download\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Windows Update download cache"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Could not clear Windows Update cache. Continuing..."
|
||||
}
|
||||
|
||||
# Clean package manager caches
|
||||
if (Test-Path -Path "C:\npm\cache") {
|
||||
Remove-Item -Path "C:\npm\cache\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared NPM cache"
|
||||
}
|
||||
|
||||
choco cache remove -y -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Chocolatey cache"
|
||||
|
||||
# Remove Docker images if Docker is installed
|
||||
try {
|
||||
if (Get-Command "docker" -ErrorAction SilentlyContinue) {
|
||||
docker image prune -a -f
|
||||
docker container prune -f
|
||||
docker volume prune -f
|
||||
Write-Host "Pruned Docker resources"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to prune Docker resources. Continuing..."
|
||||
}
|
||||
|
||||
# Remove .NET SDK/Runtime backup folders
|
||||
if (Test-Path -Path "$env:ProgramData\Microsoft\.NET\*.backup") {
|
||||
Remove-Item -Path "$env:ProgramData\Microsoft\.NET\*.backup" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Removed .NET backup folders"
|
||||
}
|
||||
|
||||
# Clear Azure artifacts cache
|
||||
if (Test-Path -Path "$env:LOCALAPPDATA\Microsoft\Azure\*") {
|
||||
Remove-Item -Path "$env:LOCALAPPDATA\Microsoft\Azure\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Cleared Azure artifacts cache"
|
||||
}
|
||||
|
||||
# Optimize Windows Component Store
|
||||
try {
|
||||
Start-Process -FilePath "dism.exe" -ArgumentList "/online /Cleanup-Image /StartComponentCleanup" -NoNewWindow -Wait
|
||||
Write-Host "Optimized Windows Component Store"
|
||||
}
|
||||
catch {
|
||||
Write-Host "Failed to optimize Windows Component Store. Continuing..."
|
||||
}
|
||||
|
||||
# Show final disk space
|
||||
Write-Host "Final disk space:"
|
||||
Get-PSDrive C | Select-Object Used,Free
|
||||
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master
|
||||
env:
|
||||
cuda: 11.6.0
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master
|
||||
env:
|
||||
cuda: 11.4.1
|
||||
- name: Cache cuda install windows cuda 11.4
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-114
|
||||
with:
|
||||
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4
|
||||
key: windows-2019-cuda-11.4
|
||||
restore-keys: windows-2019-cuda-11.4
|
||||
- name: Cache cuda install windows cuda 11.6
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-116
|
||||
with:
|
||||
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6
|
||||
key: windows-2019-cuda-11.6
|
||||
restore-keys: windows-2019-cuda-11.6
|
||||
@@ -0,0 +1,32 @@
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
cache:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master
|
||||
env:
|
||||
cuda: 11.6.0
|
||||
GCC: 7
|
||||
- name: Cache cuda 11.6 install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-116
|
||||
with:
|
||||
path: /usr/local/cuda-11.6
|
||||
key: ubuntu-22.04-cuda-11.6
|
||||
restore-keys: ubuntu-22.04-cuda-11.6
|
||||
- uses: konduitai/cuda-install/.github/actions/install-cuda-ubuntu@master
|
||||
env:
|
||||
cuda: 11.4
|
||||
GCC: 7
|
||||
- name: Cache cuda 11.4 install
|
||||
uses: actions/cache@v4
|
||||
id: cache-cuda-114
|
||||
with:
|
||||
path: /usr/local/cuda-11.4
|
||||
key: ubuntu-22.04-cuda-11.4
|
||||
restore-keys: ubuntu-22.04-cuda-11.4
|
||||
|
||||
|
||||
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
.DS_Store
|
||||
pom.xml.releaseBackup
|
||||
target/
|
||||
dependency-reduced-pom.xml
|
||||
*.ser
|
||||
application.home_IS_UNDEFINEiD
|
||||
README.md~
|
||||
*.bin
|
||||
*.releaseBackup
|
||||
*.out
|
||||
*~
|
||||
.pydevproject
|
||||
release.properties
|
||||
.idea/
|
||||
*.iml
|
||||
*.prefs
|
||||
*.settings/*
|
||||
*.log
|
||||
libnd4j/preprocessed
|
||||
.project
|
||||
.classpath
|
||||
metastore_db
|
||||
*.ipynb*
|
||||
!/dl4j-examples/tutorials/*.ipynb
|
||||
*-git.properties
|
||||
*.class
|
||||
*.jar
|
||||
*.war
|
||||
*.ear
|
||||
*.zip
|
||||
*.tar.gz
|
||||
*.rar
|
||||
hs_err_pid*
|
||||
pom.xml.tag
|
||||
pom.xml.releaseBackup
|
||||
pom.xml.versionsBackup
|
||||
pom.xml.next
|
||||
release.properties
|
||||
*dependency-reduced-pom.xml
|
||||
Test Results*.xml
|
||||
Test Results*.html
|
||||
libnd4j/tests_*
|
||||
platform-tests/*.proto
|
||||
platform-tests/*.txt
|
||||
platform-tests/*.pbtxt
|
||||
|
||||
# Specific for Nd4j
|
||||
*.md5
|
||||
*.pom
|
||||
*.sha1
|
||||
*.ser
|
||||
*.so
|
||||
*.jpg
|
||||
*.png
|
||||
*.iml
|
||||
*.prefs
|
||||
*.dylib
|
||||
lib/
|
||||
.vs/
|
||||
.vscode/
|
||||
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/resources/bin
|
||||
nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/test/resources/writeNumpy.csv
|
||||
nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/data-all*
|
||||
nd4j/nd4j-backends/nd4j-tests/src/test/resources/tf_graphs/examples/**/checkpoint
|
||||
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/onnx/
|
||||
nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/tensorflow/
|
||||
nd4j/samediff-import/samediff-import-onnx/*.txt
|
||||
nd4j/samediff-import/samediff-import-onnx/*.pbtxt
|
||||
nd4j/samediff-import/samediff-import-tensorflow/*.txt
|
||||
nd4j/samediff-import/samediff-import-tensorflow/*.pbtxt
|
||||
*.pom.xml.tmp
|
||||
libnd4j/tests_cpu/surefire-reports
|
||||
|
||||
doc_sources/
|
||||
doc_sources_*
|
||||
|
||||
*.pyc
|
||||
|
||||
# Python virtual environments
|
||||
venv/
|
||||
venv2/
|
||||
|
||||
# Ignore the nd4j files that are created by javacpp at build to stop merge conflicts
|
||||
nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java
|
||||
nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java
|
||||
|
||||
# Ignore meld temp files
|
||||
*.orig
|
||||
|
||||
*.html
|
||||
|
||||
#vim
|
||||
*.swp
|
||||
|
||||
*.dll
|
||||
*.tmp
|
||||
libnd4j/include/generated/include_ops.h
|
||||
@@ -0,0 +1,13 @@
|
||||
<extensions xmlns="http://maven.apache.org/EXTENSIONS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
|
||||
<extension>
|
||||
<groupId>com.github.gzm55.maven</groupId>
|
||||
<artifactId>project-settings-extension</artifactId>
|
||||
<version>0.2.3</version>
|
||||
</extension>
|
||||
<extension>
|
||||
<groupId>fish.payara.maven.extensions</groupId>
|
||||
<artifactId>regex-profile-activator</artifactId>
|
||||
<version>0.2</version>
|
||||
</extension>
|
||||
</extensions>
|
||||
@@ -0,0 +1,3 @@
|
||||
<settings>
|
||||
|
||||
</settings>
|
||||
@@ -0,0 +1,100 @@
|
||||
# SameDiff file format proposal
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
|
||||
Proposed by: Alex Black (15-05-2020)
|
||||
|
||||
Discussed with: raver119
|
||||
|
||||
## Context
|
||||
|
||||
SameDiff models need to be serializable - i.e., something we can save to disk or send over the network.
|
||||
Additionally, we need to be able to save and load model files in C++, and have those be readable in other languages (mainly Java).
|
||||
|
||||
Currently, we have a FlatBuffers-based format for SameDiff graph serialization, but it has a number of problems, as discussed in this issue: https://github.com/eclipse/deeplearning4j/issues/8312
|
||||
|
||||
|
||||
## Decision
|
||||
|
||||
We will transition from a pure FlatBuffers to a Zip + FlatBuffers model format.
|
||||
|
||||
FlatBuffers will be used for the graph structure only. Parameters will be stored separately to the graph structure, also within the zip.
|
||||
|
||||
We will introduce the ability to support multiple versions of a graph in the model files.
|
||||
This will enable the model file to support storing
|
||||
* Multiple data types (for example, a FP32 version and a quantized INT8 version)
|
||||
* Multiple different checkpoints (parameters after 1000 iterations, after 5000, and so on)
|
||||
* Multiple versions of a given model (English vs. Chinese, or cased/uncased, etc)
|
||||
|
||||
By default when loading a graph (unless it is otherwise specified) we will load the most recent model tag.
|
||||
Tags must be valid file/folder identifiers, and are not case sensitive.
|
||||
|
||||
|
||||
The structure of the zip file will be as follows:
|
||||
```
|
||||
tags.txt //List of graph tags, one per line, in UTF8 format, no duplicates. Oldest first, newest last
|
||||
<tag_name>/graph.fb //The graph structure, in FlatBuffers format
|
||||
<tag_name>/params.txt //The mapping between variable names and parameter file names
|
||||
<tag_name>/params/*.fb //The set of NDArrays that are the parameters, in FlatBuffers format
|
||||
<tag_name>/trainingConfig.fb //The training configuration - updater, learning rate, etc
|
||||
<tag_name>/updater.txt //The mapping between variable names and the updater state file names
|
||||
<tag_name>/updater/*.fb //The set of NDArrays that are the updater state
|
||||
```
|
||||
|
||||
Note that params.txt will allow for parameter sharing via references to other parameters:
|
||||
```
|
||||
my_normal_param 0
|
||||
shared_param <other_tag_name>/7
|
||||
```
|
||||
This means the parameters values for parameter "my_normal_param" are present at `<tag_name>/params/0.fb` within the zip file, and the parameter values for "shared_param" are available at `<other_tag_name>/params/7.fb`
|
||||
|
||||
Note also that the motivation for using the params.txt file (instead of the raw parameter name as the file name) is that some parameters will have invalid or ambiguous file names - "my/param/name", "&MyParam*" etc
|
||||
|
||||
In terms of updater state, they will be stored in a similar format. For example, for the Adam updater with the M and V state arrays (each of same shape as the parameter):
|
||||
```
|
||||
my_param 0 1
|
||||
other_param 2 3
|
||||
```
|
||||
That means my_param(M) is `<tag_name>/updater/0.fb` and my_param(V) is at `<tag_name>/updater/1.fb`
|
||||
This format also allows for updater state sharing, if we need it.
|
||||
|
||||
|
||||
**Graph Structure**
|
||||
|
||||
The graph structure will be encoded in FlatBuffers format using a schema with 2 parts:
|
||||
1. A list of variables - each with name, datatype, and (for placeholders, constants and parameters) a shape
|
||||
2. A list of operations - each with a name, op name/type, input variable names, output variable names, and arguments
|
||||
|
||||
Note that both legacy and custom ops will be encoded in the same way. For legacy ops, we simply need the operation type, and the operation number.
|
||||
|
||||
Operation argument encoding will be done using named arguments: essentially, a `Map<String,T>` structure, where T is one of `{long, double, boolean, datatype}`.
|
||||
This allows for improved backward compatibility (no ambiguity as ops are modified after a graph file was written) and improved interpretability compared to using simple arrays of iargs, bargs, targs and dargs.
|
||||
One consequence/downside of this is that we need to define mapping between our named arguments and iargs/bargs/targs/dargs. In Java we have essentially done this manually, though clearly don't want to replicate this work in C++ (or any future languages).
|
||||
|
||||
To avoid the need to do a significant amount of work (such as moving the name/arg mapping to code generation) the following is proposed:
|
||||
The `Map<String,T>` is split up in the FlatBuffers schema into 4 pairs of fields.
|
||||
* `String[] iArgNames`, `long[] iArgs`
|
||||
* `String[] tArgNames`, `double[] dArgs`
|
||||
* `String[] bArgNames`, `boolean[] bArgs`
|
||||
* `String[] dArgNames`, `DataType[] dArgs`
|
||||
|
||||
Clearly the name and value arrays (for each pair) would each be the same length, and name/value correspondence is by array index.
|
||||
|
||||
This is essentially equivalent to the `Map<String,T>` representation, but has the benefit of not needing us to define the mapping for named args to array-style args any time soon in C++, but also allowing us to add it in the future (mainly before we can write graphs from C++, or have better/proper backward compatibility after op changes)
|
||||
|
||||
|
||||
**Extensibility to Other Types**
|
||||
|
||||
Suppose in the future we want to store other data for a variable, not just an array?
|
||||
Examples include lists and maps (for example, for NLP applications).
|
||||
|
||||
While we will not implement this right now, there are a number of options for adding this without breaking backward compatibility.
|
||||
|
||||
First: we can enhance the params.txt file format, perhaps using something like the following:
|
||||
```
|
||||
map_param 0 MAP
|
||||
```
|
||||
|
||||
Second: We can add a similar text file for other types. For example, a params_maps.txt, same format as params.txt, with content at `<tag_name>/params_maps/*.fb`
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# Onnx runtime module
|
||||
|
||||
## Status
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (23-09-2020)
|
||||
|
||||
Discussed with: saudet
|
||||
|
||||
## Context
|
||||
|
||||
We need a way of providing nd4j a way of running onnx modules
|
||||
that is easily compatible with the onnx community. The gold standard for this
|
||||
is is using [onnxruntime](https://github.com/microsoft/onnxruntime/blob/master/docs/Java_API.md).
|
||||
|
||||
|
||||
## Decision
|
||||
|
||||
We will use javacpp's onnxruntime bindings in a similar manner to [nd4j-tensorflow](../nd4j-tensorflow)
|
||||
allowing nd4j to be used as an ndarray format that interops with onnxruntime.
|
||||
|
||||
We will implement a simple api similar to the [GraphRunner](../nd4j-tensorflow/src/main/java/org/nd4j/tensorflow/conversion/graphrunner/GraphRunner.java)
|
||||
This will sit on top of javacpp's lower level onnxruntime bindings.
|
||||
|
||||
This module will follow a similar structure to the nd4j-tensorflow module
|
||||
focusing on INDArrays as a data interchange format, but otherwise pass execution
|
||||
down to onnxruntime.
|
||||
|
||||
|
||||
The main api to the graph runner works as follows:
|
||||
|
||||
```java
|
||||
try(GraphRunner runner = new GraphRunner(...)) {
|
||||
Map<String,INDArray> inputs = new HashMap<>();
|
||||
// ..initialize inputs
|
||||
Map<String,INDArray> outputs = runner.run(inputs);
|
||||
// process outputs...
|
||||
}
|
||||
```
|
||||
|
||||
The core logic will contain the following components:
|
||||
|
||||
1. Loading onnx pb files
|
||||
2. A graph runner in similar nature to nd4j-tensorflow
|
||||
3. Interop with onnxruntime's version of an ndarray/tensor
|
||||
|
||||
Using different accelerators/backends
|
||||
-----------------------------------------
|
||||
|
||||
Similar to nd4j-tensorflow which uses javacpp for the specific version of
|
||||
tensorflow to use, this module will rely on the user picking the right dependency
|
||||
to link against. Different builds of cpu, gpu, .. exist [here](https://repo1.maven.org/maven2/org/bytedeco/tensorflow/1.15.3-1.5.4/)
|
||||
The equivalent of this in onnxruntime can be found [here](https://repo1.maven.org/maven2/org/bytedeco/onnxruntime/1.4.0-1.5.4/)
|
||||
|
||||
The user will need to include the version of onnxruntime they wish to use
|
||||
similar to how you link against a particular implementation in a c library
|
||||
or include a backend in nd4j. This will happen via maven.
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
# Import IR
|
||||
|
||||
## Status
|
||||
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (28-09-2020)
|
||||
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
## Context
|
||||
|
||||
Currently, there is a gap in the way samediff/nd4j operations are implemented
|
||||
vs. how other frameworks represent their models.
|
||||
|
||||
Keras, Tensorflow, and Pytorch use an attribute based format with names. Interop
|
||||
between Onnx ,Tensorflow, and Keras tends to follow the following formula:
|
||||
|
||||
1. Map names to equivalent names in the other framework for each operation
|
||||
configuration. Names being both op names and associated attributes of the
|
||||
operations such as in Conv2D where you have strides, kernel sizes.
|
||||
2. Map input/output tensors to the equivalent tensor type in each framework.
|
||||
3. Setup the complete graph in the equivalent framework. Sometimes the
|
||||
framework's concepts don't map 1 to 1. They should output equivalent results
|
||||
regardless though. In order to do this, sometimes the framework needs to
|
||||
add/remove operations in order to produce equivalent output in a different
|
||||
graph. The [tensorflow onnx import](https://github.com/onnx/tensorflow-onnx#how-tf2onnx-works)
|
||||
is a good example of this.
|
||||
|
||||
Samediff/nd4j have their internal op representations as a set of ordered
|
||||
arguments for execution in the form of:
|
||||
|
||||
1. t arguments: floating point arguments (float, double,..)
|
||||
2. integer arguments: integer arguments (long, integer)
|
||||
3. boolean argument: boolean arguments
|
||||
4. data type arguments: data types for input/output
|
||||
5. input arguments: ndarrays for input
|
||||
6. output arguments: often optional (dynamically created) output ndarray
|
||||
arguments. If the user wants to pass in outputs to control memory, they are
|
||||
allowed to do so.
|
||||
7. axis arguments: Integer arguments that represent the dimension(s) for an
|
||||
operation to be executed on.
|
||||
|
||||
[Reference implementation](https://github.com/KonduitAI/deeplearning4j/blob/master/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/DynamicCustomOp.java#L58)
|
||||
|
||||
This maps well enough for execution, but not for file formats.
|
||||
|
||||
## Related Work
|
||||
This may encourage future work to be done to the
|
||||
[samediff file format](https://github.com/KonduitAI/deeplearning4j/blob/master/nd4j/ADRs/0001-SameDiff_File_Format.md).
|
||||
Implementation of serialization of file format via flatbuffers can be found
|
||||
[here](https://github.com/eclipse/deeplearning4j/blob/master/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/SameDiff.java#L4748)
|
||||
Of note here for prior work is the
|
||||
[current code generation]
|
||||
(https://github.com/KonduitAI/dl4j-dev-tools/blob/master/codegen/src/main/ops/org/nd4j/codegen/ops/CNN.kt#L28)
|
||||
|
||||
The definitions for the kotlin dsl can be found
|
||||
[here](https://github.com/KonduitAI/dl4j-dev-tools/blob/master/codegen/src/main/kotlin/org/nd4j/codegen/dsl/OpBuilder.kt)
|
||||
|
||||
|
||||
While it does have the intended description,
|
||||
it’s kotlin specific and is only available for a very small subset
|
||||
of the ops where pre-created objects were created
|
||||
for specific operations. The goal of this ADR is to expand upon
|
||||
that and make it language agnostic by providing this information in a
|
||||
neutral file format that has code generation with it.
|
||||
|
||||
Current code generation efforts can be augmented using this file format.
|
||||
More on this decision making can be found [here](https://github.com/KonduitAI/dl4j-dev-tools/blob/master/codegen/adr/0007-configuration_objects.md)
|
||||
|
||||
|
||||
|
||||
## Proposal
|
||||
|
||||
We expose a symbol based mapping in libnd4j in protobuf format, similar to how
|
||||
other frameworks are doing it, as a bridge/intermediary format.
|
||||
|
||||
This makes it easier to implement interop with the other frameworks, because it
|
||||
adds the necessary information that is needed to be able to define a direct
|
||||
mapping.
|
||||
|
||||
This could be a future file format depending on how the framework evolves. For
|
||||
now, this is considered a work around for making writing import code easier/more
|
||||
portable.
|
||||
|
||||
Similar to [ONNX](https://onnx.ai/) and [Tensorflow](https://tensorflow.org/)
|
||||
we use protobuf to express an attribute based file format and map
|
||||
samediff/nd4j operations to this format.
|
||||
|
||||
We use a translation layer that handles mapping from attributes to the ordered
|
||||
arguments approach reflected in samediff/nd4j.
|
||||
|
||||
For each operation, we define a mapping process to/from this attribute format to the
|
||||
order based execution format.
|
||||
|
||||
A separate but similar set of rules are used for mapping ndarrays.
|
||||
|
||||
This attribute based format is an Intermediary Representation that we then
|
||||
"compile" to the equivalent calls in libnd4j.
|
||||
|
||||
|
||||
The format definitions for the IR can be found [here](./src/main/proto/nd4j/nd4j.proto)
|
||||
|
||||
## Consequences
|
||||
|
||||
Migration to an attribute based import format makes working with other deep
|
||||
learning frameworks easier in the future.
|
||||
|
||||
|
||||
### Drawbacks
|
||||
|
||||
1. Yet another file format.
|
||||
2. Risk migrating to new file format in the future.
|
||||
3. A lot of up front manual work to index set of current operations.
|
||||
4. Backwards compatibility: yet another thing to maintain. We wrote converters
|
||||
for any forward compatibility. We address this by specifying an opset schema
|
||||
scheme similar to onnx.
|
||||
|
||||
### Advantages
|
||||
|
||||
1. Easy to maintain.
|
||||
2. Backwards compatible.
|
||||
3. Easily interops with existing other deep learning frameworks.
|
||||
4. No additional dependencies from what's already normal.
|
||||
5. Protobuf allows easy code generation for other languages.
|
||||
6. Industry standard conventions being used over proprietary tooling reducing
|
||||
friction for adoption for people coming from other frameworks
|
||||
7. Straightforward mapping of arguments for import
|
||||
8. Provide an easy bridge to existing libnd4j
|
||||
9. Allow automation of op descriptors in any language that would understand how
|
||||
to pass data to the c++ library.
|
||||
|
||||
|
||||
## Appendix A: Comparison with other Frameworks, implicit vs. explicit
|
||||
|
||||
We can find the existing attributes from the conventions of the
|
||||
libnd4j code base. The libnd4j [conv1d.cpp](https://github.com/KonduitAI/deeplearning4j/blob/master/libnd4j/include/ops/declarable/generic/nn/convo/conv1d.cpp#L104)
|
||||
file contains the following declaration:
|
||||
|
||||
```
|
||||
auto inputShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
sd::LongType const* biasShapeInfo = block.width() > 2 ? inputShape->at(2) : nullptr;
|
||||
|
||||
int kW = INT_ARG(0) > 0 ? INT_ARG(0) : static_cast<int>(shape::sizeAt(weightsShapeInfo, 0)); // filter(kernel) width
|
||||
int sW = INT_ARG(1); // strides width
|
||||
int pW = INT_ARG(2); // paddings width
|
||||
int dW = INT_ARG(3); // dilations width
|
||||
int paddingMode = INT_ARG(4); // 0-VALID, 1-SAME
|
||||
int isNCW = block.getIArguments()->size() > 5 ? !INT_ARG(5) : 1; // INT_ARG(4): 1-NWC, 0-NCW
|
||||
int wFormat = block.getIArguments()->size() > 6 ? INT_ARG(6) : 0; // 0 - [kW, iC, oC], 1 - [oC, iC, kW], 2 - [oC, kW, iC]
|
||||
```
|
||||
|
||||
We can see that there are macros in the libnd4j code base, which reflect how
|
||||
each argument is accessed. Each list of arguments has an expected order, that we
|
||||
need to explicitly map to a parseable structure.
|
||||
|
||||
In comparison, the
|
||||
[onnx Convolution operator](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Conv)
|
||||
has *explicit* attributes of various types such as lists of ints and named
|
||||
tensors.
|
||||
|
||||
As shown above, these concepts exist internally in the operations and layers
|
||||
themselves in nd4j/samediff, but they are not exposed directly to the user.
|
||||
|
||||
|
||||
A theoretical op descriptor from libnd4j is as follows:
|
||||
```java
|
||||
private String name;
|
||||
private int nIn,nOut,tArgs,iArgs;
|
||||
private boolean inplaceAble;
|
||||
private List<String> inArgNames;
|
||||
private List<String> outArgNames;
|
||||
private List<String> tArgNames;
|
||||
private List<String> iArgNames;
|
||||
private List<String> bArgNames;
|
||||
private OpDeclarationType opDeclarationType;
|
||||
|
||||
public enum OpDeclarationType {
|
||||
CUSTOM_OP_IMPL,
|
||||
BOOLEAN_OP_IMPL,
|
||||
LIST_OP_IMPL,
|
||||
LOGIC_OP_IMPL,
|
||||
OP_IMPL,
|
||||
DIVERGENT_OP_IMPL,
|
||||
CONFIGURABLE_OP_IMPL,
|
||||
REDUCTION_OP_IMPL,
|
||||
BROADCASTABLE_OP_IMPL,
|
||||
BROADCASTABLE_BOOL_OP_IMPL
|
||||
}
|
||||
```
|
||||
|
||||
It contains all the op declarations and fields associated with a descriptor.
|
||||
|
||||
In the libnd4j code base, we represent the op descriptor types above
|
||||
*implicitly* through validation as well as the different macros present in the
|
||||
code base representing what an op execution looks like.
|
||||
|
||||
Validation for what can be present in the various names can be found
|
||||
[here](https://github.com/KonduitAI/deeplearning4j/blob/master/libnd4j/include/ops/declarable/impl/DeclarableOp.cpp#L734-L765)
|
||||
|
||||
The set of macro declarations in libnd4j can be found
|
||||
[here](https://github.com/eclipse/deeplearning4j/blob/master/libnd4j/include/system/op_boilerplate.h)
|
||||
|
||||
|
||||
## Appendix B: Format Comparison to other frameworks
|
||||
|
||||
An add op in tensorflow looks like:
|
||||
|
||||
```
|
||||
op {
|
||||
name: "Add"
|
||||
input_arg {
|
||||
name: "x"
|
||||
type_attr: "T"
|
||||
}
|
||||
input_arg {
|
||||
name: "y"
|
||||
type_attr: "T"
|
||||
}
|
||||
output_arg {
|
||||
name: "z"
|
||||
type_attr: "T"
|
||||
}
|
||||
attr {
|
||||
name: "T"
|
||||
type: "type"
|
||||
allowed_values {
|
||||
list {
|
||||
type: DT_BFLOAT16
|
||||
type: DT_HALF
|
||||
type: DT_FLOAT
|
||||
type: DT_DOUBLE
|
||||
type: DT_UINT8
|
||||
type: DT_INT8
|
||||
type: DT_INT16
|
||||
type: DT_INT32
|
||||
type: DT_INT64
|
||||
type: DT_COMPLEX64
|
||||
type: DT_COMPLEX128
|
||||
type: DT_STRING
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Onnx’s add can be found here
|
||||
https://github.com/onnx/onnx/blob/master/docs/Operators.md#Add
|
||||
|
||||
Onnx and tensorflow are purely attribute based formats.
|
||||
@@ -0,0 +1,195 @@
|
||||
|
||||
# Libnd4j NdArray padded buffers, strides for Arm_Compute Library wrapper
|
||||
|
||||
## Status
|
||||
Implemented
|
||||
|
||||
Proposed by: Abdelrauf (23/09/2020)
|
||||
|
||||
Discussed with:
|
||||
|
||||
## Context
|
||||
During the integration process of our library with arm_compute, I faced that our NdArray strides are not flexible. (i.e it cant be set properly without **special and manual handling**).
|
||||
Let's say our Nd Array shapes are `[3,4,2]` and the last index is moving faster (i.e C order). Then our strides will be `[ 8, 2, 1 ]`.
|
||||
As far as I know, our last index stride can be different (called as ews), but overall strides should follow the cyclic strict rule of dependency.:
|
||||
|
||||
strides[index-1] = strides[index] * shapes[index];
|
||||
On arm_compute besides strides there is also Padding `{top, right, bottom, left}` that can be used to increase strides and change offsets adn as well as total size. its mostly done for performance reasons. As from above we can see that **its just hosting NdArray shape in the buffer of the bigger NdArray shape**. In arm_compute those paddings applied to last 2 dimensions (on NCHW it will be H and W}. We can define it like this:
|
||||
|
||||
newH = pad.top + H + pad.bottom;
|
||||
newW = pad.left + W + pad.right;
|
||||
|
||||
so strides will be calculated for the shape `{N,C, newH, newW}` and offset of the first element will be:
|
||||
|
||||
offset = pad.left * strideOfNewW + pad.top * strideOfNewH
|
||||
|
||||
|
||||
## Proposal
|
||||
Introduce helper functions checking below case :
|
||||
|
||||
strides[index-1] >= strides[index] * shapes[index];
|
||||
|
||||
Add **generic method for the padded buffer** ( we can simulate arm_compute 2d padding and more)
|
||||
|
||||
int paddings[rank] = {...}; // total padding
|
||||
int paddingOffsets[rank] = {...}; //offset indices of the first element
|
||||
|
||||
This could be used to padd ndArray shapes and calculate strides based on it while keeping original shape, paddOffsets could be used to determine the beginning of the first element. Though this interface ismore generic its drawback is that on armcompute its possible to padd 1d into 2D while keeping rank but on this one we should supply 2d with one of its dimensions being 1.
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
1. All tests that were not tested **against subArray** could break. So they will require a fix
|
||||
2. Writing additional test cases
|
||||
|
||||
### Advantages
|
||||
- alignment possibility for CPUs where alignment is required for speed and vectorization.
|
||||
- easier integration with libraries. in the case of arm_compute, the last two dimensions are sometimes padded.
|
||||
|
||||
|
||||
### Disadvantages
|
||||
- its advantage is not so big for modern CPUs where unaligned vector loads possible
|
||||
- exposing it for users is not desirable: (excessive usage creates unnecessary memory spaces and performance problems)
|
||||
- could result in unnecessary complications for some function implementations
|
||||
- possibility of requiring additional tests and fixes
|
||||
|
||||
|
||||
### Technical details about the addition of this functionality into NdArray
|
||||
A little investigation showed that the current NdArray actually has constructors to specify strides.
|
||||
Here is the constructor that could be used
|
||||
[ShapeDescriptor.h](https://github.com/KonduitAI/deeplearning4j/blob/qwr_armcompute/libnd4j/include/array/ShapeDescriptor.h)
|
||||
Here are additions into ShapeDescriptor:
|
||||
- validate() //it willbe used for validation of strides and et cetera. This way we can create NdArray by just using ShapeDescriptor alone. And it will be more flexible with correctness
|
||||
- allocLength() //returns minimal buffer size for the given strides and shapes. (this was missing on libnd4j side)
|
||||
- paddedBufferDescriptor(..) //helper method for returning ShapeDescriptor for padded buffer.
|
||||
|
||||
|
||||
|
||||
#### [NdArrayFactory](https://github.com/KonduitAI/deeplearning4j/blob/qwr_armcompute/libnd4j/include/array/impl/NDArrayFactory.cpp#L39-L80)
|
||||
The method that is using ShapeDescriptor validation, and ShapeDescriptor paddedBuffer .
|
||||
|
||||
Furthermore to indicate that shape of the NdArray is using paddedBuffer we will flag with `ARRAY_HAS_PADDED_BUFFER` . so it will be possible to know if NdArray is padded.
|
||||
|
||||
Furthermore, it is still possible to recover Paddings from the allocation size of the padded NdArray. But its not an easy task to get PaddingOffsets from offset and recovered full shape. Thats why it requires storing them. Fortunately, for arm_compute tensors **manual padding** we just need to know **total size and the offset** of the first element. So we dont need to change internals that much
|
||||
|
||||
As our padded Buffer follows the strict ews() rule instead of the loose one. Paddings will be obtained from this rule:
|
||||
|
||||
strides[index-1] == strides[index] * shapes[index];
|
||||
|
||||
pseudo code for C order:
|
||||
|
||||
for (int j = rank - 1; j >= 0; j--) {
|
||||
shapesAfterPadding[j] = strides[j - 1] / strides[j]
|
||||
}
|
||||
shapesAfterPadding[0] = buffer.AllocSize / strides[0]
|
||||
//Paddings for index in 0..rank-1
|
||||
paddings[index] = shapesAfterPadding[index] - shape[index]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### Technical notes on arm_compute library
|
||||
|
||||
The main drive for the above proposal to avoid unnecessary performance and memory allocation. And also we should keep on mind :
|
||||
- in each newer version of arm_compute there are new implementations in which the padding requirements were removed.
|
||||
|
||||
This **can diminish the necessity for the proposed changes** if such versions of the desired functions are implemented.
|
||||
|
||||
##### Notes on arm_compute tensors
|
||||
Arm_compute tensors are mostly 3d 4d with max 6d dimensions.
|
||||
So lets show C order NdArray({2,2,5,5},)
|
||||
|
||||
shapeInfo shapeInfo: [4, 2,2,5,5, 50,25,5,1, 8192,1,99]
|
||||
|
||||
of float type and its arm_compute tensor equivalent :
|
||||
- first of all, we map NdArray dataTypes into arm_compute [armcomputeUtils.cpp#L35-L75](https://github.com/KonduitAI/deeplearning4j/blob/qwr_armcompute/libnd4j/include/ops/declarable/platform/armcompute/armcomputeUtils.cpp#L35-L75)
|
||||
- it will be with the reversed shape. **`NdArray{n,z,y,x} -> TensorShape{x,y,z,n}`**
|
||||
-
|
||||
|
||||
total length in bytes: 400
|
||||
shapes: 5,5,2,2,1,1,
|
||||
strides in bytes: 4,20,100,200,0,0,
|
||||
strides as elements: (1,5,25,50)
|
||||
|
||||
Paddings in arm_compute Tensors. `Padding{left,right, top, bottom}`
|
||||
As both OpenCL and NEON use vector loads and stores instructions to access the data in buffers, so in order to avoid having special cases to handle for the borders all the images and tensors used in this library must be padded
|
||||
There are different ways padding can be calculated:
|
||||
|
||||
- Accurate padding.
|
||||
in this case it is importan to configure and then after that to allocate
|
||||
- auto padding.
|
||||
It guarantees that the allocation will have enough padding to run any of the provided functions
|
||||
- no padding
|
||||
- manual padding
|
||||
|
||||
#### how padding affects strides offset and total size
|
||||
in arm_compute Tensor:
|
||||
it's 2d {Width Height} can be padded and thats why it affects strides.
|
||||
Lets show it with the picture:
|
||||
|
||||
\ top /
|
||||
\ _____________________ /
|
||||
left | ^ | right
|
||||
| Width |
|
||||
| <-Height |
|
||||
| |
|
||||
| |
|
||||
----------------------
|
||||
/ bottom \
|
||||
/ \
|
||||
|
||||
Here is the stride calculation pseudo code for Tensor {x,y,z}
|
||||
|
||||
stride_x = element_size(); //float will be 4
|
||||
stride_y = (padding.left + _tensor_shape[0] + padding.right) * stride_x;
|
||||
stride_z = (padding.top + _tensor_shape[1] + padding.bottom) * stride_y;
|
||||
|
||||
required_offset_first_element = padding.left * stride_x + padding.top * stride_y;
|
||||
|
||||
|
||||
For example: if arm_tensor had `padding: left 0, right 1, top 0, bottom 1` :
|
||||
|
||||
total: 576
|
||||
shapes: 5,5,2,2,1,1,
|
||||
strides in bytes: 4,24,144,288,0,0,
|
||||
|
||||
|
||||
|
||||
### Notes on the current wrapper implementation
|
||||
|
||||
This is a simple wrapper for arm functions with input and output tensors:
|
||||
[armcomputeUtils.h#L95-L165](https://github.com/KonduitAI/deeplearning4j/blob/qwr_armcompute/libnd4j/include/ops/declarable/platform/armcompute/armcomputeUtils.h#L85-L133)
|
||||
|
||||
From above we could see :
|
||||
- we had to flag padded NdArrays so that we can use manual padding version of arm_compute Tensors
|
||||
- when padding information is changed during configure process we **have to copy** our NdArray buffer into **new allocated** arm_tensor buffer. and the same with the output.
|
||||
- for cases without padding , arm_tensor could use our buffer if its ews()==1.
|
||||
- its desired to call configure and run separately to avoid multiple configure calls ( this is not discussed here, for now)
|
||||
|
||||
|
||||
## arm_compute wrapper proposal
|
||||
|
||||
|
||||
So from above we can conclude that we have two options:
|
||||
|
||||
- creating our NdArray with auto_padding strides and modifying the current wrapper. Still configure will be called foreach run. But with auto padding it is using more memory for small ndarrays
|
||||
- to be able to use accurate padding properly we should call configure before NdArray memory allocation so that we can import it. For that I should investigate graph, DeclarableOps and NdArrays usage lifecycle.
|
||||
|
||||
Here is auto padding:
|
||||
|
||||
// Some kernels compute 32 elements at the time, worst case scenario they
|
||||
// will read 32 values after the last element
|
||||
extra_pad_x = _tensor_shape.num_dimensions() < 1 ? 0 : 32;
|
||||
pad_x = _tensor_shape.num_dimensions() < 1 ? 0 : 4;
|
||||
pad_y = _tensor_shape.num_dimensions() < 2 ? 0 : 4;
|
||||
|
||||
PaddingSize(pad_y, pad_x + extra_pad_x, pad_y, pad_x);
|
||||
|
||||
## Discussion
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
# Import IR
|
||||
|
||||
## Status
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (28-09-2020)
|
||||
|
||||
Discussed with: N/A
|
||||
|
||||
## Context
|
||||
|
||||
Generally, every neural network file format defines a sequence of operations
|
||||
to execute mathematical operations that comprises a neural network.
|
||||
|
||||
Each element in the sequence is a node that contains information such as the
|
||||
desired operation, and a set of attributes that represent parameters
|
||||
in to the mathematical function to execute.
|
||||
|
||||
In order to write import/export for different frameworks, we need to adapt
|
||||
an attribute based format from various popular deep learning frameworks.
|
||||
Nd4j has a different list based format for operation execution arguments.
|
||||
In the [previous ADR](./Import_IR.md), we added an IR which makes it easier to
|
||||
interop with other frameworks.
|
||||
|
||||
In this ADR, this work is extended to add a file format for
|
||||
describing lists of operations as MappingRules which allow transformations
|
||||
from one framework to another.
|
||||
|
||||
These transformations manipulate protobuf as input and output Nd4j's
|
||||
new OpDescriptor format as output.
|
||||
|
||||
|
||||
##Related work
|
||||
|
||||
See [the import IR](./0003-Import_IR.md)
|
||||
|
||||
## Decision
|
||||
|
||||
We implement a mapping process framework that defines transforms on an input file format.
|
||||
A MappingProcess defines a list of MappingRules which represent a sequence of transformations
|
||||
on each attribute of an op definition.
|
||||
|
||||
To assist in mapping, a mapping context with needed information like rule arguments
|
||||
for transformation, current node, and whole graph are used as input.
|
||||
|
||||
The input is a protobuf file for a specific framework and the output is an op descriptor
|
||||
described [here](./0003-Import_IR.md).
|
||||
|
||||
A MappingRule converts 1 or more attributes in to 1 more or arg definitions. A potential definition
|
||||
can be found in Appendix E.
|
||||
|
||||
Attributes are named values supporting a wide variety of types from floats/doubles
|
||||
to lists of the same primitive types. See Appendix C for a theoretical definition.
|
||||
|
||||
Arg Definitions are the arguments for an OpDescriptor described in [the import IR ADR.](./0003-Import_IR.md)
|
||||
See Appendix D for a potential definition of arg definitions.
|
||||
|
||||
All of this together describes how to implement a framework agnostic
|
||||
interface to convert between a target deep learning framework and the nd4j format.
|
||||
|
||||
|
||||
## Implementation details
|
||||
|
||||
In order to implement proper mapping functionality, a common interface is implemented.
|
||||
Below are the needed common types for mapping:
|
||||
|
||||
1. IRNodeDef: A node definition in a graph
|
||||
2. IRTensor: A tensor type for mapping
|
||||
3. IROpList: A list of operations
|
||||
4. IRAttrDef: An attribute definition
|
||||
5. IRAttrValue: An attribute value
|
||||
6. IROpDef: An op definition for the IR
|
||||
7. IRDataType: A data type
|
||||
8. IRGraph: A graph abstraction
|
||||
|
||||
Each one of these types is a wrapper around a specific framework's input types
|
||||
of the equivalent concepts.
|
||||
|
||||
Each of these wrappers knows how to convert the specific concepts
|
||||
in to the nd4j equivalents for interpretation by a mapper which applies
|
||||
the mapping rules for a particular framework.
|
||||
|
||||
Doing this will allow us to share logic between mappers and making 1 implementation of
|
||||
mapping possible by calling associated getter methods for concepts like data types and nodes.
|
||||
|
||||
## Serialization
|
||||
|
||||
In order to persist rules using protobuf, all rules will know how to serialize themselves.
|
||||
A simple serialize() and load() methods are implemented which covers conversion using
|
||||
interface methods up to the user to implement which describes how to persist the protobuf
|
||||
representation. This applies to any of the relevant functionality such as rules and processes.
|
||||
|
||||
|
||||
|
||||
## Custom types
|
||||
|
||||
Some types will not map 1 to 1 or are directly applicable to nd4j.
|
||||
In order to combat this, when an unknown type is discovered during mapping,
|
||||
adapter functions for specific types must be specified.
|
||||
|
||||
Supported types include:
|
||||
|
||||
1. Long/Int
|
||||
2. Double/Float
|
||||
3. String
|
||||
4. Boolean
|
||||
5. Bytes
|
||||
6. NDArrays
|
||||
|
||||
|
||||
An example:
|
||||
|
||||
A Dim in tensorflow can be mapped to a long in nd4j.
|
||||
|
||||
Shape Information can be a list of longs or multiple lists depending on the
|
||||
context.
|
||||
|
||||
## Consequences
|
||||
### Advantages
|
||||
* Allows a language neutral way of describing a set of transforms necessary
|
||||
for mapping an set of operations found in a graph from one framework to the nd4j format.
|
||||
|
||||
* Allows a straightforward way of writing an interpreter as well as mappers
|
||||
for different frameworks in nd4j in a standardized way.
|
||||
|
||||
* Replaces the old import and makes maintenance of imports/mappers more straightforward.
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* More complexity in the code base instead of a more straightforward java implementation.
|
||||
|
||||
* Risks introducing new errors due to a rewrite
|
||||
|
||||
|
||||
## Appendix A: Contrasting MappingRules with another implementation
|
||||
|
||||
We map names and types to equivalent concepts in each framework.
|
||||
Onnx tensorflow does this with an [attribute converter](https://github.com/onnx/onnx-tensorflow/blob/08e41de7b127a53d072a54730e4784fe50f8c7c3/onnx_tf/common/attr_converter.py)
|
||||
|
||||
This is done by a handler (one for each op).
|
||||
More can be found [here](https://github.com/onnx/onnx-tensorflow/tree/master/onnx_tf/handlers/backend)
|
||||
|
||||
|
||||
## Appendix B: Challenges when mapping nd4j ops
|
||||
|
||||
The above formats are vastly different. Onnx and tensorflow
|
||||
are purely attribute based. Nd4j is index based.
|
||||
This challenge is addressed by the IR by adding names to each property.
|
||||
|
||||
|
||||
In order to actually map these properties, we need to define rules for doing so.
|
||||
Examples of why these mapping rules are needed below:
|
||||
|
||||
1. Different conventions for the same concept. One example that stands out from conv
|
||||
is padding. Padding can be represented as a string or have a boolean that says what a string equals.
|
||||
In nd4j, we represent this as a boolean: isSameMode. We need to do a conversion inline in order
|
||||
to invoke nd4j correctly.
|
||||
|
||||
2. Another issue is implicit concepts. Commonly, convolution requires you to configure a layout
|
||||
of NWHC (Batch size, Height, Width, Channels)
|
||||
or NCHW (Batch size, Channels,Height, Width). Tensorflow allows you to specify it,
|
||||
nd4j also allows you to specify it. Onnx does not.
|
||||
|
||||
A more in depth conversation on this specific issue relating to the
|
||||
2 frameworks can be found [here](https://github.com/onnx/onnx-tensorflow/issues/31)
|
||||
In order to address these challenges, we introduce a MappingRule allowing
|
||||
us to define a series of steps to map the input format to the nd4j format
|
||||
in a language neutral way via a protobuf declaration.
|
||||
|
||||
|
||||
## Appendix C: A theoretical attribute definition
|
||||
```kotlin
|
||||
enum class AttributeValueType {
|
||||
FLOAT,
|
||||
LIST_FLOAT,
|
||||
BYTE,
|
||||
LIST_BYTE,
|
||||
INT,
|
||||
LIST_INT,
|
||||
BOOL,
|
||||
LIST_BOOL,
|
||||
STRING,
|
||||
LIST_STRING
|
||||
}
|
||||
|
||||
interface IRAttribute<ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE> {
|
||||
|
||||
fun name(): String
|
||||
|
||||
fun floatValue(): Double
|
||||
|
||||
fun listFloatValue(): List<Float>
|
||||
|
||||
fun byteValue(): Byte
|
||||
|
||||
fun listByteValue(): List<Byte>
|
||||
|
||||
fun intValue(): Long
|
||||
|
||||
fun listIntValue(): List<Long>
|
||||
|
||||
fun boolValue(): Boolean
|
||||
|
||||
fun listBoolValue(): List<Boolean>
|
||||
|
||||
fun attributeValueType(): AttributeValueType
|
||||
|
||||
fun internalAttributeDef(): ATTRIBUTE_TYPE
|
||||
|
||||
fun internalAttributeValue(): ATTRIBUTE_VALUE_TYPE
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Appendix D: A theoretical kotlin definition of argument descriptors and op descriptors can be found below:
|
||||
```kotlin
|
||||
interface IRArgDef<T,DATA_TYPE> {
|
||||
fun name(): String
|
||||
|
||||
fun description(): String
|
||||
|
||||
fun dataType(): IRDataType<DATA_TYPE>
|
||||
|
||||
fun internalValue(): T
|
||||
|
||||
fun indexOf(): Integer
|
||||
}
|
||||
|
||||
interface IROpDef<T,ARG_DEF_TYPE,DATA_TYPE,ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE> {
|
||||
fun opName(): String
|
||||
|
||||
fun internalValue(): T
|
||||
|
||||
fun inputArgs(): List<IRArgDef<ARG_DEF_TYPE,DATA_TYPE>>
|
||||
|
||||
fun outputArgs(): List<IRArgDef<ARG_DEF_TYPE,DATA_TYPE>>
|
||||
|
||||
fun attributes(): List<IRAttribute<ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE>>
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
##Appendix E: A theoretical kotlin definition of Mapping Rules, MappingProcess and ArgDef can be found below:
|
||||
```kotlin
|
||||
interface MappingProcess<T,TENSOR_TYPE,ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE,DATA_TYPE> {
|
||||
fun opName(): String
|
||||
|
||||
fun frameworkVersion(): String
|
||||
|
||||
fun inputFramework(): String
|
||||
|
||||
fun rules(): List<MappingRule<ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE>>
|
||||
|
||||
|
||||
fun applyProcess(inputNode: IRNode<T,TENSOR_TYPE,ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE,DATA_TYPE>): OpDeclarationDescriptor
|
||||
|
||||
fun applyProcessReverse(input: OpDeclarationDescriptor): IRNode<T,TENSOR_TYPE,ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE,DATA_TYPE>
|
||||
|
||||
fun createDescriptor(argDescriptors: List<OpNamespace.ArgDescriptor>): OpDeclarationDescriptor
|
||||
}
|
||||
|
||||
interface MappingRule<ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE> {
|
||||
fun name(): String
|
||||
|
||||
/**
|
||||
* Convert 1 or more attributes in to a list of {@link ArgDescriptor}
|
||||
*/
|
||||
fun convert(inputs: List<IRAttribute<ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE>> ): List<OpNamespace.ArgDescriptor>
|
||||
|
||||
fun convertReverse(input: List<OpNamespace.ArgDescriptor>): List<IRAttribute<ATTRIBUTE_TYPE,ATTRIBUTE_VALUE_TYPE>>
|
||||
|
||||
}
|
||||
|
||||
```
|
||||
@@ -0,0 +1,81 @@
|
||||
# Interpreter
|
||||
|
||||
## Status
|
||||
Rejected
|
||||
|
||||
Proposed by: Adam Gibson (28-09-2020)
|
||||
|
||||
Discussed with: N/A
|
||||
|
||||
## Context
|
||||
|
||||
|
||||
## Decision
|
||||
|
||||
An interpreter uses the [import IR](./0003-Import_IR.md) and the [mapping rule IR](./0004-Mapping_IR.md)
|
||||
to execute and map operations from one framework to nd4j's file format and back.
|
||||
|
||||
This also allows execution of different frameworks via conversion in the nd4j engine.
|
||||
|
||||
|
||||
A combination of the 2 allows a uniform interface to be used for the interpreter.
|
||||
|
||||
1 or more MappingRules will be used to transform 1 file format to another.
|
||||
|
||||
|
||||
## Mapping Rules Execution
|
||||
|
||||
Mapping Rules are named functions that contain the function signature
|
||||
(input and outputs). These mapping rules are used by the interpreter
|
||||
to know which functions to execute.
|
||||
|
||||
The interpreter has built in implementations of the defined functions
|
||||
for the desired transforms.
|
||||
|
||||
|
||||
## Import process
|
||||
|
||||
An import process is defined for an overall framework.
|
||||
It maps input graphs to samediff graphs using
|
||||
specified mapping processes for op names and frameworks.
|
||||
An import process is all that is needed to create a graph.
|
||||
Below are the needed concepts for an import process to implement.
|
||||
|
||||
|
||||
## Graph creation
|
||||
|
||||
In order for execution to happen, a graph needs to be built.
|
||||
This happens in java via the samediff builder.
|
||||
|
||||
The conversion happens as follows:
|
||||
input node -> convert node to op descriptor via defined mapping rules -> add op descriptor to graph
|
||||
|
||||
The op descriptor is converted to a CustomOp which is then added to the graph via
|
||||
[addArgsFor](https://github.com/KonduitAI/deeplearning4j/blob/88d3c4867fb87ec760b445c6b9459ecf353cec47/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/samediff/SameDiff.java#L1078)
|
||||
|
||||
This handles declarative graph creation setting dependencies up. Delegation of the graph structure
|
||||
creation to the existing Samediff library enables the scope of this interpreter to be focused on
|
||||
mapping operations.
|
||||
|
||||
## Custom Sub graphs
|
||||
|
||||
One common use case is mapping sub graphs to custom layers. A custom layer can be thought of as a sequence of operations.
|
||||
In order to map this, a named process can be created. Generally, if you know what ops the sub graph is made of,
|
||||
you only need to declare a set of rules based on the rules that map individual ops in the existing framework.
|
||||
|
||||
## Consequences
|
||||
### Advantages
|
||||
* Uses a common interface across different frameworks making maintenance simple
|
||||
|
||||
* Allows an easy to maintain abstraction for interop with different file formats
|
||||
|
||||
* Allows an easy entry point in to the framework without knowing much about the framework.
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* Need to ensure compatibility across different frameworks
|
||||
|
||||
* Requires extensive testing to ensure proper compatibility
|
||||
|
||||
* May not necessarily support all ops people are expecting. This will be addressed
|
||||
in a new ADR.
|
||||
@@ -0,0 +1,66 @@
|
||||
# Junit 5 tag usage
|
||||
|
||||
## Status
|
||||
Proposed
|
||||
|
||||
Proposed by: Adam Gibson (21-03-2021)
|
||||
|
||||
Discussed with: N/A
|
||||
|
||||
## Context
|
||||
DL4J was a junit 4 based code based for testing.
|
||||
It's now based on junit 5's jupiter API, which has support for [Tags](https://junit.org/junit5/docs/5.0.1/api/org/junit/jupiter/api/Tag.html).
|
||||
|
||||
DL4j's code base has a number of different kinds of tests that fall in to several categories:
|
||||
1. Long and flaky involving distributed systems (spark, parameter-server)
|
||||
2. Code that requires large downloads, but runs quickly
|
||||
3. Quick tests that test basic functionality
|
||||
4. Comprehensive integration tests that test several parts of a code base
|
||||
|
||||
Due to the variety of behaviors across different tests, it's hard to tell what's actually needed
|
||||
for running and validating whether changes work against such a complex test base.
|
||||
|
||||
Much of the time, most of the tests aren't related to a given change.
|
||||
Often times, quick sanity checks are all that's needed in order to make sure a change works.
|
||||
|
||||
A common set of tags is used to filter which tests are needed to run when.
|
||||
This allows us to retain complex integration tests and run them on a set schedule
|
||||
to catch regressions while allowing a defined subset of tests to run for a quick feedback loop.
|
||||
|
||||
|
||||
|
||||
|
||||
## Decision
|
||||
|
||||
A few kinds of tags exist:
|
||||
1. Time based: long-time,short-time
|
||||
2. Network based: has-download
|
||||
3. Distributed systems: spark, multi-threaded
|
||||
4. Functional cross-cutting concerns: multi module tests, similar functionality (excludes time based)
|
||||
5. Platform specific tests that can vary on different hardware: cpu, gpu
|
||||
6. JVM crash: (jvm-crash) Tests with native code can crash the JVM for tests. It's useful to be able to turn those off when debugging.: jvm-crash
|
||||
7. RNG: (rng) for RNG related tests
|
||||
8. Samediff:(samediff) samediff related tests
|
||||
9. Training related functionality
|
||||
|
||||
|
||||
## Consequences
|
||||
### Advantages
|
||||
* Ability to sort through and filter tests based on different running environments
|
||||
|
||||
* Ability to reason about test suites as a whole dynamically across modules
|
||||
|
||||
* Avoid the need to define test suites
|
||||
|
||||
* Ability to define groups of tags based in profiles
|
||||
|
||||
* Ability to dynamically filter tests from the maven command line
|
||||
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* Documentation and maintenance burden needing to know what tags do what
|
||||
|
||||
* Test maintenance for newcomers who may not know how to tag tests
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# Nd4j Classifiers
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
|
||||
Proposed by: Adam Gibson (5-5-2021)
|
||||
|
||||
Discussed with: saudet (Samuel Audet)
|
||||
|
||||
## Context
|
||||
Nd4j relies upon the c++ library [libnd4j](../libnd4j) for native math execution.
|
||||
It uses [javacpp](https://github.com/bytedeco/javacpp) to link against
|
||||
libnd4j. Libnd4j is capable of being compiled a myriad of different ways allowing different trade offs to be made
|
||||
in terms of performance and dependencies. This presents complexity in exposing this flexibility to the end user.
|
||||
|
||||
|
||||
|
||||
## Decision
|
||||
|
||||
In order to allow users to pick which configuration they would like to use, while avoiding adding a lot of different artifact
|
||||
ids to the project, the following javacpp platform extensions are used:
|
||||
compiled type (avx etc or blank if normal) - software linked against (cudnn, onednn, armcompute) - version
|
||||
|
||||
|
||||
An example for the one dnn platform extension could be:
|
||||
dnnl-2.2
|
||||
avx256-dnnl-2.2
|
||||
|
||||
This presents 2 examples where a special compilation is enabled and one where it's not
|
||||
both linking against dnnl/onednn 2.2.
|
||||
|
||||
|
||||
## Discussion
|
||||
|
||||
Saudet: Javacpp's extensions can actually support optional
|
||||
inclusion. It plays well with other platform declarations.
|
||||
This means you can use platform like:
|
||||
```bash
|
||||
mvn -Djavacpp.platform.extension=-avx512 -Djavacpp.platform=windows-x86_64 clean ...
|
||||
```
|
||||
|
||||
to enable certain extensions.
|
||||
|
||||
## Consequences
|
||||
### Advantages
|
||||
* Adds more extensions than the previous release. This allows nd4j-native/cuda to play nice
|
||||
with the standard javacpp.platform scheme when [reducing the number of dependencies](https://github.com/bytedeco/javacpp-presets/wiki/Reducing-the-Number-of-Dependencies)
|
||||
while also enabling the new accelerated extensions, but in an optional manner.
|
||||
|
||||
* Allows users to pick how they want libnd4j to be included in their build
|
||||
|
||||
* Maintains 2 artifact ids people have to know without too much extensive knowledge.
|
||||
|
||||
* Allows us to keep a sane default for people with optimizations being optional
|
||||
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* Could be deprecated in the future depending on how libnd4j evolves
|
||||
|
||||
* Complexity for the user with the number of new extensions to be used.
|
||||
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# Nd4j eager shape computation
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
|
||||
Proposed by: Adam Gibson (11-19-2021)
|
||||
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
## Context
|
||||
Nd4j's model import framework often has the need to
|
||||
compute shapes as variables are created.
|
||||
|
||||
This is in order to resolve how to properly
|
||||
create a graph based on a graph descriptor from another framework
|
||||
such as tensorflow or pytorch.
|
||||
|
||||
This is often called eager mode. This proposal focuses on just eager shape computation
|
||||
intended for use in model import. The assumption is that we could
|
||||
build on this later for fully eager computation.
|
||||
|
||||
|
||||
## Decision
|
||||
|
||||
In order to aid building model import easier,
|
||||
this proposal is focused on implementing just dynamic shape computation
|
||||
for use in the model import context.
|
||||
|
||||
This will be composed of a few parts:
|
||||
|
||||
1. Each outputVariables() call in SDVariable triggers
|
||||
an Nd4j.getExecutioner().exec(..) call on the relevant operation
|
||||
to extract out op shapes. It then sets the appropriate shapes
|
||||
based on the result for each SDVariable field.
|
||||
|
||||
|
||||
2. This will intentionally include dummy calls for control flow ops
|
||||
such as if, enter, and while. Shapes from these don't matter
|
||||
beyond knowing the number of outputs.
|
||||
|
||||
|
||||
3. Each SameDiff instance will have an eager mode boolean
|
||||
that will determine whether this functionality is invoked.
|
||||
This eager mode variable will be required for some model import use cases.
|
||||
Usually the model import framework will turn eager on as needed
|
||||
without the user needing to be involved.
|
||||
|
||||
|
||||
4. Each SameDiff instance will have a separate ArrayHolder
|
||||
that will be used for looking up ndarrays relevant
|
||||
to the eager computation. This will not use proper sessions
|
||||
but instead store that will be used once for computing shapes.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Discussion
|
||||
Paul: Originally we would need full eager mode
|
||||
|
||||
Adam: We don't need a fully implemented eager mode
|
||||
just for model import. Full eager mode would mean proper session support,
|
||||
training support. This would just be incremental shape calculations
|
||||
for model import.
|
||||
|
||||
## Consequences
|
||||
### Advantages
|
||||
|
||||
* Allows more model import flexibility
|
||||
* Adds a base for real eager mode later on
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* Adds more complexity to model import with the addition
|
||||
dynamic shape calculations during model import
|
||||
|
||||
* Could be hard to debug if you want to see the full would be imported graph
|
||||
when a computation is blocking it
|
||||
|
||||
* The import workflow has more state attached to it
|
||||
with the eager array holder attached to each samediff instance
|
||||
and the need for another flag for turning the feature on/off
|
||||
@@ -0,0 +1,72 @@
|
||||
# Import node pre processing
|
||||
|
||||
## Status
|
||||
Discsusion
|
||||
|
||||
Proposed by: Adam Gibson (11-25-2021)
|
||||
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
## Context
|
||||
Nd4j's model import framework supports different protobuf based frameworks
|
||||
for importing and executing models. This was introduced in [0003-Import_IR.md](0003-Import_IR.md)
|
||||
One problem with importing models is compatibility between different versions of frameworks.
|
||||
Often,migrations are needed to handle compatibility between versions. A node pre processor is proposed
|
||||
that: when combined with the model import framework allows for
|
||||
annotation based automatic upgrades of graphs.
|
||||
|
||||
## Decision
|
||||
|
||||
In order to handle preprocessing a node to handle things like upgrades.
|
||||
An end user can specify a pre processor via a combination of 2 interfaces:
|
||||
1. An annotation for specifying a class that implements a relevant rule
|
||||
for processing. This will automatically be discoverable via annotation scanning
|
||||
similar to other frameworks. This annotation looks as follows:
|
||||
```kotlin
|
||||
annotation class NodePreProcessor(val nodeTypes: Array<String>, val frameworkName: String)
|
||||
|
||||
```
|
||||
The information include the nodeTypes which are the operation types to scan for when doing upgrades on a graph.
|
||||
The framework name: relevant if multiple import modules are on the classpath. Filters rules
|
||||
by their intended framework for import.
|
||||
|
||||
2. The necessary pre processing hook that will handle processing the node
|
||||
and may modify the graph. Graph modification maybe necessary if we need to add new nodes to compensate
|
||||
for modification of a node such as an attribute moving to being an input.
|
||||
|
||||
```kotlin
|
||||
|
||||
interface NodePreProcessorHook<NODE_TYPE : GeneratedMessageV3,
|
||||
TENSOR_TYPE : GeneratedMessageV3,
|
||||
ATTRIBUTE_TYPE : GeneratedMessageV3,
|
||||
ATTRIBUTE_VALUE_TYPE : GeneratedMessageV3, DATA_TYPE>
|
||||
where DATA_TYPE: ProtocolMessageEnum {
|
||||
|
||||
|
||||
fun modifyNode(
|
||||
node: IRNode<NODE_TYPE, TENSOR_TYPE, ATTRIBUTE_TYPE, ATTRIBUTE_VALUE_TYPE, DATA_TYPE>,
|
||||
graph: IRGraph<GeneratedMessageV3, GeneratedMessageV3, GeneratedMessageV3, GeneratedMessageV3, GeneratedMessageV3, GeneratedMessageV3, ProtocolMessageEnum>
|
||||
): IRNode<NODE_TYPE, TENSOR_TYPE, ATTRIBUTE_TYPE, ATTRIBUTE_VALUE_TYPE, DATA_TYPE>
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Discussion
|
||||
|
||||
## Consequences
|
||||
### Advantages
|
||||
|
||||
* An automatic way of extending support for model import by providing users a
|
||||
hook mechanism for handling graph modification
|
||||
|
||||
* Extends the model import process to handle nodes in an op specific way
|
||||
allowing a way of handling op specific interactions simplifying maintenance
|
||||
of other aspects of the import framework
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* Adds a 3rd kind of hook for model import, thus more for users to learn
|
||||
* Can be difficult to implement if user doesn't know how to work with graphs
|
||||
* May have unforeseen consequences of testing due to graph modification
|
||||
after creation
|
||||
@@ -0,0 +1,98 @@
|
||||
# Testing
|
||||
|
||||
## Status
|
||||
**Proposed**
|
||||
|
||||
Proposed by: Adam Gibson (13th December 2021)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
Testing historically on a large code base like deeplearning4j often involves
|
||||
platform specific code with several categories as documented in
|
||||
[the Test Architectures ADR](./0006 - Test architecture.md)
|
||||
|
||||
|
||||
There are multiple levels of testing which test ever larger chunks of the application.
|
||||
|
||||
**Unit tests** typically test code at the smallest possible unit. In Java this usually encompasses just a single class.
|
||||
|
||||
**Component tests** are meant to test a component consisting of multiple units working together. A logical component
|
||||
usually consists of a few classes at most and don't cross the boundary between two components.
|
||||
|
||||
**Integration tests** are meant to test how components integrate with each other. Their most important job is to ensure
|
||||
that those components properly interface with each other.
|
||||
|
||||
**End-to-End tests**, often also called system tests, are meant to test the entire system. They interface with the
|
||||
application through the same UI as a regular user does.
|
||||
|
||||
**Regression tests** are meant to mimic a specific behavior or usage that results in a bug. They are created before
|
||||
the bug fix and need to reproduce the bug but expect the correct behavior, i.e. they should fail at first. Once the bug
|
||||
is fixed, they should pass without any change in the test definition. These test cases accumulate as bug reports come in
|
||||
and guard us from recreating that particular bug in that particular situation.
|
||||
|
||||
|
||||
The Eclipse Deeplearning4j project has mostly what we would call End-To-End tests.
|
||||
We want to run a set of tests on different classifiers (eg: cuda version + cudnn, cuda version + non cudnn, cpu, arm32,arm64,..)
|
||||
in order to verify platform specific behavior works as intended.
|
||||
|
||||
When testing, we generally have a few things we test the behavior of:
|
||||
1. Compatibility across backends
|
||||
2. Performance
|
||||
3. Regressions in behavior (gradient checks failing, ops providing wrong results)
|
||||
4. Different runtime tests: standalone, spark
|
||||
|
||||
|
||||
Verifying behavior across these different backends even at release time
|
||||
is time-consuming and error-prone taking hours to run with some tests
|
||||
being inconsistent (oftentimes spark and multi threading clashing with OMP math threads causing crahses/slowdowns)
|
||||
|
||||
|
||||
## Proposal
|
||||
|
||||
We put anything that is considered an end-to-end test requiring platform
|
||||
specific behavior in to its own module.
|
||||
These tests would already be tagged. We would have an accompanying pom.xml
|
||||
that accommodated downloading snapshots to allow us to run specified tests
|
||||
on different classifiers.
|
||||
|
||||
The goal would be to allow specifying the following parameters from the command line:
|
||||
1. Classifiers to run
|
||||
2. Groups of tests to run
|
||||
3. Version to run (defaults to the latest snapshots)
|
||||
|
||||
|
||||
Future work may extend this behavior to add performance tests as well.
|
||||
|
||||
The intended workflow would be to allow the following steps:
|
||||
1. Clone the code base
|
||||
2. Cd in to the test module
|
||||
3. Specify the combination of tests you want to run on which platform
|
||||
|
||||
This allows easy configuration on CI and creation of different scripts for validation
|
||||
along the lines of behavior we want to run. Examples include:
|
||||
1. run model import tests (keras, tensorflow, onnx)
|
||||
2. Run spark tests
|
||||
3. Run basic dl4j tests
|
||||
|
||||
|
||||
These distinctions would be achieved through a mix of test tags and test
|
||||
name filters.
|
||||
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
* Tests become more accessible
|
||||
* It becomes much easier to set up test suites to be run on different classifiers on CI
|
||||
as a recurring job
|
||||
* Release testing/validation on specific platforms like embedded pis, nanos don't require you to build the
|
||||
binaries, but instead you can just download them and run binaries cross compiled on CI to verify behavior
|
||||
* Allows specifying older versions of library as necessary
|
||||
|
||||
### Disadvantages
|
||||
* Lose old behavior with tests breaking old assumptions causing contributors
|
||||
to learn a specific way of running tests
|
||||
* Requires discipline when tagging tests
|
||||
* A fairly complex pom.xml will be required for flexibly running tests
|
||||
@@ -0,0 +1,129 @@
|
||||
# Model Hub Zoo - Download
|
||||
|
||||
## Status
|
||||
**Discussion**
|
||||
|
||||
Proposed by: Adam Gibson (1st Jan 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
Model zoos or hubs are web services from different vendors to provide
|
||||
a venue for researchers and engineering teams who want to open source their
|
||||
work to publish models. The use case is typically to finetune models.
|
||||
|
||||
Finetuning models means adapting a model trained for one task to generalize for another
|
||||
by replacing its objective.
|
||||
|
||||
Coupling this with binary file formats, distributing model files typically happens
|
||||
as large binary files + some optional metadata. In the case of tensorflow and onnx
|
||||
it's using protobuf. With pytorch it uses python pickle archives.
|
||||
|
||||
Model hubs provide SDKs for downloading and using models within python.
|
||||
|
||||
|
||||
|
||||
|
||||
## Proposal
|
||||
|
||||
The goal is to interop with these model hubs using an integrated python library
|
||||
and add the appropriate tooling for converting these models to something consumable
|
||||
by the model import framework.
|
||||
|
||||
|
||||
A user is able to download models with a standard python interface using
|
||||
ModelHub. A ModelHub implementation might look like:
|
||||
```python
|
||||
class ModelHub(object):
|
||||
def __init__(self):
|
||||
self.hub_url = ''
|
||||
self.framework_name = ''
|
||||
|
||||
def download_model(self,path: string):
|
||||
...
|
||||
|
||||
def stage_model(self,model):
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
The storage type will be specific to the model hub. The concrete functions
|
||||
enums like this have is to specify to the underlying web service what kind of model we want.
|
||||
|
||||
|
||||
|
||||
In order to load a model, a model hub tends to provide different ways of
|
||||
downloading a model. This can be via a compressed archive or uncompressed.
|
||||
|
||||
In order to use this we need to be able to specify the access type.
|
||||
This will be an enum such as:
|
||||
|
||||
```python
|
||||
enum StorageType {
|
||||
COMPRESSED,UNCOMPRESSED
|
||||
}
|
||||
```
|
||||
Loading a model will be done using either samediff or deeplearning4j.
|
||||
This will leverage and extend the existing work in the model import
|
||||
work built previously.
|
||||
|
||||
|
||||
### Storage directory
|
||||
|
||||
Every model downloaded by this interface will be stored in an uncompressed
|
||||
format (.onnx,.pb,..) files with their original names under a standard unified directory
|
||||
separated by framework. This ensures ease of use and debugging
|
||||
in case a user wants to directly import a model or view
|
||||
it in a model viewer like netron.
|
||||
|
||||
This directory will default to a .modelhub directory under $USER.
|
||||
A user can also override this directory with a MODELHUB_PATH
|
||||
environment variable.
|
||||
|
||||
Note that the models will be stored as duplicates copied under
|
||||
the $MODELHUB_PATH. The reason for this is to preserve
|
||||
the original framework's underlying model in case a user
|
||||
needs to work around bugs or needs to work with the underlying
|
||||
libraries in a separate environment.
|
||||
|
||||
This also has the added benefit of allowing a previous model cache from the underlying
|
||||
libraries to avoid download. In this case, models will just be
|
||||
stored under the $MODELHUB_PATH in the form they need to be in
|
||||
to work with the model import framework.
|
||||
|
||||
|
||||
### Staging models
|
||||
|
||||
Every model will be downloaded by its original SDK and then preprocessed.
|
||||
We will call this staging. Staging is a secondary step that takes
|
||||
each model and adjusts it to be its end form that can be worked with.
|
||||
|
||||
For example in tensorflow, we may need to freeze models first
|
||||
before storing them for use with import.
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
* Allow interop of different ecosystems
|
||||
* Provide a foundation for finetuning models (finetuning models is out of the scope of this ADR)
|
||||
* Provide a way to download and manage models for testing model import functionality
|
||||
* Provide a way to enhance the built in dl4j model zoo by importing models from other ecosystems
|
||||
in a standardized way
|
||||
* Unified way of downloading and accessing models for multiple frameworks
|
||||
* Standardized directory allowing models to be easily worked with.
|
||||
* Keeps underlying framework models around for debugging. Also prevents underlying libraries
|
||||
from re-downloading libraries and benefiting from already downloaded models
|
||||
if a user uses the underlying model libraries elsewhere.
|
||||
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* More complexity in maintaining an ongoing SDK for downloading and maintaining models in different ecosystems
|
||||
* Potential storage complexity involved in running and maintaining/testing models
|
||||
* No way to know when a model hub changes
|
||||
* Loading and importing models from different ecosystems can be messy. Additional work may need to be done per model
|
||||
in order to make them usable. That additional work is out of the scope of this ADR.
|
||||
* More storage on a user's system due to the secondary cache
|
||||
@@ -0,0 +1,122 @@
|
||||
# Model Hub Zoo Download Implementation
|
||||
|
||||
## Status
|
||||
**Discussion**
|
||||
|
||||
Proposed by: Adam Gibson (1st Jan 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
Following on from work in [downloading models](0011%20-%20Model%20Hub-Zoo%20Download.md)
|
||||
We need to be able to interop in different ecosystems. This ADR will address the
|
||||
specs for implementing interop with the following ecosystems:
|
||||
1. [Onnx](https://github.com/onnx/models/)
|
||||
2. [Tensorflow](https://www.tensorflow.org/hub)
|
||||
3. [Huggingface](https://huggingface.co/spaces)
|
||||
4. [Pytorch model zoo](https://pytorch.org/serve/model_zoo.html)
|
||||
5. [Keras applications](https://keras.io/api/applications/)
|
||||
|
||||
## Proposal
|
||||
|
||||
This proposal will be broken up in to separate sections detailing
|
||||
the work and implementation needed to implement the loading of models
|
||||
from each of these ecosystems.
|
||||
|
||||
Each section will cover how to implement the download and loading of model
|
||||
download workflow described in [downloading models](0011%20-%20Model%20Hub-Zoo%20Download.md)
|
||||
|
||||
We will also cover how we will handle staging of models for each framework.
|
||||
|
||||
|
||||
### Onnx
|
||||
|
||||
Onnx is pretty straightforward as a github repo download.
|
||||
These models do not have any special structure beyond the zip file.
|
||||
Our downloader will focus on the already uncompressed models
|
||||
for ease of simplicity.
|
||||
|
||||
### Tensorflow
|
||||
|
||||
Tensorflow will use the tf hub web service. Our access will be focused
|
||||
on using the uncompressed pb models + handling other conversion code
|
||||
for freezing models to be reused.
|
||||
|
||||
For our purposes a staged model will be a frozen model
|
||||
that can be directly imported.
|
||||
|
||||
|
||||
### Pytorch
|
||||
|
||||
Pytorch will need to be converted to onnx. Pytorch serving uses
|
||||
the [model archive tool](https://github.com/pytorch/serve/tree/master/model-archiver/)
|
||||
for handling model storage.
|
||||
|
||||
Unfortunately, this requires a bit of to integrate with.
|
||||
Pytorch serve archives can vary in format. We will typically
|
||||
want to extract the model from it to manipulate it.
|
||||
|
||||
Separately, pytorch has various model zoos both official and community provided:
|
||||
1. [Example of community provided](https://github.com/rwightman/pytorch-image-models)
|
||||
2. [Torchvision model zoo](https://pytorch.org/vision/stable/models.html)
|
||||
3. [Pytorch hub](https://pytorch.org/hub/)
|
||||
|
||||
At the end, we will want to convert the model to onnx.
|
||||
This will be considered a staged model that is consumable
|
||||
by the framework.
|
||||
|
||||
|
||||
|
||||
|
||||
### Huggingface
|
||||
|
||||
Huggingface spaces uses git repositories to store models.
|
||||
URLs are accessible using the [huggingface hub SDK](https://huggingface.co/docs/hub/how-to-downstream)
|
||||
|
||||
Huggingface hub supports 3 frameworks: pytorch, tensorflow, JAX
|
||||
|
||||
Our initial support will only focus on pytorch and tensorflow.
|
||||
JAX will come at a later date when we have implemented JAX
|
||||
for the model import framework.
|
||||
|
||||
When loading a model, we will need to know which model type we are running
|
||||
so we can convert it to onnx. We will know this by letting a user specify
|
||||
the model type when they go to download it.
|
||||
|
||||
|
||||
For each of tensorflow and pytorch we will be storing models under their respective
|
||||
frameworks reusing the staging techniques from the tensorflow and onnx frameworks.
|
||||
|
||||
Huggingface paths should be repositories + the framework_name specifier.
|
||||
We use AutoModel(pytorch) and AutoTFModel(tensorflow) for converting models
|
||||
to onnx and saved_model -> pb respectively.
|
||||
|
||||
### Keras
|
||||
|
||||
Keras applications are simple archives that contain .h5 files.
|
||||
We will use the keras applications library to download and cache the models.
|
||||
|
||||
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Greatly strengthens our ability to test and execute models
|
||||
needed for different use cases
|
||||
* Allows us to be flexible in what we can enable users to do with
|
||||
our framework as a starting point
|
||||
* Further, builds out the work from downloading models and greatly increases
|
||||
the testing allowed for our model import framework
|
||||
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* Different work is needed for each ecosystem
|
||||
* APIs may change and need to be updated
|
||||
* Just pre-processing models does not mean they are guaranteed to be imported. Additional
|
||||
work will need to be done on model import to allow models to execute. This comes with additional validation work.
|
||||
* Not a comprehensive solution, users will still need to know things like the inputs and outputs
|
||||
and may still need to refer to the underlying docs for a given model to use effectively.
|
||||
* A user may still need to understand how to pre-process different kinds of models
|
||||
@@ -0,0 +1,79 @@
|
||||
# Model Hub Zoo Download Implementation
|
||||
|
||||
## Status
|
||||
**Accepted**
|
||||
|
||||
Proposed by: Adam Gibson (3rd Jan 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
In order to properly consume models from OmniHub a proper interface
|
||||
is needed to allow people to download models. Due to the sheer volume
|
||||
of models out there on the market now a scalable way of interfacing with
|
||||
various pretrained models from java is needed.
|
||||
|
||||
## Proposal
|
||||
|
||||
An interface allowing people to create pretrained model instances
|
||||
using a new interface. Note that this will supplant the old dl4j
|
||||
model zoo. A user will be able to download models via the following interface:
|
||||
|
||||
```java
|
||||
//access the samediff interface to create a resnet18 model
|
||||
Pretrained.samediff().resnet18(..).create();
|
||||
|
||||
|
||||
```
|
||||
|
||||
A user will have 2 interfaces from the Pretrained namespace class:
|
||||
samediff() and dl4j().
|
||||
|
||||
|
||||
Similar to the [codegen work](../contrib/codegen-tools/codegen)
|
||||
this will use poet to generate interfaces that are dynamically generated
|
||||
from a DSL. This DSL uses kotlin and poet.
|
||||
|
||||
We download model configurations and setup code that downloads a model
|
||||
and instantiates it in the user's code. These models will be loaded locally
|
||||
using a .omnihub directory downloaded from a pre specified github repo.
|
||||
|
||||
|
||||
Both the directory and the URL of the model zoo will be configurable
|
||||
using environment variables. These environment variables + default values will be:
|
||||
```bash
|
||||
export OMNIHUB_HOME="$USER/.omnihub"
|
||||
export OMNIHUB_URL="https://media.githubusercontent.com/media/KonduitAI/omnihub-zoo/main"
|
||||
```
|
||||
The default directory will be under $HOME/.omnihub/samediff
|
||||
$HOME/.omnihub/dl4j for downloaded models.
|
||||
|
||||
Note this is the same directory as the python omnihub downloads
|
||||
specified in [zoo download](./0011%20-%20OmniHub-Zoo%20Download.md)
|
||||
Samediff and dl4j will be subdirectories for converted models to be downloaded to.
|
||||
|
||||
Underneath the covers each Pretrained namespace will point to different
|
||||
URL sub folders to resolve models from. dl4j() will point to a /dl4j folder
|
||||
and samediff() will point to a /samediff folder.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Java will be used to consume models and allow for creation of the model zoo
|
||||
repository using git lfs as the base
|
||||
* Increases the number of models have access to without needing to convert them manually
|
||||
|
||||
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* Workflow for converting models is not completely automated and requires
|
||||
manual curation
|
||||
* The tool isn't a complete solution to adding new models as they come out
|
||||
* Models found are not guaranteed to be converted and may need manual interference
|
||||
@@ -0,0 +1,57 @@
|
||||
# Replace old model zoo
|
||||
|
||||
## Status
|
||||
**Accepted**
|
||||
|
||||
Proposed by: Adam Gibson (3rd Jan 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
The deeplearning4j-zoo module has been around for a long time
|
||||
and provides a few models out of the box. It also relies on manually implementing models
|
||||
and does not allow deeplearning4j to benefit from the innovation happening in the space.
|
||||
|
||||
There is also no samediff support in this current model zoo.
|
||||
|
||||
|
||||
## Proposal
|
||||
Replace the model zoo with omnihub. Migrate existing models
|
||||
from the azure hosting to the omnihub github repo at:
|
||||
https://github.com/KonduitAI/omnihub-zoo
|
||||
|
||||
This will allow users to selectively download pretrained models
|
||||
from the deeplearning4j zoo.
|
||||
|
||||
|
||||
Redirect all URL calls in the old zoo to the new one
|
||||
to prevent disruption of user's workflows.
|
||||
|
||||
Extend deeplearning4j-zoo's ZooModel to support the new
|
||||
omnihub zoo.
|
||||
|
||||
Provide a bridge interface to ZooModel with OmniHubZooModel
|
||||
allowing for seamless transition and expansion of new models.
|
||||
|
||||
Migrate off of azure storage for the models saving costs
|
||||
and reducing complexity.
|
||||
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Reduced maintenance cost
|
||||
* Increased model availability for users
|
||||
* Allows for support of samediff
|
||||
* Reuse existing model zoo as is but increase support for new models
|
||||
|
||||
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* Potential bugs
|
||||
* New infra means manually uploading new models
|
||||
* Need to ensure smooth migration of ZooModel interface to seamlessly
|
||||
work with the new model zoo
|
||||
@@ -0,0 +1,94 @@
|
||||
# Replace old model zoo
|
||||
|
||||
## Status
|
||||
**Discussion**
|
||||
|
||||
Proposed by: Adam Gibson (12th Jan 2022)
|
||||
|
||||
TODO:
|
||||
1. centralized MD5 sum directory
|
||||
2. List of directories stored in a local .dl4jresources config file
|
||||
3. Configuration file format for listing directories and their types
|
||||
4. Additional checks for old directories at default values not covered by newer support
|
||||
5. Pre cataloging based on default dataset directories found from prior releases
|
||||
6.
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
A number of current downloaders exist for various resources
|
||||
deeplearning4j needs to function. These include the following:
|
||||
1. Strumpf resource resolver (manages test resources)
|
||||
and relies on azure. The original code for strumpf can be found [here](https://github.com/KonduitAI/strumpf)
|
||||
2. Deeplearning4j model zoo (the legacy model zoo)
|
||||
3. Omnihub: The new model zoo replacing #2.
|
||||
4. Deeplearning4j datasets: dataset download for various datasetiterators
|
||||
|
||||
|
||||
These have accumulated over the years and have made maintenance of download related logic
|
||||
complex.
|
||||
|
||||
Relevant ADRs include:
|
||||
[Omnihub zoo download](./0011%20-%20OmniHub-Zoo%20Download.md)
|
||||
[Omnihub zoo download implementations](./0012%20-%20OmniHub-Zoo%20Download%20Implementations.md)
|
||||
[Omnihub replace old model zoo](./0013%20-%20OmniHub-Zoo%20Consumption.md)
|
||||
|
||||
|
||||
## Proposal
|
||||
|
||||
All resources are hosted on github LFS.
|
||||
A resource abstraction for binding the various resource types in
|
||||
to 1 abstraction and downloader.
|
||||
|
||||
A Resource is how we handle this. It is be aware of the following concepts:
|
||||
1. A base url for downloading a file
|
||||
2. A cache directory for managing the resource
|
||||
3. Common download + retry logic for ensuring a download succeeds
|
||||
|
||||
|
||||
A Resource manages a remote resource like a file. Similar to the current resource types
|
||||
in deeplearning4j-common. These resources are mostly be stored on git LFS.
|
||||
|
||||
As part of this introduction of a unified resource abstraction
|
||||
is cache aware exposing the cache so users can delete if they wish.
|
||||
|
||||
For existing datasets we use the old sources but have a common abstraction
|
||||
for knowing which dataset we want to download.
|
||||
|
||||
Another problem is file verification.
|
||||
|
||||
The legacy model zoo uses simpler adler checksums for verification.
|
||||
Some download cache verification implementations use md5sum.
|
||||
|
||||
We use md5sum and standardize this for all resources.
|
||||
|
||||
|
||||
Note that in order to avoid maintenance burdens md5 checksum verification
|
||||
is optional. By default, if a resource returns null or an empty
|
||||
string verification is not performed. This distinction is important
|
||||
for resource types such as test resources vs end user assets like pretrained model
|
||||
weights.
|
||||
|
||||
This is also important for compatibility. Due to the legacy checksum
|
||||
verification in the zoo module, md5 checksum verification can come later.
|
||||
|
||||
This leads us to 5 resource types:
|
||||
1. Omnihub: The omnihub pretrained models
|
||||
2. Datasets: the legacy datasets for custom iterators like mnist and lfw
|
||||
3. Dl4j zoo: The legacy zoo models
|
||||
4. Strumpf: the legacy test resource manager
|
||||
5. Custom: custom resources where a user can specify a URL and file destination
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Reduced costs by migrating to github
|
||||
* One module for handling resources
|
||||
* Replaces legacy abstractions while preserving backwards compatibility
|
||||
* Allows easier management of local resources
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* Potential bugs
|
||||
* Migration will take time
|
||||
@@ -0,0 +1,45 @@
|
||||
# Java 9+ Support
|
||||
|
||||
## Status
|
||||
**Discussion**
|
||||
|
||||
Proposed by: Adam Gibson (7th Feb 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
With the next LTS upcoming (java 17) forcing module metadata to be present. This means supporting java 9+ modules
|
||||
with mdoule-info.java being present.
|
||||
|
||||
This ADR addresses the changes made to support java 9 modules. Many of the changes are related to adding
|
||||
[module-info.java](https://www.oracle.com/corporate/features/understanding-java-9-modules.html)
|
||||
|
||||
## Proposal
|
||||
|
||||
1. [moditect plugin](https://github.com/moditect/moditect) metadata for every module
|
||||
a. Each module has a module.name with 1 inherited declaration in the root pom for generating and adding
|
||||
module metadata.
|
||||
b. Github Workflow upgrades for generating module metadata before publishing
|
||||
c. Moditect plugin declarations are optional by default and selectively invoked
|
||||
|
||||
2. Add build steps invoking the plugin ensuring proper metadata gets added
|
||||
|
||||
3. Each maven jar plugin invocation needs a unique module name for jars that have classifiers such as
|
||||
the nd4j backends this avoids duplicate name errors when trying to invoke metadata
|
||||
|
||||
4. Proper package split package declarations enable proper module exposure. This means some classes have been moved
|
||||
around. Mainly internal classes are affected such as the preset names.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Allows easier integration in to various module systems including jigsaw and OSGI
|
||||
* Allows proper integration in to newer java versions enabling users to produce modularized applications
|
||||
* Allows better support for newer java versions
|
||||
* Our plugin approach enables backwards compatibility with java 8
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* Breaking backwards compatibility
|
||||
* May introduce new bugs due to renaming
|
||||
@@ -0,0 +1,92 @@
|
||||
# SDValue
|
||||
|
||||
## Status
|
||||
**Discussion**
|
||||
|
||||
Proposed by: Adam Gibson (8th Mar 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
Onnx and Tensorflow both support miscellaneous data structures to support the
|
||||
building and execution of data flow graphs.
|
||||
|
||||
Programming primitives such as lists, maps and optional types
|
||||
can be valuable in building data flow graphs allowing the developer to express a
|
||||
wide variety of operations encoded within a neural network.
|
||||
|
||||
Tensorflow uses [RaggedTensors](https://www.tensorflow.org/guide/ragged_tensor).
|
||||
Onnx uses [sequences](https://github.com/onnx/onnx/blob/main/docs/IR.md)
|
||||
Of note is onnx also supports optionals and maps.
|
||||
|
||||
|
||||
These primitives can actually be passed in as inputs and returned as outputs
|
||||
from a graph as well.
|
||||
|
||||
In order to simplify the usage of these primitives a value type is used
|
||||
to indicate what type of value it is being passed in (maps, lists, tensors,..)
|
||||
|
||||
|
||||
SDValue is the Samediff interpretation of this and allows [TensorArrays](./nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ops/impl/shape/tensorops/TensorArray.java) to be passed around.
|
||||
as well as other types to be used within execution of a graph.
|
||||
|
||||
|
||||
|
||||
## Proposal
|
||||
|
||||
Create an SDValue class and associated enum abstraction for passing around and manipulating
|
||||
variables of different types.
|
||||
|
||||
The following types will be supported:
|
||||
1. LIST
|
||||
2. DICT
|
||||
3. TENSOR
|
||||
|
||||
|
||||
An SDValue is passed around similar to placeholders and are utilized within InferenceSession to
|
||||
execute operations within a graph.
|
||||
|
||||
An SDValue's real underlying type will map to the following:
|
||||
1. LIST: INDArray[]
|
||||
2. DICT: Map<String,INDArray>
|
||||
3. TENSOR: INDArray
|
||||
|
||||
Method calls that will use this off of samediff and InferenceSession will be:
|
||||
```java
|
||||
sd.output(Map<String,SDValue> values,...);
|
||||
|
||||
```
|
||||
|
||||
Each of these are named values with a value type. The scope of this for execution just augmenti
|
||||
arrays of arrays or TensorArrays.
|
||||
|
||||
The below describes how these operations will work:
|
||||
|
||||
|
||||
### Operations
|
||||
The following operations on sequences are supported:
|
||||
1. addItemToSequence(String varName,INDArray item,int atIndex): add an array at a particular index
|
||||
2. removeItemFromSequence(String varName,int indexOfItem): remove an item at a particular index, note that empty variables are removed
|
||||
3. SDVariable sequence(String name,INDArray[] arrays): create the sequence with the given variable name
|
||||
4. setItemForSequenceAtIndex(String varName,INDArray item,int index): set the item at the particular index
|
||||
5. sequenceLength(String varName): returns the length of a sequence
|
||||
|
||||
|
||||
Note that all indices above also support negative indexing allowing you to index from the end of the list similar
|
||||
to python.
|
||||
|
||||
### Model import
|
||||
For model import, this also extends the work in [0004-Mapping_IR.md](0004-Mapping_IR.md)
|
||||
to support sequences/lists of tensors. This means IRTensor and similar concepts will also support sequences.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Allows flexible import of models that use sequences to pass around ndarrays
|
||||
* Allows flexible management of ndarrays using a variable name as a group.
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* Not robust enough to execute ops on
|
||||
* Still need to implement Ragged Tensors at some point separately.
|
||||
@@ -0,0 +1,67 @@
|
||||
# Invoke
|
||||
|
||||
## Status
|
||||
**Discussion**
|
||||
|
||||
Proposed by: Adam Gibson (10th April 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
A common use case in model import is the ability to modularize graphs as sub functions.
|
||||
Typically, these graphs are attributes of a node. Samediff already possesses the ability to store
|
||||
samediff graphs in a dictionary called SameDiffFunctionInstances.
|
||||
|
||||
Graphs with control flow ops such as switch, if and while also tend to heavily use sub graphs.
|
||||
|
||||
|
||||
## Proposal
|
||||
|
||||
Invoke builds on the work from [SDValue](./0018%20-%20SDValue.md) and leverages returning an ExecutionResult (a dictionary of name to SDValue)
|
||||
passing the result of the invoke function as an op output in a larger parent graph.
|
||||
|
||||
|
||||
### InvokeParams
|
||||
Invoke takes node outputs from the parent graph and passes them through renamed to match the expected inputs and outputs from the sub graph.
|
||||
This takes the form of InvokeParams:
|
||||
```java
|
||||
public static class InvokeParams {
|
||||
private String functionName;
|
||||
private SDVariable[] inputs;
|
||||
private String[] inputVarNames;
|
||||
private String[] outputVarNames;
|
||||
private String[] subGraphInputVarNames;
|
||||
private String[] subGraphOutputVarNames;
|
||||
}
|
||||
|
||||
|
||||
```
|
||||
|
||||
InvokeParams has the expected input and output names for the graph and matching subgraph input and output variable names.
|
||||
|
||||
|
||||
### Outputs
|
||||
|
||||
Invoke has a special outputVariables() method for returning output values that match the outputs of the subgraph.
|
||||
All outputs will have the same data types as the underlying result.
|
||||
|
||||
These outputs are all converted to the ARRAY type
|
||||
since the output variables are derived from the output of a function. For easy readability these op output variables will have
|
||||
the same name of the output with a _functionName where functionName is the name of the function used to lookup the subgraph
|
||||
to invoke.
|
||||
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Allows invocation of subgraphs easily as an op
|
||||
* Allows for more flexibility in how a user sets a graph up
|
||||
* Can be combined with control flow to easily build loops
|
||||
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* More complexity in a graph
|
||||
* Potential source of bugs when naming parameters and setting up the proper invoke calls
|
||||
@@ -0,0 +1,160 @@
|
||||
# Control flow
|
||||
|
||||
## Status
|
||||
**Discussion**
|
||||
|
||||
Proposed by: Adam Gibson (10th April 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
Samediff supports control flow such as if statements and while loops. However this is not enough and common looping structures
|
||||
are still hard to use. Onnx has introduced a Loop operation. Loop requires a graph with pre configured graph.
|
||||
The graph takes in and outputs:
|
||||
1. current iteration
|
||||
2. max number of iterations
|
||||
3. extra condition to use
|
||||
|
||||
It approximates a for loop with the following code:
|
||||
```java
|
||||
boolean cond = ...;
|
||||
int maxIterations = ...;
|
||||
for(int i = 0; i < maxIterations && cond; i++) {
|
||||
loop body...
|
||||
}
|
||||
|
||||
|
||||
```
|
||||
|
||||
The loop body is represented as a sub graph attribute on the operation.
|
||||
|
||||
|
||||
## Proposal
|
||||
|
||||
Similar to onnx's loop operation coupled with [Invoke](./0019%20-%20Invoke.md)
|
||||
we provide a new loop that leverages invoke and some fixed conventions of the graph to use a loop body:
|
||||
```java
|
||||
/**
|
||||
* Loop with conditions.
|
||||
* For more information see the underlying class
|
||||
* {@link ControlFlow#loopWithConditions(String[], String, SameDiff, SameDiff, String, SDVariable[], String[], String[])}
|
||||
* @param loopParams the loop parameters to loop with
|
||||
* @return
|
||||
*/
|
||||
public SDVariable[] loopWithConditions(ControlFlow.LoopParams loopParams) {
|
||||
```
|
||||
|
||||
|
||||
### LoopParams
|
||||
LoopParams looks like the following:
|
||||
```java
|
||||
public static class LoopParams {
|
||||
private String[] outputVarNames;
|
||||
private String loopName;
|
||||
private SameDiff parent;
|
||||
private SameDiff functionBody;
|
||||
private String functionName;
|
||||
private SDVariable[] loopVars;
|
||||
private String[] functionBodyInputs;
|
||||
private String[] functionBodyOutputs;
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
|
||||
LoopParams has the following fields:
|
||||
1. outputVarNames: the output variable names for the loop
|
||||
2. The name of the loop controls the name in the control flow with scopeName/loopName/variableName as the convention
|
||||
scopeName is the current frame (such as within the loop body), the loop name is the loop name showing which loop it is
|
||||
and the variableName represents the variable within the loop and frame.
|
||||
3. Parent: the invoking samediff function
|
||||
4. functionName: The name of the function to be looked up from the parent and invoked using invoke
|
||||
5. loopVars: the input and outputs of the loops
|
||||
6. functionBodyInputs: the function input names to use with Invoke
|
||||
7. functionBodyOutputs: the function output names to be returned from Invoke
|
||||
|
||||
|
||||
|
||||
### Example Usage
|
||||
|
||||
```java
|
||||
//setup the parent graph to pass inputs to the lambda
|
||||
SameDiff parent = SameDiff.create();
|
||||
SDVariable input = parent.placeHolder("input",DataType.FLOAT);
|
||||
//setup the loop body
|
||||
SameDiff loopBody = SameDiff.create();
|
||||
SDVariable loopInput = loopBody.placeHolder("input", DataType.FLOAT);
|
||||
SDVariable output = loopBody.math().add("output",loopInput,1.0);
|
||||
//initialize the control flow with the default parameters such as the current iteration, the max number of iterations and the conditional output from the graph
|
||||
SDVariable[] args = ControlFlow.initializeLoopBody(new String[]{"curr_iteration", "max_iterations", "cond_in"}, parent, 5, true);
|
||||
SDVariable[] childArgs = ControlFlow.initializeLoopBody(new String[]{"curr_iteration", "max_iterations", "cond_in"}, loopBody, 5, true);
|
||||
|
||||
//input names for the input graph with the 4th input being the input from the parent
|
||||
String[] inputNames = {
|
||||
"curr_iteration",
|
||||
"max_iterations",
|
||||
"cond_in",
|
||||
"input"
|
||||
};
|
||||
|
||||
//output names from the output of the lmabda with the 4th being the result of the lamda's application of the input
|
||||
String[] outputNames = {
|
||||
"curr_iteration",
|
||||
"max_iterations",
|
||||
"cond_in",
|
||||
"output"
|
||||
};
|
||||
|
||||
|
||||
//setup the loop variables for input
|
||||
SDVariable[] finalArgs = new SDVariable[args.length + 1];
|
||||
for(int i = 0; i < args.length; i++) {
|
||||
finalArgs[i] = args[i];
|
||||
}
|
||||
finalArgs[3] = input;
|
||||
|
||||
|
||||
//put it all together in the loop parameters
|
||||
ControlFlow.LoopParams loopParams = ControlFlow.LoopParams.builder()
|
||||
.parent(parent)
|
||||
.functionBody(loopBody)
|
||||
.functionBodyInputs(inputNames)
|
||||
.functionBodyOutputs(outputNames)
|
||||
.loopVars(finalArgs)
|
||||
.loopName("loop")
|
||||
.functionName("func")
|
||||
.build();
|
||||
|
||||
//control the output parameter names
|
||||
String[] finalOutputNames = new String[outputNames.length];
|
||||
for(int i = 0; i < finalOutputNames.length; i++) {
|
||||
finalOutputNames[i] = outputNames[i] + "_final";
|
||||
}
|
||||
|
||||
//test the output variables, the names will match the specified output names
|
||||
SDVariable[] loopWithConditions = parent.loopWithConditions(finalOutputNames,loopParams);
|
||||
|
||||
INDArray assertion = Nd4j.ones(5).addi(5);
|
||||
Map<String, INDArray> output2 = parent.output(Collections.singletonMap("input", Nd4j.ones(5)), "output_final");
|
||||
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* More explicit wrapper for loop
|
||||
* More conventions for looping
|
||||
* Easier to use
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* More parameters than while loop
|
||||
* Inherits the complexity of invoke with the number of parameters needed
|
||||
* Number of inputs and outputs must match. Might not be intuitive for the user.
|
||||
@@ -0,0 +1,76 @@
|
||||
# Create View
|
||||
|
||||
## Status
|
||||
**Discussion**
|
||||
|
||||
Proposed by: Adam Gibson (27th April 2022)
|
||||
Discussed with: Paul Dubs
|
||||
Finalized by: Adam Gibson (5th May 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
Samediff's op graph mainly focuses on immutability to prevent bugs making many variable inputs and outputs for various ops copy on write.
|
||||
|
||||
This prevents performance gains with in place ops such as +=.
|
||||
|
||||
Currently, the samediff strided_slice operation works to provide a view of an array
|
||||
but the view is a copy. This limitation prevents performance gains you typically see in views.
|
||||
|
||||
|
||||
## Decision
|
||||
|
||||
CreateView is an op that takes in a set of SDVariables that represents index information similar to nd4j's point, interval,all, and new axis. This op allows for dynamic generation of views of variables.
|
||||
|
||||
CreateView is a building block for other ops to execute in place operations. Usage of CreateView should be deliberate and only used in certain circumstances.
|
||||
CreateView simply (using the aforementioned index inputs) creates a view using the existing data buffer and returns an output that wraps the same exact buffer as is rendered as an alternative view in a similar way as nd4j's indexing mechanisms.
|
||||
|
||||
|
||||
|
||||
These inputs are represented as follows:
|
||||
|
||||
1. Interval: INTERVAL_TYPE,2,1,start,end,stride,inclusive
|
||||
2. Point: POINT_TYPE,1,1,offset, DEFAULT_INCLUSIVE
|
||||
3. All: ALL_TYPE,0,1, DEFAULT_INCLUSIVE
|
||||
4. New Axis: NEW_AXIS,1,10, DEFAULT_INCLUSIVE
|
||||
|
||||
|
||||
This describes the general pattern the above described buffers follow:
|
||||
1. type of index
|
||||
2. Number of indices (representing offsets)
|
||||
3. Stride
|
||||
4. Inclusive/exclusive
|
||||
|
||||
|
||||
Of note here are a few constants representing types to be passed to the ops:
|
||||
1. *_TYPE: a pre-defined constant representing the kind of index this is
|
||||
2. DEFAULT_INCLUSIVE: whether the index's end is inclusive or not (only needed for intervals)
|
||||
this is by default 0 most of the time since the value is only relevant for intervals.
|
||||
|
||||
These are created as INT64 ndarrays passed in to the
|
||||
operation itself.
|
||||
|
||||
An omission of indexing here is SpecifiedIndex. Since SpecifiedIndex requires a copy most of the time, this op will mainly be focused on indexing that is guaranteed to use the same buffer.
|
||||
|
||||
|
||||
## In place exception in gradient checks
|
||||
|
||||
|
||||
Usually, arrays during training should not modify their outputs. Instead, new output arrays are allocated with calculated results being inserted into these pre-defined outputs.
|
||||
|
||||
However, CreateView is, by definition, special since it is a building block for enabling manipulation of a view of the same data buffer as the input. The gradient checks in the [NonInplaceValidationListener](https://github.com/eclipse/deeplearning4j/blob/master/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/autodiff/validation/listeners/NonInplaceValidationListener.java#L43) make an exception to this rule to account for this particular behaviour.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Self contained op for creating a view leveraging nd4j's existing indexing engine but
|
||||
better integrated as a libnd4j op allowing for better dynamic indexing of arrays
|
||||
* Similar in usage to indexes
|
||||
* Contains the potential bugs from views to a pre-specified op
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* Could introduce new bugs upon use
|
||||
* Not the easiest interface requiring some constant methods
|
||||
* Not fully integrated in to the main engine/not as transparent as a tool as it should be
|
||||
@@ -0,0 +1,44 @@
|
||||
# Dynamic Indexing
|
||||
|
||||
## Status
|
||||
**Discussion**
|
||||
|
||||
Proposed by: Adam Gibson (30th April 2022)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
Dynamic indexing has a wide variety of applications for building deep learning graphs.
|
||||
|
||||
Expressing a dynamic index that gets resolved at runtime entails specifying a negative index as the desired index for an indexing operation.
|
||||
At runtime, the indexing engine will then count backwards from the element the user specified. An example being: -1 starts at the end, -2 starts at the second to last.
|
||||
|
||||
Samediff and nd4j have numpy style indexing support.
|
||||
|
||||
|
||||
Samediff does this 1 of 2 ways. Either through nd4j's indexing engine building on [previous work](./0021%20-%20Create%20View.md) or using a similar interface to nd4j's indexing engine to build a strided slice operation call.
|
||||
Previously, the indexing would not allow initialization with a negative index without passing in an ndarray. The problem with this is ndarrays are not known in samediff till execution time.
|
||||
|
||||
|
||||
|
||||
## Proposal
|
||||
|
||||
|
||||
We support the ability to dynamically resolve indices during execution. This happens as follows:
|
||||
1. Each index has a concept of initialized().
|
||||
This returns a boolean indicating whether the operation was initialized or not. Initialized means there are no negative values present in the index and a specific boolean flag has been set representing the index being fully initialized.
|
||||
2. If a negative index is specified, the index is set on the index itself but it will not be considered initialized.
|
||||
3. At runtime, upon use a deep copy of the index happens and then initialization will occur upon use relative to the ndarray using it. This currently happens at the java level. Indexing does not exist fully at the c++ level.
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Allows for more flexible indexing only possible with just in time resolution.
|
||||
* Less errors are thrown during indexing
|
||||
|
||||
|
||||
### Disadvantages
|
||||
* A bit more overhead in the indexing process
|
||||
* Harder to debug
|
||||
@@ -0,0 +1,198 @@
|
||||
# UDFs
|
||||
|
||||
## Status
|
||||
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (31 Jan 2023)
|
||||
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
Finalized by: Adam Gibson (2nd Feb 2023)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
Users should be able to define their own custom operations in SameDiff, including custom gradients.
|
||||
Currently, defining a User-Defined Function (UDF) is not properly integrated into SameDiff and requires handling multiple aspects of the system, such as:
|
||||
|
||||
1. Registering the operation with the ImportClassMappings operation registry
|
||||
2. Implementing special code paths in the operation executioners to support calling user-defined code
|
||||
3. Implementing special code paths for serialization to correctly load a SameDiff graph with the operation
|
||||
4. Making a direct function call in SameDiff to properly register the operation as part of the graph.
|
||||
|
||||
## Proposal
|
||||
To support custom UDFs in SameDiff, the following components will be created:
|
||||
|
||||
1. An operation type for the FlatBuffersMapper (used for saving graphs) to know how to save and load UDFs
|
||||
2. A base class extending DynamicCustom with clear method and constructor overrides for defining a custom operation
|
||||
3. An execution method where the user passes in relevant inputs, which will be used by the operation executioner (instead of accessing the low-level code)
|
||||
4. A hook in SameDiff to register the UDF, such as sd.udf(...)
|
||||
5. An annotation or subclass scanner to discover and register user-defined operations in relevant areas, such as the ImportClassMappings.
|
||||
|
||||
These components work together to allow for the following:
|
||||
|
||||
1. Scanning for annotations to discover and register user-defined operations with the operation registry
|
||||
2. Users extending the base class to create their custom UDFs
|
||||
3. Integrating UDFs into SameDiff by registering the operation with the graph, for example:
|
||||
java
|
||||
```java
|
||||
SameDiff sd = SameDiff.create();
|
||||
UserDefinedCustomOp userDefinedCustomOp = ...;
|
||||
SDVariable[] opOutputs = sd.doUdf(userDefinedCustomOp);
|
||||
```
|
||||
|
||||
When an operation is registered, it is saved and loaded with the graph like any other operation.
|
||||
Dynamic creation of operations via reflection when a graph is loaded, using the annotation scanning.
|
||||
|
||||
Below is an example:
|
||||
```java
|
||||
@UserDefinedOp // Annotation for discovering custom ops to register
|
||||
public class TestAddUdf extends UserDefinedCustomOp { // Class to extend
|
||||
|
||||
|
||||
// Empty constructor. Used when creating a graph from flatbuffers in the underlying { org.nd4j.autodiff.samediff.serde.FlatBuffersMapper}.
|
||||
public TestAddUdf() {
|
||||
super();
|
||||
}
|
||||
|
||||
// Other constructors can be whatever the user wishes. Custom ops usually take in a
|
||||
// SameDiff instance and one or more SDVariable args. These are the minimum components to instantiate an op.
|
||||
// Each of these calls super(...) to properly configure the op to be used within the SameDiff graph passed in.
|
||||
|
||||
public TestAddUdf(SameDiff sameDiff, SDVariable arg) {
|
||||
super(sameDiff, arg);
|
||||
}
|
||||
|
||||
public TestAddUdf(SameDiff sameDiff, SDVariable[] args) {
|
||||
super(sameDiff, args);
|
||||
}
|
||||
|
||||
// Used to calculate output variables when registering an op with a graph.
|
||||
@Override
|
||||
public List<DataType> calculateOutputDataTypes(List<DataType> dataTypes) {
|
||||
// A user must implement this method. It is used in SameDiff to determine the number of
|
||||
// output variables needed when it can't be determined from getNumOutputs().
|
||||
return Arrays.asList(dataTypes.get(0));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setPropertiesForFunction(Map<String, Object> properties) {
|
||||
// A user can define properties as fields. If so, they must implement this method and propertiesForFunction().
|
||||
// These are used to create an op from scratch when saving/loading a model.
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> propertiesForFunction() {
|
||||
// Returns properties (fields on the java class) as a map. Properties can be any value that
|
||||
// is a field on the op itself. These properties are optional and may not be needed,
|
||||
// depending on the op. All properties will end up being passed to the underlying iArguments,
|
||||
// tArguments, and other associated data structures inherited from DynamicCustomOp.
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumOutputs() {
|
||||
// Returns the number of outputs for the op. If an op has a variable number of outputs,
|
||||
// a user will need to use an SDVariable.eval() call to return an int to determine the number of outputs.
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String opName() {
|
||||
// The op name, required for proper registration with the registry.
|
||||
return "test_add_udf";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configureFromArguments() {
|
||||
// A hook for configuring the op after creation. Used for configuration from specified arguments,
|
||||
// such as ints, floats/doubles, and input variables. The arguments referenced are the underlying
|
||||
// arguments that get passed to every c/c++ ops, including iArguments, tArguments, dArguments,
|
||||
// inputArguments, and outputArguments.
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configureWithSameDiff(SameDiff sameDiff) {
|
||||
this.sameDiff = sameDiff;
|
||||
// Implemented this method for handling initialization after the op is created. It initiates values using relevant
|
||||
// SameDiff metadata, such as obtaining input and output argument metadata from SDVariable found as args().
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean isInplaceCall() {
|
||||
// Indicates whether the inputs are also the outputs.
|
||||
// Note that extra care should be taken to avoid bugs when an operation is in-place.
|
||||
// This is particularly important when an input to an operation is a view.
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public List<LongShapeDescriptor> calculateOutputShape() {
|
||||
// Describes how to calculate the output shape based on the inputs.
|
||||
// Note that calculateOutputShape is called when dynamically creating output arrays to store the result
|
||||
// of an operation's execution.
|
||||
// It is not called when an operation is in-place.
|
||||
return Arrays.asList(inputArguments.get(0).shapeDescriptor());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<LongShapeDescriptor> calculateOutputShape(OpContext oc) {
|
||||
// Describes how to calculate the output shape based on the inputs from the operation context.
|
||||
// Note that calculateOutputShape is called when dynamically creating output arrays to store
|
||||
// the result of an operation's execution.
|
||||
// This is different from the above method as the inputs are obtained from the operation
|
||||
// context instead of the operation itself.
|
||||
// It is not called when an operation is in-place.
|
||||
return Arrays.asList(oc.getInputArrays().get(0).shapeDescriptor());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SDVariable> doDiff(List<SDVariable> f1) {
|
||||
// The doDiff method must be implemented by the user if the operation is to be used for training.
|
||||
// It should return one gradient for each input.
|
||||
return new AddBpOp(sameDiff, larg(), rarg(), f1.get(0)).outputs();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exec() {
|
||||
// The exec method for the operation itself, consisting of operation execution
|
||||
// and setting the outputs for the operation.
|
||||
AddOp addOp = new AddOp();
|
||||
addOp.addInputArgument(inputArguments.get(0), inputArguments.get(1));
|
||||
Nd4j.getExecutioner().exec(addOp);
|
||||
this.outputArguments.addAll(addOp.outputArguments);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exec(OpContext opContext) {
|
||||
// The exec method for the operation itself, consisting of operation execution and
|
||||
// setting the outputs for the operation context.
|
||||
Nd4j.getExecutioner().exec(new AddOp(), opContext);
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
With the above definition, a user just has to pass in a created op as an instantiated object.
|
||||
As long as an op is annotated it is properly integrated with the samediff graph.
|
||||
|
||||
When executing, the special code paths in the op executioners will call exec() or exec(opContext)
|
||||
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Allows users to define their own ops to make optimizations or to introduce custom ops
|
||||
for use within samediff
|
||||
* Augments model import by combining annotation scanning from model import with
|
||||
annotation registration of udfs with similar annotations
|
||||
|
||||
### Disadvantages
|
||||
* A bit lower level which means users can misuse the api or encounter bugs they might not
|
||||
otherwise with ops maintained in the core framework.
|
||||
@@ -0,0 +1,67 @@
|
||||
# Graph Execution Trace Collection and Reproduction
|
||||
|
||||
## Status
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (20 Mar 2023)
|
||||
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
Finalized by: Adam Gibson (24 Mar 2023)
|
||||
|
||||
## Context
|
||||
|
||||
Reproducing a specific graph execution between the SameDiff and DL4J APIs can be
|
||||
challenging, as both use the underlying libnd4j operations to execute code.
|
||||
Currently, users enable verbose or debug mode in the op executioner to observe
|
||||
executed operations and manually compare the output of the two APIs. This method is
|
||||
suboptimal and time-consuming.
|
||||
|
||||
In the context of this proposal, the term "vector" refers to an `std::vector` in C++
|
||||
that stores the metadata of each operation execution. It does not refer to a
|
||||
mathematical vector or a tensor typically used in deep learning libraries. The
|
||||
`std::vector` is a dynamic array-like container provided by the C++ Standard Library,
|
||||
which is used here to store the sequence of operation executions.
|
||||
|
||||
## Decision
|
||||
|
||||
To improve the process, we will save execution traces in a format that can generate a
|
||||
SameDiff graph, emulating the executed steps. Once enabled, operation executions will
|
||||
be collected in a vector, storing only metadata such as input/output shapes and
|
||||
arguments for each operation. These executions will be stored in the vector
|
||||
sequentially.
|
||||
|
||||
For instance, when executing a convolution operation, we can trigger the scope in C++
|
||||
to indicate the current operation. This enables tracking the execution of the
|
||||
convolution operation and its nested operations, like the im2col operation.
|
||||
|
||||
Graph tracing can be enabled using the following command:
|
||||
```java
|
||||
Nd4j.toggleTrace(true);
|
||||
```
|
||||
|
||||
Using the vector of executions, we can reproduce a graph. To save the graph, use:
|
||||
```java
|
||||
SameDiff sd = SameDiff.collectTrace();
|
||||
sd.save(new File("mygraph.fb"));
|
||||
```
|
||||
|
||||
Afterward, purge the trace to prevent memory leaks:
|
||||
```java
|
||||
Nd4j.purgeTrace();
|
||||
```
|
||||
|
||||
When purge is done you can disable trace with:
|
||||
```java
|
||||
Nd4j.toggleTrace(false);
|
||||
```
|
||||
|
||||
## Consequences
|
||||
### Advantages
|
||||
* Simplifies graph reproduction
|
||||
* Enables decomposition of nested op execution, such as attention
|
||||
* Increases complexity when implementing ops, requiring the developer to notify the tracer of the current parent op
|
||||
|
||||
### Disadvantages
|
||||
* Generated graph may lack names or other metadata
|
||||
* Execution tracing should be performed only when executing one op at a time
|
||||
@@ -0,0 +1,183 @@
|
||||
# Workspaces
|
||||
|
||||
## Status
|
||||
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (14 Mar 2023)
|
||||
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
Neural networks require a significant amount of memory during execution, often in the range of billions of parameters.
|
||||
To improve performance and manage memory usage, we can take advantage of the fact that neural network allocations are cyclic in nature.
|
||||
Since most workloads repeatedly allocate the same ndarrays, we create a memory abstraction known as "workspaces" to avoid redundant memory allocation.
|
||||
This approach helps to optimize memory usage and enhance overall performance.
|
||||
|
||||
## Proposal
|
||||
|
||||
This architecture decision record discusses the implementation of the workspaces concept using ringbuffers within a
|
||||
namespace-like abstraction and Java's try/with resources for memory allocation and garbage collection.
|
||||
Workspaces require a configuration with several parameters for controlling memory allocation. (See the description section for more details.)
|
||||
|
||||
A MemoryManager is used to allocate an INDArray, and an operation (element-wise multiplication) is performed.
|
||||
The workspace and INDArray are automatically closed and released when the try blocks are exited.
|
||||
|
||||
The workspace tracks different types of memory, including:
|
||||
1. allocated memory
|
||||
2. external memory
|
||||
3. unreferenced memory
|
||||
4. workspace memory
|
||||
5. gradient memory.
|
||||
|
||||
We reduce memory usage by reusing the ring buffers described above. The key trick in reducing allocations is to reuse the same memory for the same operations
|
||||
learned through the learning policy. This is done by using a ring buffer to store the memory for each operation.
|
||||
In doing this, the user can reuse existing memory for training/inference increasing performance and reducing memory usage.
|
||||
|
||||
|
||||
|
||||
## Description
|
||||
|
||||
To create a named scope that reuses memory instead of allocating it again, you can use ringbuffers within a namespace-like abstraction,
|
||||
and combine it with java's try/with resources to indicate a scope of memory as well as to automatically garbage collect relevant memory.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
In order to use a workspace we need to have a configuration to determine how a workspace is created
|
||||
and how it allocates memory. The following parameters are possible:
|
||||
|
||||
1. initialSize: The initial size of the workspace in bytes. If the workspace exceeds this size, it will be automatically expanded.
|
||||
|
||||
2. maxSize: The maximum size of the workspace in bytes. If the workspace tries to expand beyond this size, an exception will be thrown.
|
||||
|
||||
3. overallocationLimit: The amount of extra memory to allocate beyond the initial size when the workspace is created.
|
||||
This is useful for workloads that have high variability in their memory usage.
|
||||
|
||||
4. policyAllocation: The allocation policy for the workspace, which can be STRICT (strict allocation),
|
||||
OVERALLOCATE (overallocation), or ALWAYS (always allocate new memory).
|
||||
|
||||
5. policyLearning: The learning policy for the workspace, which can be NONE (no learning),
|
||||
OPTIMIZED (optimized learning), or TRAINING (full training mode).
|
||||
|
||||
6. policyMirroring: The mirroring policy for the workspace, which can be ENABLED (enable mirroring),
|
||||
DISABLED (disable mirroring), or HOST_ONLY (mirror only to host memory).
|
||||
|
||||
7. policySpill: The spill policy for the workspace, which can be FAIL (fail if workspace runs out of memory),
|
||||
REALLOCATE (reallocate memory on the fly), or EXTERNAL (spill to external memory).
|
||||
|
||||
8. overallocationLimit: The amount of extra memory to allocate beyond the initial size when the workspace is created.
|
||||
This is useful for workloads that have high variability in their memory usage.
|
||||
|
||||
9. tempBlockSize: The size of the temporary memory blocks used by the workspace, in bytes.
|
||||
|
||||
10. useCycleDetector: Whether to enable the cycle detector for the workspace, which detects and prevents memory leaks.
|
||||
workspaceMode: The workspace mode, which can be ENABLED (enable workspace mode), SINGLE (use a single global workspace),
|
||||
or NONE (disable workspace mode).
|
||||
|
||||
11. helperAllowFallback: Whether to allow fallback to the CPU when using GPU memory.
|
||||
|
||||
12. helperMinSize: The minimum size in bytes for workspace helper operations.
|
||||
|
||||
|
||||
|
||||
Example usage:
|
||||
In this example, we use try/with blocks to automatically close the workspace and release the INDArray from the workspace memory when
|
||||
the try block is exited.
|
||||
|
||||
We create a workspace with the specified configuration within the try block, and get the MemoryManager for the workspace.
|
||||
We allocate an INDArray using the workspace memory within another try block, and perform some operation
|
||||
on it (in this case, an element-wise multiplication).
|
||||
|
||||
|
||||
```java
|
||||
// create a workspace configuration with 1 GB initial size and host memory only
|
||||
WorkspaceConfiguration config = WorkspaceConfiguration.builder()
|
||||
.initialSize(1024 * 1024 * 1024) // 1 GB initial size
|
||||
.policyMirroring(MirroringPolicy.HOST_ONLY) // use host memory only
|
||||
.build();
|
||||
|
||||
// create a workspace with the specified configuration
|
||||
try (Workspace workspace = Nd4j.getWorkspaceManager().createNewWorkspace(config)) {
|
||||
|
||||
// get the memory manager for the workspace
|
||||
MemoryManager memMgr = workspace.getMemoryManager();
|
||||
|
||||
// allocate an INDArray using the workspace memory
|
||||
try (INDArray input = memMgr.allocate(new long[]{32, 32}, DataBuffer.Type.FLOAT)) {
|
||||
|
||||
// use the INDArray for some operation, e.g. element-wise multiplication
|
||||
input.muli(2);
|
||||
|
||||
} // the INDArray is automatically released from the workspace memory when the try block is exited
|
||||
|
||||
} // the workspace is automatically closed when the try block is exited
|
||||
```
|
||||
|
||||
Since we used try/with blocks to create the workspace and allocate the INDArray, they will be automatically
|
||||
closed and released from the workspace memory when the try blocks are exited, regardless of
|
||||
whether an exception is thrown or not.
|
||||
|
||||
|
||||
In order to create a workspace we need to track the following kinds of memory:
|
||||
Allocated memory: This is memory that has been explicitly allocated by the workspace for a particular operation or computation.
|
||||
|
||||
External memory: This is memory that has been allocated outside of the workspace, but is being used by operations within the workspace.
|
||||
External memory can be useful when working with large datasets or models that do not fit entirely within the workspace.
|
||||
|
||||
Unreferenced memory: This is memory that has been allocated by the workspace, but is no longer being used by any operations or computations.
|
||||
Unreferenced memory can be automatically deallocated by the workspace to free up memory resources.
|
||||
|
||||
Workspace memory: This is memory that has been explicitly allocated by the workspace itself for managing memory and workspace state.
|
||||
Workspace memory can include things like memory for managing scope, tracking allocations and deallocations, and managing internal structures.
|
||||
|
||||
Gradient memory: This is memory that is used for storing gradients during backpropagation during training.
|
||||
DL4J's workspaces can track different types of gradient memory, including standard gradients, external gradients, and deferred gradients.
|
||||
|
||||
|
||||
Note that misuse can cause memory leaks in the following ways:
|
||||
Not closing the workspace properly: If a workspace is not properly closed after use, it can cause a memory leak.
|
||||
This can happen when a user forgets to close the workspace or when an exception occurs and the workspace is not closed in the catch block.
|
||||
|
||||
Using a workspace for too long: If a workspace is used for too long, it can cause a memory leak.
|
||||
This can happen if the workspace is reused too many times or if it is not cleared after each use.
|
||||
|
||||
Holding onto references: If references to objects created within a workspace are held onto for too long, it can cause a memory leak.
|
||||
This can happen if objects are not released from the workspace after they are no longer needed.
|
||||
|
||||
Using too many workspaces: If too many workspaces are created, it can cause a memory leak.
|
||||
This can happen if workspaces are created unnecessarily or if they are not properly managed.
|
||||
|
||||
Incorrect workspace configuration: If the workspace is configured incorrectly, it can cause a memory leak.
|
||||
This can happen if the workspace is not allocated enough memory or if the allocation policy is not set correctly.
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Memory allocation: Workspaces allow for pre-allocation of memory to avoid the overhead associated with dynamic memory allocation during training.
|
||||
|
||||
* Memory reuse: By reusing allocated memory rather than allocating new memory for each operation, workspaces help to reduce memory fragmentation and improve performance.
|
||||
|
||||
* Scope management: Workspaces are created within a particular scope and can be closed once they are no longer needed. This allows for efficient memory management and prevents memory leaks.
|
||||
|
||||
* Automatic deallocation: When a workspace is closed, any memory that was allocated within the workspace is automatically deallocated, freeing up memory resources for other operations.
|
||||
|
||||
Multiple workspaces: DL4J allows for the creation of multiple workspaces, which can be useful when running multiple models or training processes simultaneously.
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* Increased code complexity: Implementing workspaces in your code can add an additional layer of complexity and require more careful management of workspace creation and usage.
|
||||
|
||||
* Memory overhead: Workspaces require some overhead for workspace creation, management, and tracking, which can increase memory usage.
|
||||
|
||||
* Workspace size limitations: Since workspaces are pre-allocated with a fixed size, there may be cases where the allocated size is not sufficient for larger models or datasets. This can limit the performance and accuracy of the training process.
|
||||
|
||||
* Training slowdowns: Depending on the specific use case and how workspaces are implemented, there may be cases where using workspaces could actually slow down the training process rather than speed it up.
|
||||
|
||||
* Learning curve: Using workspaces effectively requires a good understanding of how they work and how to manage them properly, which may require some additional learning and training time.
|
||||
@@ -0,0 +1,72 @@
|
||||
# JavaCPP Pointer Tracking with AspectJ
|
||||
|
||||
## Status
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (18 Apr 2023)
|
||||
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
Finalized by: Adam Gibson (18 Apr 2023)
|
||||
|
||||
## Context
|
||||
|
||||
Tracking memory allocations and deallocations for JavaCPP pointers is challenging. This is
|
||||
important for understanding memory usage patterns and identifying memory leaks in
|
||||
applications that use JavaCPP. Currently, developers rely on manual tracking
|
||||
and debugging techniques,
|
||||
which are time-consuming and error-prone.
|
||||
|
||||
Aspect Oriented Programming (AOP) with AspectJ is used to intercept the allocation and
|
||||
deallocation of JavaCPP pointers, allowing for automatic tracking and reporting of memory usage.
|
||||
In this context, we are implementing an aspect and a memory counter for tracking JavaCPP pointer
|
||||
allocations and deallocations.
|
||||
|
||||
AOP is only enabled when used with compile time weaving. You have to build the profiler with
|
||||
the specified modules in order to enable the weaving.
|
||||
|
||||
The code should not be built with AOP by default due to the overhead.
|
||||
|
||||
## Decision
|
||||
|
||||
We use AspectJ to create an aspect called `MemoryCounterAspect` that intercepts
|
||||
the allocation and deallocation of JavaCPP pointers. This aspect leverages two
|
||||
around advice methods, `allocateMemory` and `deallocate`, to track the memory usage.
|
||||
|
||||
The `allocateMemory` method is triggered when a new JavaCPP pointer is created. It calculates
|
||||
the difference in physical bytes before and after the pointer allocation, and then increments
|
||||
the memory counter accordingly.
|
||||
|
||||
The `deallocate` method is triggered when a JavaCPP pointer is deallocated. It calculates the difference
|
||||
in physical bytes before and after the pointer deallocation, and then decrements the memory counter accordingly.
|
||||
|
||||
The memory counter, `MemoryCounter`, maintains two counters: `allocated` for tracking the total allocated memory,
|
||||
and `instanceCounts` for tracking the number of instances of each JavaCPP pointer class.
|
||||
|
||||
Example usage:
|
||||
|
||||
```java
|
||||
|
||||
|
||||
// Perform operations involving JavaCPP pointers
|
||||
// ...
|
||||
|
||||
// Get memory usage information
|
||||
Map<String, Long> allocatedMemory = MemoryCounter.getAllocated().getCounts();
|
||||
Map<String, Long> instanceCounts = MemoryCounter.getInstanceCounts().getCounts();
|
||||
|
||||
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
* Simplifies memory tracking for JavaCPP pointers
|
||||
* Reduces manual debugging efforts
|
||||
* Provides valuable insights into memory usage patterns and potential memory leaks
|
||||
|
||||
### Disadvantages
|
||||
* AspectJ introduces additional overhead and may slightly impact performance
|
||||
* The tracking aspect may not cover all possible allocation and deallocation scenarios, depending on the JavaCPP library's
|
||||
behavior and the application's usage patterns
|
||||
* The aspect may need to be updated to stay in sync with changes in the JavaCPP library
|
||||
@@ -0,0 +1,76 @@
|
||||
Function Tracing in nd4j and libnd4j Code Base
|
||||
Implemented by: Adam Gibson (26-04-2023)
|
||||
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
Overview
|
||||
The nd4j and libnd4j code base can be difficult to debug due to the lack of tools for tracing program execution.
|
||||
We have implemented function tracing using the -finstrument-functions flag provided by GCC to trace
|
||||
function calls and exits in the code base.
|
||||
This feature will help developers better understand the flow of the program and identify potential issues.
|
||||
|
||||
Function tracing can also work with CUDA, as long as GCC is the underlying compiler.
|
||||
|
||||
How to Enable Function Tracing
|
||||
To enable function tracing, follow these steps:
|
||||
|
||||
Build the code base with -Dlibnd4j.functrace=ON.
|
||||
Add the correct compiler flags to the javacpp plugin.
|
||||
Add the correct flags to the cmake build.
|
||||
Add a maven profile to each nd4j backend containing the correct compiler flags to work with -finstrument-functions.
|
||||
Build with Maven
|
||||
Here are the example Maven build commands for the CPU and CUDA backends, respectively:
|
||||
|
||||
For CPU backend:
|
||||
|
||||
```
|
||||
mvn -Dlibnd4j.functrace=ON -Pcpu clean install -DskipTests
|
||||
```
|
||||
For CUDA backend:
|
||||
|
||||
```
|
||||
mvn -Dlibnd4j.functrace=ON -Pcuda clean install -DskipTests
|
||||
```
|
||||
Make sure you have the necessary profiles configured in your pom.xml to enable tracing for the respective backend.
|
||||
|
||||
Implementation Details
|
||||
Compiler Flags
|
||||
We have used the following compiler flags to enable function tracing:
|
||||
|
||||
-Bsymbolic: Bind references to global symbols at link time, reducing the runtime overhead.
|
||||
-rdynamic: Export all dynamic symbols to the dynamic symbol table, making them available for backtracing.
|
||||
-fno-omit-frame-pointer: Do not omit the frame pointer, allowing for accurate backtraces.
|
||||
-fno-optimize-sibling-calls: Disable sibling call optimization to maintain the correct call stack.
|
||||
-finstrument-functions: Enable instrumentation of function entry and exit points.
|
||||
-g: Enable debugging information.
|
||||
-O0: Set the optimization level to zero for easier debugging.
|
||||
Setting the Output File
|
||||
We have implemented a method in each backend to set the output file for tracing results.
|
||||
|
||||
For the CPU backend, use the following code:
|
||||
|
||||
java
|
||||
```Copy code
|
||||
Nd4jCpu nd4jCpu = (Nd4jCpu) NativeOpsHolder.getInstance().getDeviceNativeOps();
|
||||
nd4jCpu.setInstrumentOut("profilerout.txt");
|
||||
```
|
||||
|
||||
|
||||
For the CUDA backend, use the following code:
|
||||
|
||||
java
|
||||
```
|
||||
Nd4jCuda nd4jCuda = (Nd4jCuda) NativeOpsHolder.getInstance().getDeviceNativeOps();
|
||||
nd4jCuda.setInstrumentOut("profilerout.txt");
|
||||
```
|
||||
These calls set the appropriate file to use for each backend. Note that we don't put this in NativeOps (the parent backend agnostic interface for this) because this normally should not be included in any builds due to overhead.
|
||||
|
||||
LD_PRELOAD and LD_DEBUG
|
||||
To ensure the correct implementation of the enter/exit functions is used, LD_PRELOAD is utilized to preload the libnd4j binary generated during the build. The built-in libc implementation of these functions is a no-op, so preloading the libnd4j binary is necessary.
|
||||
|
||||
LD_DEBUG can be used to verify that the correct implementation is being used by showing the symbols being loaded and their origin.
|
||||
|
||||
Sample Log Output
|
||||
bash
|
||||
Copy code
|
||||
g long> > >::end() (/home/agibsonccc/Documents/GitHub/deeplearning4j/libnd4j/blasbuild/cpu/blas/libnd4
|
||||
@@ -0,0 +1,63 @@
|
||||
# Implement ND4J Log Analyzer for Operation Execution Analysis
|
||||
|
||||
## Status
|
||||
**Proposed**
|
||||
|
||||
Proposed by: Adam Gibson(2024-08-02)
|
||||
|
||||
## Context
|
||||
|
||||
DeepLearning4J applications rely heavily on ND4J operations for neural network computations. As the library evolves, there's a need to identify regressions between different versions and analyze the execution patterns and performance metrics of ND4J operations. Currently, there's no standardized tool for recording, storing, and analyzing these operations in detail.
|
||||
|
||||
## Proposal
|
||||
|
||||
Implement an ND4J Log Analyzer as a Java agent with the following key features:
|
||||
|
||||
1. Record ND4J operation executions in real-time and store them in an H2 database.
|
||||
2. Index the specified DeepLearning4J codebase for reference.
|
||||
3. Provide utilities for querying and analyzing recorded operations:
|
||||
- StackTraceCodeFinder for locating source code lines
|
||||
- JsonComparisonReport for comparing operation logs between different runs or versions
|
||||
- JsonReport for exporting operation logs to JSON format
|
||||
4. Allow injection into running DeepLearning4J applications as a Java agent.
|
||||
|
||||
The analyzer will use two main configuration options:
|
||||
- `sourceCodeIndexerPath`: Path to the DeepLearning4J codebase to index.
|
||||
- `javaagent`: Path to the compiled ND4J Log Analyzer JAR file.
|
||||
|
||||
Data will be stored in two main tables:
|
||||
1. `OpLogEvent`: For storing ND4J operation executions.
|
||||
2. `SourceCodeLine`: For storing indexed source code information.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Enables detailed analysis of ND4J operations across different versions.
|
||||
* Provides a standardized method for recording and storing operation executions.
|
||||
* Allows for easy identification of regressions or unexpected changes in behavior.
|
||||
* Supports both real-time analysis and post-execution comparisons.
|
||||
* Integrates seamlessly with existing DeepLearning4J applications through Java agent injection.
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* Adds computational overhead to the running application due to real-time logging.
|
||||
* Requires additional storage for the H2 database and generated reports.
|
||||
* May require updates to maintain compatibility with future versions of DeepLearning4J and ND4J.
|
||||
* Users need to learn how to use and interpret the new analysis tools.
|
||||
|
||||
### Risks
|
||||
|
||||
* Potential for performance impact on production systems if not used carefully.
|
||||
* Possibility of generating large amounts of data that need to be managed and stored securely.
|
||||
* Risk of false positives in regression detection due to non-deterministic behaviors in some ND4J operations.
|
||||
|
||||
## Action Items
|
||||
|
||||
1. Develop the core Java agent for ND4J operation interception and logging.
|
||||
2. Implement the H2 database schema and data storage mechanisms.
|
||||
3. Create utilities for source code indexing and stack trace analysis.
|
||||
4. Develop tools for JSON report generation and comparison.
|
||||
5. Write comprehensive documentation and usage guides.
|
||||
6. Conduct thorough testing with various DeepLearning4J applications and versions.
|
||||
7. Create a release plan and migration guide for existing DeepLearning4J users.
|
||||
@@ -0,0 +1,76 @@
|
||||
# Refactor nd4j to Centralize Offset Storage and Introduce OpaqueNDArray
|
||||
|
||||
## Status
|
||||
|
||||
**Proposed**
|
||||
|
||||
Proposed by: Adam Gibson Oct 24,2024
|
||||
|
||||
## Context
|
||||
|
||||
The current nd4j codebase stores offsets in multiple locations, including opaque data buffers and data buffers. This scattered approach leads to inconsistencies, potential bugs, and difficulties in maintenance. Additionally, the current method of passing NDArray components from Java to C++ involves manually unpacking various elements, which is error-prone and cumbersome.
|
||||
|
||||
## Proposal
|
||||
|
||||
We propose to refactor the nd4j codebase to centralize offset storage within NDArrays and introduce a new OpaqueNDArray type for improved Java-C++ interoperability. The key features of this proposal include:
|
||||
|
||||
1. Moving all offset information into NDArray objects, removing them from opaque data buffers and data buffers.
|
||||
2. Introducing an OpaqueNDArray type in C++, which is an alias for NDArray*.
|
||||
3. Updating the Java-C++ interop layer to use OpaqueNDArray for passing NDArray information.
|
||||
4. Refactoring existing code to use the new centralized offset storage and OpaqueNDArray.
|
||||
5. Updating documentation and coding standards to reflect these changes.
|
||||
|
||||
Example of the proposed change:
|
||||
|
||||
// Before
|
||||
void someOperation(void* dataBuffer, sd::LongType* shapeBuffer, sd::LongType offset, ...) {
|
||||
// Manual unpacking and offset handling
|
||||
}
|
||||
|
||||
// After
|
||||
typedef NDArray* OpaqueNDArray;
|
||||
|
||||
void someOperation(OpaqueNDArray array) {
|
||||
// Access all necessary information through the NDArray object
|
||||
sd::LongType offset = array->offset();
|
||||
// ...
|
||||
}
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* Improves code consistency and reduces the risk of offset-related bugs.
|
||||
* Simplifies the Java-C++ interop by encapsulating NDArray information.
|
||||
* Reduces the likelihood of errors from manual unpacking of NDArray components.
|
||||
* Makes the codebase more maintainable and easier to understand.
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* Requires a significant refactoring effort across the nd4j codebase.
|
||||
* May introduce temporary bugs during the transition if not done carefully.
|
||||
* Could potentially impact performance if not optimized properly.
|
||||
* Might require updates to external code that interacts with nd4j.
|
||||
|
||||
### Risks
|
||||
|
||||
* Risk of introducing bugs during the refactoring process, especially in complex operations.
|
||||
* Potential for decreased performance if the new offset access methods are not efficiently implemented.
|
||||
* May cause confusion for developers who are accustomed to the current system.
|
||||
* Could potentially break existing code that relies on the current offset storage mechanism.
|
||||
|
||||
## Action Items
|
||||
|
||||
1. Develop a comprehensive guide for implementing and using the new offset storage system and OpaqueNDArray.
|
||||
2. Create a set of unit tests to verify the correctness of offset handling and OpaqueNDArray usage.
|
||||
3. Update the team's coding standards to include guidelines on using the new offset storage and OpaqueNDArray.
|
||||
4. Conduct a pilot implementation on a small, isolated part of the codebase.
|
||||
5. Schedule the refactoring process, prioritizing the most frequently used operations.
|
||||
6. Update all relevant documentation, including comments in the code and API documentation.
|
||||
7. Implement benchmarks to compare performance before and after the changes.
|
||||
8. Conduct thorough code reviews and testing for each refactored section.
|
||||
9. Plan for a grace period to allow developers to familiarize themselves with the new system.
|
||||
10. Monitor and address any issues or feedback arising from the new offset storage and OpaqueNDArray usage.
|
||||
11. Consider creating a static analysis tool to ensure consistent usage of the new system across the codebase.
|
||||
12. Update any build scripts or configuration files that may be affected by the changes.
|
||||
13. Prepare a migration guide for users of nd4j to update their code to work with the new system.
|
||||
@@ -0,0 +1,320 @@
|
||||
# Publish a Smaller Artifact with Limited Type Support
|
||||
|
||||
## Status
|
||||
|
||||
**Proposed**
|
||||
|
||||
Proposed by: [Adam Gibson] Oct 22, 2024
|
||||
|
||||
## Context
|
||||
|
||||
The current C++ library published via Java Maven supports multi-type arithmetic, achieved through extensive template usage in C++ and built using CMake. While this approach provides flexibility by accommodating various data types, it results in a higher binary size. For many users, the full spectrum of type support may be unnecessary, leading to increased storage requirements and longer download times. To address these concerns, there is consideration to publish a smaller artifact that supports only specific types, thereby reducing the binary size.
|
||||
|
||||
## Proposal
|
||||
|
||||
We propose to publish a smaller Maven artifact of the C++ library that supports a limited set of data types. This specialized artifact will cater to users who do not require multi-type arithmetic, offering a more lightweight alternative. The key aspects of this proposal include:
|
||||
|
||||
1. **Create a Limited Type Support Artifact:**
|
||||
- Develop a separate build configuration using CMake that includes only the necessary type specializations.
|
||||
- Ensure that the limited artifact excludes type support beyond the specified types to minimize binary size.
|
||||
|
||||
2. **Maintain the Existing Multi-Type Artifact:**
|
||||
- Continue to offer the current multi-type arithmetic artifact for users who need comprehensive type support.
|
||||
|
||||
3. **Clear Naming and Versioning:**
|
||||
- Use distinct naming conventions (e.g., `mylib-core`, `mylib-lite`) to differentiate between the full and limited artifacts.
|
||||
- Align versioning strategies to ensure compatibility and ease of maintenance.
|
||||
|
||||
4. **Update Documentation and Support Materials:**
|
||||
- Clearly document the differences between the full and limited artifacts.
|
||||
- Provide guidelines on selecting the appropriate artifact based on user needs.
|
||||
|
||||
5. **Implement Testing for Both Artifacts:**
|
||||
- Develop separate test suites to validate the functionality and performance of each artifact.
|
||||
- Ensure that the limited artifact maintains the core functionality required by its target users.
|
||||
|
||||
### Example Test
|
||||
|
||||
As a sample test to ensure the limited artifact functions correctly, consider the following parameterized test:
|
||||
|
||||
```java
|
||||
@ParameterizedTest
|
||||
@MethodSource("org.nd4j.linalg.BaseNd4jTestWithBackends#configs")
|
||||
public void testMixedDataTypeViews(Nd4jBackend backend) {
|
||||
INDArray arrFloat = Nd4j.arange(24).reshape(4, 6).castTo(DataType.FLOAT);
|
||||
INDArray arrDouble = Nd4j.arange(24).reshape(4, 6).castTo(DataType.DOUBLE);
|
||||
INDArray arrLong = Nd4j.arange(24).reshape(4, 6).castTo(DataType.LONG);
|
||||
|
||||
INDArray viewFloat = arrFloat.get(NDArrayIndex.interval(1, 3), NDArrayIndex.interval(2, 5));
|
||||
INDArray viewDouble = arrDouble.get(NDArrayIndex.interval(1, 3), NDArrayIndex.interval(2, 5));
|
||||
INDArray viewLong = arrLong.get(NDArrayIndex.interval(1, 3), NDArrayIndex.interval(2, 5));
|
||||
|
||||
assertEquals(8.0f, viewFloat.getFloat(0, 0), 1e-5);
|
||||
assertEquals(16.0f, viewFloat.getFloat(1, 2), 1e-5);
|
||||
assertEquals(8.0, viewDouble.getDouble(0, 0), 1e-5);
|
||||
assertEquals(16.0, viewDouble.getDouble(1, 2), 1e-5);
|
||||
assertEquals(8L, viewLong.getLong(0, 0));
|
||||
assertEquals(16L, viewLong.getLong(1, 2));
|
||||
}
|
||||
```
|
||||
```cpp
|
||||
Key Macros for Type Promotion Approach
|
||||
To manage type promotion effectively within the limited artifact, the following macros and templates are utilized:
|
||||
|
||||
|
||||
/*
|
||||
* Type Ranking System:
|
||||
* type_rank template and its specializations assign an integer rank to each supported type.
|
||||
* This ranking helps in determining the "promoted" type when combining different types.
|
||||
* Type Promotion Traits:
|
||||
* promote_type and promote_type3 templates determine the promoted type between two or three types based on their ranks.
|
||||
* Type Name System:
|
||||
* type_name template and its specializations provide a string representation for each supported type.
|
||||
* Helper Functions and Macros:
|
||||
* promote function template converts a value to the promoted type.
|
||||
* Macros like INSTANTIATE_PROMOTE and CALLBACK_INSTANTIATE_PROMOTE help in instantiating the promote function for different type combinations.
|
||||
* PROMOTE_ARGS macro handles function arguments correctly.
|
||||
*/
|
||||
|
||||
// Type ranking system
|
||||
template<typename T> struct type_rank;
|
||||
|
||||
#if defined(HAS_BOOL)
|
||||
template<> struct type_rank<bool> : std::integral_constant<int, 0> {};
|
||||
#endif
|
||||
|
||||
#if defined(HAS_INT8)
|
||||
template<> struct type_rank<int8_t> : std::integral_constant<int, 1> {};
|
||||
#endif
|
||||
|
||||
#if defined(HAS_UINT8)
|
||||
template<> struct type_rank<uint8_t> : std::integral_constant<int, 1> {};
|
||||
#endif
|
||||
|
||||
#if defined(HAS_INT16)
|
||||
template<> struct type_rank<int16_t> : std::integral_constant<int, 2> {};
|
||||
#endif
|
||||
|
||||
#if defined(HAS_UINT16)
|
||||
template<> struct type_rank<uint16_t> : std::integral_constant<int, 2> {};
|
||||
#endif
|
||||
|
||||
#if defined(HAS_INT32)
|
||||
template<> struct type_rank<int32_t> : std::integral_constant<int, 3> {};
|
||||
#endif
|
||||
|
||||
#if defined(HAS_UINT32)
|
||||
template<> struct type_rank<uint32_t> : std::integral_constant<int, 3> {};
|
||||
#endif
|
||||
|
||||
template<> struct type_rank<int64_t> : std::integral_constant<int, 4> {};
|
||||
template<> struct type_rank<long long int> : std::integral_constant<int, 4> {};
|
||||
template<> struct type_rank<uint64_t> : std::integral_constant<int, 4> {};
|
||||
|
||||
#if defined(HAS_FLOAT16)
|
||||
template<> struct type_rank<float16> : std::integral_constant<int, 5> {};
|
||||
#endif
|
||||
|
||||
#if defined(HAS_BFLOAT16)
|
||||
template<> struct type_rank<bfloat16> : std::integral_constant<int, 5> {};
|
||||
#endif
|
||||
|
||||
#if defined(HAS_FLOAT32)
|
||||
template<> struct type_rank<float> : std::integral_constant<int, 6> {};
|
||||
#endif
|
||||
|
||||
#if defined(HAS_DOUBLE)
|
||||
template<> struct type_rank<double> : std::integral_constant<int, 7> {};
|
||||
#endif
|
||||
|
||||
// promote_type trait
|
||||
template<typename T1, typename T2>
|
||||
struct promote_type {
|
||||
using type = typename std::conditional<
|
||||
(type_rank<T1>::value >= type_rank<T2>::value),
|
||||
T1,
|
||||
T2
|
||||
>::type;
|
||||
};
|
||||
|
||||
// promote function template
|
||||
template <typename Type1, typename Type2, typename ValueType>
|
||||
typename promote_type<Type1, Type2>::type promote(ValueType value) {
|
||||
return static_cast<typename promote_type<Type1, Type2>::type>(value);
|
||||
}
|
||||
|
||||
// promote_type3 trait for three types
|
||||
template<typename T1, typename T2, typename T3>
|
||||
struct promote_type3 {
|
||||
using type = typename promote_type<
|
||||
typename promote_type<T1, T2>::type,
|
||||
T3
|
||||
>::type;
|
||||
};
|
||||
|
||||
// Primary template for type_name - undefined to trigger a compile-time error for unsupported types
|
||||
template<typename T>
|
||||
struct type_name;
|
||||
|
||||
#if defined(HAS_BOOL)
|
||||
template<> struct type_name<bool> { static const char* get() { return "bool"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_INT8)
|
||||
template<> struct type_name<int8_t> { static const char* get() { return "int8_t"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_UINT8)
|
||||
template<> struct type_name<uint8_t> { static const char* get() { return "uint8_t"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_INT16)
|
||||
template<> struct type_name<int16_t> { static const char* get() { return "int16_t"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_UINT16)
|
||||
template<> struct type_name<uint16_t> { static const char* get() { return "uint16_t"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_INT32)
|
||||
template<> struct type_name<int32_t> { static const char* get() { return "int32_t"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_UINT32)
|
||||
template<> struct type_name<uint32_t> { static const char* get() { return "uint32_t"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_INT64)
|
||||
template<> struct type_name<int64_t> { static const char* get() { return "int64_t"; } };
|
||||
template<> struct type_name<long long int> { static const char* get() { return "long long int"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_UINT64)
|
||||
template<> struct type_name<uint64_t> { static const char* get() { return "uint64_t"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_FLOAT16)
|
||||
template<> struct type_name<float16> { static const char* get() { return "float16"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_BFLOAT16)
|
||||
template<> struct type_name<bfloat16> { static const char* get() { return "bfloat16"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_FLOAT32)
|
||||
template<> struct type_name<float> { static const char* get() { return "float"; } };
|
||||
#endif
|
||||
|
||||
#if defined(HAS_DOUBLE)
|
||||
template<> struct type_name<double> { static const char* get() { return "double"; } };
|
||||
#endif
|
||||
|
||||
// Helper function to get type name
|
||||
template<typename T>
|
||||
const char* get_type_name() {
|
||||
return type_name<T>::get();
|
||||
}
|
||||
|
||||
// Macro to instantiate the promote function
|
||||
#define INSTANTIATE_PROMOTE(a1, b1, FUNC_NAME, ARGS) \
|
||||
template promote_type<GET_SECOND(a1), GET_SECOND(b1)>::type \
|
||||
promote<GET_SECOND(a1), GET_SECOND(b1), GET_SECOND(a1)>(GET_SECOND(a1));
|
||||
|
||||
// Callback macro
|
||||
#define CALLBACK_INSTANTIATE_PROMOTE(a1, b1, FUNC_NAME, ARGS) \
|
||||
INSTANTIATE_PROMOTE(a1, b1, FUNC_NAME, ARGS)
|
||||
|
||||
// Macro to define functions with advanced type promotion and debugging
|
||||
#define SD_PROMOTE_FUNC(FUNC_NAME, BODY) \
|
||||
template<typename T, typename U = T, typename Z = T> \
|
||||
Z FUNC_NAME(T val1, U val2) { \
|
||||
using calc_type = typename promote_type3<T, U, Z>::type; \
|
||||
calc_type promoted_val1 = static_cast<calc_type>(val1); \
|
||||
calc_type promoted_val2 = static_cast<calc_type>(val2); \
|
||||
calc_type result = BODY; \
|
||||
return static_cast<Z>(result); \
|
||||
}
|
||||
|
||||
template <typename T, typename Z>
|
||||
SD_HOST_DEVICE SD_INLINE Z sd_abs(T value);
|
||||
|
||||
template <typename T, typename Z>
|
||||
SD_HOST_DEVICE SD_INLINE Z sd_eq(T value, T value2, double eps) {
|
||||
return sd_abs<T, Z>(value - value2) <= eps;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SD_HOST_DEVICE SD_INLINE void sd_swap(T& val1, T& val2);
|
||||
|
||||
SD_PROMOTE_FUNC(sd_max, (promoted_val1 > promoted_val2 ? promoted_val1 : promoted_val2))
|
||||
|
||||
SD_PROMOTE_FUNC(sd_min, (promoted_val1 < promoted_val2 ? promoted_val1 : promoted_val2))
|
||||
|
||||
SD_PROMOTE_FUNC(sd_add, (promoted_val1 + promoted_val2))
|
||||
SD_PROMOTE_FUNC(sd_subtract, (promoted_val1 - promoted_val2))
|
||||
SD_PROMOTE_FUNC(sd_multiply, (promoted_val1 * promoted_val2))
|
||||
SD_PROMOTE_FUNC(sd_divide, (promoted_val1 / promoted_val2))
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
#### Reduced Binary Size
|
||||
The smaller artifact consumes less storage and reduces download times, making it suitable for environments with limited resources or slow network connections.
|
||||
|
||||
#### Faster Compilation and Build Times
|
||||
Limiting the supported types decreases the complexity of the codebase, leading to quicker compilation and faster build processes.
|
||||
|
||||
#### Simpler Maintenance
|
||||
A streamlined codebase with fewer type specializations simplifies maintenance, allowing for easier bug fixes and feature enhancements.
|
||||
|
||||
#### Specific Target Audience
|
||||
Tailoring the artifact to specific user needs ensures optimal performance and usability for those requiring only certain data types.
|
||||
|
||||
### Disadvantages
|
||||
|
||||
#### Fragmentation
|
||||
Offering multiple artifacts can lead to user confusion regarding which version to use, complicating documentation and support.
|
||||
|
||||
#### Increased Maintenance Overhead
|
||||
Maintaining separate artifacts requires additional effort to ensure consistency and compatibility across versions.
|
||||
|
||||
#### User Flexibility
|
||||
Users may need to switch to the larger artifact in the future if their requirements evolve, potentially leading to compatibility issues.
|
||||
|
||||
#### Inconsistent Performance
|
||||
Differences in optimization between artifacts may result in varying performance characteristics, causing confusion among users.
|
||||
|
||||
#### Dependency Management Complexity
|
||||
Managing dependencies for multiple artifacts increases the risk of conflicts and integration issues within user projects.
|
||||
|
||||
### Risks
|
||||
|
||||
#### Introduction of Bugs
|
||||
Refactoring to create a limited artifact may inadvertently introduce bugs, especially if the transition is not meticulously managed.
|
||||
|
||||
#### Performance Impacts
|
||||
If the limited artifact is not properly optimized, it could underperform compared to expectations.
|
||||
|
||||
#### Developer Confusion
|
||||
Developers accustomed to the multi-type system may find the limited artifact restrictive, leading to potential misuse or frustration.
|
||||
|
||||
#### Breaking Existing Code
|
||||
Users relying on the full type support may experience disruptions if they transition to the limited artifact without proper migration.
|
||||
|
||||
|
||||
## Conclusion
|
||||
|
||||
Publishing a smaller artifact with limited type support offers significant benefits
|
||||
in terms of reduced binary size, faster build times, and simplified maintenance.
|
||||
|
||||
|
||||
However, it also introduces challenges related to fragmentation, increased maintenance
|
||||
overhead, and potential user confusion. By carefully planning the implementation,
|
||||
maintaining clear documentation, and providing robust support, the advantages can be
|
||||
leveraged while mitigating the disadvantages.
|
||||
|
||||
This strategic approach ensures that the library remains flexible and user-friendly,
|
||||
catering to a broader range of use cases
|
||||
without compromising on performance or usability.
|
||||
@@ -0,0 +1,232 @@
|
||||
# ADR-0031: Handling Type Combinations and Template Instantiations with Macros
|
||||
|
||||
## Status
|
||||
|
||||
**Proposed**
|
||||
|
||||
Proposed by: Adam Gibson Oct 22, 2024
|
||||
|
||||
## Context
|
||||
|
||||
In the current C++ library published via Java Maven, multi-type arithmetic is supported through extensive use of templates and built using CMake. While this approach offers flexibility by accommodating various data types, it leads to an increased binary size. Many users do not require the full spectrum of type support, resulting in unnecessary storage consumption and longer download times. To optimize performance and reduce the binary footprint, there is a need to streamline type combinations and template instantiations.
|
||||
|
||||
## Decision
|
||||
|
||||
To efficiently manage type combinations and template instantiations within the limited artifact, a series of preprocessor macros are employed. These macros automate the retrieval and processing of type lists, enabling systematic instantiation of template functions based on various type combinations. Below is a detailed explanation of the key macros and their functionalities:
|
||||
|
||||
### GET Macro
|
||||
|
||||
```cpp
|
||||
#define GET(n, list) CAT(GET_, n) list
|
||||
|
||||
#define GET_0(t1, ...) t1
|
||||
#define GET_1(t1, t2, ...) t2
|
||||
#define GET_2(t1, t2, t3, ...) t3
|
||||
#define GET_3(t1, t2, t3, t4, ...) t4
|
||||
#define GET_4(t1, t2, t3, t4, t5, ...) t5
|
||||
#define GET_5(t1, t2, t3, t4, t5, t6, ...) t6
|
||||
#define GET_6(t1, t2, t3, t4, t5, t6, t7, ...) t7
|
||||
#define GET_7(t1, t2, t3, t4, t5, t6, t7, t8, ...) t8
|
||||
#define GET_8(t1, t2, t3, t4, t5, t6, t7, t8, t9, ...) t9
|
||||
#define GET_9(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, ...) t10
|
||||
#define GET_10(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, ...) t11
|
||||
#define GET_11(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, ...) t12
|
||||
#define GET_12(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, ...) t13
|
||||
#define GET_13(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14, ...) t14
|
||||
#define GET_14(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14, t15, ...) t15
|
||||
#define GET_15(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14, t15, t16, ...) t16
|
||||
Explanation:
|
||||
|
||||
The GET macro retrieves the nth element from a provided list by concatenating GET_ with the index n and applying it to the list.
|
||||
Each GET_n macro is defined to extract the nth element from a variadic list of parameters.
|
||||
GET_FIRST and GET_SECOND Macros
|
||||
```cpp
|
||||
#define GET_FIRST(tuple) GET_FIRST_IMPL tuple
|
||||
#define GET_FIRST_IMPL(a, b) a
|
||||
|
||||
#define GET_SECOND(tuple) GET_SECOND_IMPL tuple
|
||||
#define GET_SECOND_IMPL(a, b) b
|
||||
```
|
||||
Explanation:
|
||||
|
||||
GET_FIRST and GET_SECOND macros extract the first and second elements from a tuple, respectively.
|
||||
They expand the tuple and apply the corresponding implementation macros to retrieve the desired element.
|
||||
PROCESS_COMBINATION and CALLBACK_PROCESS_COMBINATION Macros
|
||||
```cpp
|
||||
#define PROCESS_COMBINATION(a1, b1, a2, b2, FUNC_NAME, ARGS) \
|
||||
std::cout << "(" << a1 << ", " << b1 << ", " << a2 << ", " << b2 << ")\n";
|
||||
|
||||
#define CALLBACK_PROCESS_COMBINATION(outer, inner, FUNC_NAME, ARGS) \
|
||||
PROCESS_COMBINATION(GET_FIRST(outer), GET_SECOND(outer), GET_FIRST(inner), GET_SECOND(inner), FUNC_NAME, ARGS)
|
||||
```
|
||||
|
||||
|
||||
Explanation:
|
||||
|
||||
PROCESS_COMBINATION defines how to process a combination of types. In this example, it simply prints the combination.
|
||||
CALLBACK_PROCESS_COMBINATION retrieves elements from the outer and inner lists and passes them to PROCESS_COMBINATION.
|
||||
INNER_LOOP Macros
|
||||
```cpp
|
||||
|
||||
#define INNER_LOOP_1(OUTER_ELEMENT, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CALLBACK(OUTER_ELEMENT, GET(0, INNER_LIST), FUNC_NAME, ARGS)
|
||||
|
||||
#define INNER_LOOP_2(OUTER_ELEMENT, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
INNER_LOOP_1(OUTER_ELEMENT, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CALLBACK(OUTER_ELEMENT, GET(1, INNER_LIST), FUNC_NAME, ARGS)
|
||||
|
||||
#define INNER_LOOP_3(OUTER_ELEMENT, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
INNER_LOOP_2(OUTER_ELEMENT, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CALLBACK(OUTER_ELEMENT, GET(2, INNER_LIST), FUNC_NAME, ARGS)
|
||||
|
||||
#define INNER_LOOP_4(OUTER_ELEMENT, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
INNER_LOOP_3(OUTER_ELEMENT, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CALLBACK(OUTER_ELEMENT, GET(3, INNER_LIST), FUNC_NAME, ARGS)
|
||||
|
||||
#define INNER_LOOP_5(OUTER_ELEMENT, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
INNER_LOOP_4(OUTER_ELEMENT, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CALLBACK(OUTER_ELEMENT, GET(4, INNER_LIST), FUNC_NAME, ARGS)
|
||||
```
|
||||
|
||||
|
||||
// ... Similarly up to INNER_LOOP_16
|
||||
Explanation:
|
||||
|
||||
INNER_LOOP_n macros iterate over the INNER_LIST up to the nth element.
|
||||
Each iteration applies the CALLBACK macro to process the current combination.
|
||||
OUTER_LOOP Macros
|
||||
```cpp
|
||||
#define OUTER_LOOP_1(OUTER_LIST, INNER_LIST, INNER_SIZE, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CAT(INNER_LOOP_, INNER_SIZE)(GET(0, OUTER_LIST), INNER_LIST, CALLBACK, FUNC_NAME, ARGS)
|
||||
|
||||
#define OUTER_LOOP_2(OUTER_LIST, INNER_LIST, INNER_SIZE, CALLBACK, FUNC_NAME, ARGS) \
|
||||
OUTER_LOOP_1(OUTER_LIST, INNER_LIST, INNER_SIZE, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CAT(INNER_LOOP_, INNER_SIZE)(GET(1, OUTER_LIST), INNER_LIST, CALLBACK, FUNC_NAME, ARGS)
|
||||
|
||||
#define OUTER_LOOP_3(OUTER_LIST, INNER_LIST, INNER_SIZE, CALLBACK, FUNC_NAME, ARGS) \
|
||||
OUTER_LOOP_2(OUTER_LIST, INNER_LIST, INNER_SIZE, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CAT(INNER_LOOP_, INNER_SIZE)(GET(2, OUTER_LIST), INNER_LIST, CALLBACK, FUNC_NAME, ARGS)
|
||||
```
|
||||
// ... Similarly up to OUTER_LOOP_16
|
||||
Explanation:
|
||||
|
||||
OUTER_LOOP_n macros iterate over the OUTER_LIST up to the nth element.
|
||||
For each element in the OUTER_LIST, they invoke the corresponding INNER_LOOP_n macro to process combinations with the INNER_LIST.
|
||||
ITERATE_COMBINATIONS Macro
|
||||
```cpp
|
||||
#define ITERATE_COMBINATIONS(OUTER_LIST, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CAT(OUTER_LOOP_, PP_NARGS(EXPAND OUTER_LIST))(OUTER_LIST, INNER_LIST, PP_NARGS(EXPAND INNER_LIST), CALLBACK, FUNC_NAME, ARGS)
|
||||
```
|
||||
Explanation:
|
||||
|
||||
ITERATE_COMBINATIONS initiates the nested iteration over OUTER_LIST and INNER_LIST.
|
||||
It determines the number of elements in the OUTER_LIST and INNER_LIST using PP_NARGS (a macro to count arguments) and dispatches to the appropriate OUTER_LOOP_n macro.
|
||||
Template Instantiation Macros
|
||||
|
||||
```cpp
|
||||
|
||||
#define INSTANT_PROCESS_COMBINATION(a1, b1, FUNC_NAME, ARGS) \
|
||||
template void FUNC_NAME<GET_SECOND(a1), GET_SECOND(b1)>ARGS;
|
||||
|
||||
#define INSTANT_PROCESS_COMBINATION_3(a1, b1, c1, FUNC_NAME, ARGS) \
|
||||
template void FUNC_NAME<GET_SECOND(a1), GET_SECOND(b1), GET_SECOND(c1)>ARGS;
|
||||
|
||||
#define INSTANT_PROCESS_COMBINATION_CLASS(a1, b1, FUNC_NAME, ARGS) \
|
||||
template class FUNC_NAME<GET_SECOND(a1), GET_SECOND(b1)>ARGS;
|
||||
|
||||
#define INSTANT_PROCESS_COMBINATION_CLASS_3(a1, b1, c1, FUNC_NAME, ARGS) \
|
||||
extern template class FUNC_NAME<GET_SECOND(a1), GET_SECOND(b1), GET_SECOND(c1)>ARGS;
|
||||
```
|
||||
|
||||
Explanation:
|
||||
|
||||
These macros instantiate template functions with specific type combinations extracted from type lists.
|
||||
INSTANT_PROCESS_COMBINATION handles two-type combinations, while INSTANT_PROCESS_COMBINATION_3 handles three-type combinations.
|
||||
Similarly, class templates can be instantiated or declared extern using the corresponding macros.
|
||||
ITERATE_COMBINATIONS_3 Macro
|
||||
cpp
|
||||
Copy code
|
||||
#define ITERATE_COMBINATIONS_3(OUTER_LIST, MIDDLE_LIST, INNER_LIST, CALLBACK, FUNC_NAME, ARGS) \
|
||||
CAT(OUTER_LOOP_, CAT(PP_NARGS(EXPAND OUTER_LIST), _3))(OUTER_LIST, MIDDLE_LIST, INNER_LIST, PP_NARGS(EXPAND MIDDLE_LIST), PP_NARGS(EXPAND INNER_LIST), CALLBACK, FUNC_NAME, ARGS)
|
||||
Explanation:
|
||||
|
||||
ITERATE_COMBINATIONS_3 iterates over three type lists (SD_COMMON_TYPES) and applies the INSTANT_PROCESS_COMBINATION_3 macro to each combination.
|
||||
This results in the instantiation of the PairWiseTransform::exec function for each combination of types.
|
||||
Usage Example
|
||||
The following example demonstrates how the macros are used to instantiate template functions based on combinations of types:
|
||||
|
||||
```cpp
|
||||
/*
|
||||
*
|
||||
*
|
||||
ITERATE_COMBINATIONS_3: This macro iterates over three lists of data types (SD_COMMON_TYPES) and applies the INSTANT_PROCESS_COMBINATION_3 macro to each combination. This results in the instantiation of the PairWiseTransform::exec function for each combination of data types.
|
||||
ITERATE_COMBINATIONS: This macro iterates over two lists of data types (SD_COMMON_TYPES) and applies the CALLBACK_INSTANTIATE_PROMOTE macro to each combination. This is likely used for promoting data types.
|
||||
Function Instantiation:
|
||||
The PairWiseTransform::exec function is instantiated for various combinations of data types. The function signature includes parameters for operation number (opNum), input arrays (x, y), their shape information (xShapeInfo, yShapeInfo), output array (z), its shape information (zShapeInfo), extra parameters (extraParams), and the range of elements to process (start, stop).
|
||||
*/
|
||||
ITERATE_COMBINATIONS_3(
|
||||
(SD_COMMON_TYPES),
|
||||
(SD_COMMON_TYPES),
|
||||
(SD_COMMON_TYPES),
|
||||
INSTANT_PROCESS_COMBINATION_3,
|
||||
functions::pairwise_transforms::PairWiseTransform,
|
||||
::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, sd::LongType start, sd::LongType stop)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
(SD_COMMON_TYPES),
|
||||
(SD_COMMON_TYPES),
|
||||
CALLBACK_INSTANTIATE_PROMOTE,
|
||||
promote,
|
||||
;
|
||||
)
|
||||
```
|
||||
|
||||
Explanation:
|
||||
|
||||
ITERATE_COMBINATIONS_3 iterates over three type lists (SD_COMMON_TYPES) and instantiates the PairWiseTransform::exec function for each combination of types.
|
||||
ITERATE_COMBINATIONS iterates over two type lists and applies the CALLBACK_INSTANTIATE_PROMOTE macro to handle type promotion.
|
||||
Summary
|
||||
The macros defined above provide a systematic approach to handling multiple type combinations and template instantiations. By automating the retrieval and processing of type lists, the codebase ensures that all necessary template instances are generated without manual intervention. This not only reduces the potential for errors but also streamlines the maintenance and scalability of the library.
|
||||
|
||||
Benefits:
|
||||
|
||||
Automation: Reduces the need for repetitive code by automating template instantiations.
|
||||
Scalability: Easily handles a large number of type combinations without increasing code complexity.
|
||||
Maintainability: Simplifies updates and maintenance by centralizing type handling logic within macros.
|
||||
Considerations:
|
||||
|
||||
Complexity: Macros can be difficult to debug and understand, especially for those unfamiliar with advanced preprocessor techniques.
|
||||
Compilation Time: Extensive use of templates and macros may lead to longer compilation times.
|
||||
By incorporating these macros into the limited artifact, the library achieves a balance between flexibility and efficiency, ensuring that only necessary type combinations are included, thereby reducing binary size and optimizing performance.
|
||||
|
||||
# Consequences
|
||||
|
||||
## Advantages
|
||||
|
||||
### Flexibility in Compile-Time Code Generation
|
||||
- **Automated Template Instantiation:** The combination macros enable the automatic generation of multiple template instantiations during compile time. This reduces the need for repetitive manual code, ensuring that all necessary type combinations are systematically covered.
|
||||
- **Dynamic Type Support:** By leveraging macros, the library can easily support new data types without significant changes to the core codebase. Adding or modifying type combinations becomes a matter of updating macro definitions, enhancing the library's adaptability.
|
||||
- **Consistent Code Patterns:** Macros ensure that template instantiations follow a uniform pattern, minimizing discrepancies and maintaining consistency across different type combinations. This uniformity simplifies the integration of new functionalities and type supports.
|
||||
|
||||
## Disadvantages
|
||||
|
||||
### Increased Code Complexity
|
||||
- **Complex Macro Definitions:** The use of advanced preprocessor macros introduces a layer of complexity that can be challenging to understand and manage. Developers unfamiliar with intricate macro techniques may find the codebase harder to navigate and modify.
|
||||
- **Obfuscated Code Flow:** Macros can obscure the actual code being generated, making it difficult to trace the flow of template instantiations. This obscurity can complicate debugging efforts and hinder the comprehension of how different type combinations are handled.
|
||||
- **Maintenance Challenges:** Updating or extending macros to accommodate new type combinations requires careful adjustments to prevent introducing bugs. The intricate nature of macro-based code generation demands meticulous attention, increasing the maintenance overhead.
|
||||
- **Limited Tooling Support:** Many development tools and IDEs offer limited support for macro-heavy code, reducing the effectiveness of features like code completion, refactoring, and error highlighting. This limitation can slow down development and increase the likelihood of unnoticed issues.
|
||||
|
||||
|
||||
## Conclusion
|
||||
|
||||
Publishing a smaller artifact with limited type support offers significant benefits,
|
||||
including reduced binary size, faster build times, and simplified maintenance.
|
||||
However, it also brings challenges such as fragmentation, increased maintenance
|
||||
overhead, and potential user confusion. By carefully planning the implementation,
|
||||
maintaining clear documentation, and providing robust support, the advantages can
|
||||
be maximized while mitigating the disadvantages. This strategic approach ensures
|
||||
that the library remains flexible and user-friendly, catering to a broader range
|
||||
of use cases without compromising on performance or usability.
|
||||
@@ -0,0 +1,196 @@
|
||||
# C++ Print Debugging Utilities
|
||||
|
||||
## Status
|
||||
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (20-11-2024)
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
## Context
|
||||
|
||||
We need a way to provide debugging utilities for C++ code in the nd4j library. These utilities help in identifying issues such as out-of-bounds crashes and unexpected behavior in mathematical operations. The goal is to have a set of tools that can be easily enabled or disabled via configuration flags.
|
||||
|
||||
## Decision
|
||||
|
||||
We implement three distinct debugging utilities that can be controlled through configuration flags:
|
||||
|
||||
1. Print Indices - For tracking loop execution and array access
|
||||
2. Print Math - For debugging mathematical operations
|
||||
3. Preprocessor Output - For debugging macro behavior
|
||||
|
||||
Each utility serves a specific debugging purpose and can be independently enabled or disabled through both Maven and CMake configuration.
|
||||
|
||||
## Configuration
|
||||
|
||||
### Maven Configuration (pom.xml)
|
||||
```xml
|
||||
<!-- Print Indices Configuration -->
|
||||
<libnd4j.printindices>OFF</libnd4j.printindices>
|
||||
|
||||
<!-- Print Math Configuration -->
|
||||
<libnd4j.printmath>OFF</libnd4j.printmath>
|
||||
|
||||
<!-- Preprocessor Configuration -->
|
||||
<libnd4j.preprocess>OFF</libnd4j.preprocess>
|
||||
```
|
||||
|
||||
### Build System Integration
|
||||
The utilities are integrated into the build system through a shell script that configures CMake. The script:
|
||||
|
||||
1. Echoes configuration status:
|
||||
```bash
|
||||
echo PRINT_INDICES = "$PRINT_INDICES"
|
||||
echo PRINT_MATH = "$PRINT_MATH"
|
||||
echo PREPROCESS = "$PREPROCESS"
|
||||
```
|
||||
|
||||
2. Passes configuration to CMake:
|
||||
```bash
|
||||
eval "$CMAKE_COMMAND" \
|
||||
-DPRINT_MATH="$PRINT_MATH" \
|
||||
-DPRINT_INDICES="$PRINT_INDICES" \
|
||||
# ... other CMake configurations
|
||||
```
|
||||
|
||||
3. Special handling for preprocessing:
|
||||
```bash
|
||||
if [ "$PREPROCESS" == "ON" ]; then
|
||||
# Re-run CMake with preprocessing enabled
|
||||
eval "$CMAKE_COMMAND" \
|
||||
-DPRINT_MATH="$PRINT_MATH" \
|
||||
-DPRINT_INDICES="$PRINT_INDICES" \
|
||||
-DSD_PREPROCESS="$PREPROCESS" \
|
||||
# ... other configurations
|
||||
echo "Running preprocessing step..."
|
||||
exit 0
|
||||
fi
|
||||
```
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### 1. Print Indices Utility
|
||||
|
||||
#### Purpose
|
||||
Tracks loop iterations and array access patterns to help identify out-of-bounds issues and iteration-related bugs.
|
||||
|
||||
#### Build Configuration
|
||||
1. Set through Maven property `libnd4j.printindices`
|
||||
2. Passed to CMake as `PRINT_INDICES`
|
||||
3. Defines `PRINT_INDICES` macro in C++ code
|
||||
|
||||
#### Implementation
|
||||
```cpp
|
||||
#if defined(PRINT_INDICES)
|
||||
printf("i: %lld xEws %lld ReduceBoolFunction<X, Z>::execScalar\n", i, xEws);
|
||||
#endif
|
||||
```
|
||||
|
||||
#### Example Usage
|
||||
```cpp
|
||||
else {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
#if defined(PRINT_INDICES)
|
||||
printf("i: %lld xEws %lld ReduceBoolFunction<X, Z>::execScalar\n", i, xEws);
|
||||
#endif
|
||||
intermediate[thread_id] = OpType::update(
|
||||
intermediate[thread_id],
|
||||
OpType::op(x[i * xEws], extraParams),
|
||||
extraParams
|
||||
);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Print Math Utility
|
||||
|
||||
#### Purpose
|
||||
Provides detailed tracking of mathematical operations, including input values, outputs, and execution flow.
|
||||
|
||||
#### Build Configuration
|
||||
1. Set through Maven property `libnd4j.printmath`
|
||||
2. Passed to CMake as `PRINT_MATH`
|
||||
3. Defines `SD_GCC_FUNCTRACE` macro in C++ code when enabled
|
||||
|
||||
#### Implementation
|
||||
```cpp
|
||||
template <>
|
||||
SD_INLINE SD_HOST void sd_print_math2<uint16_t>(char* func_name, uint16_t input1, uint16_t input2, uint16_t output) {
|
||||
#if defined(SD_GCC_FUNCTRACE)
|
||||
PRINT_IF_NECESSARY(func_name);
|
||||
#endif
|
||||
printf("%s: input1 = %d, input2 = %d, output = %d\n", func_name, input1, input2, output);
|
||||
fflush(stdout);
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Preprocessor Output Utility
|
||||
|
||||
#### Purpose
|
||||
Generates preprocessed source files to help debug macro-related issues and understand macro expansion.
|
||||
|
||||
#### Build Configuration
|
||||
1. Set through Maven property `libnd4j.preprocess`
|
||||
2. Passed to CMake as `SD_PREPROCESS`
|
||||
3. Triggers special build flow in shell script when enabled
|
||||
|
||||
#### Build Process
|
||||
When preprocessing is enabled:
|
||||
1. Maven sets `libnd4j.preprocess=ON`
|
||||
2. Shell script detects `PREPROCESS=ON`
|
||||
3. CMake is run with `-DSD_PREPROCESS=ON`
|
||||
4. Build exits after preprocessing step
|
||||
5. Preprocessed files are generated in the build directory
|
||||
|
||||
#### Implementation
|
||||
```cmake
|
||||
if("${SD_PREPROCESS}" STREQUAL "ON")
|
||||
message("Preprocessing enabled ${CMAKE_BINARY_DIR}")
|
||||
include_directories(${CMAKE_BINARY_DIR}/../../include)
|
||||
list(REMOVE_DUPLICATES ALL_SOURCES)
|
||||
# Generate preprocessed files
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Print Indices
|
||||
#### Advantages
|
||||
- Easily identifies loop boundary issues
|
||||
- Helps debug array access patterns
|
||||
- Minimal code overhead
|
||||
|
||||
#### Drawbacks
|
||||
- Can generate large amounts of output
|
||||
- Performance impact when enabled
|
||||
|
||||
### Print Math
|
||||
#### Advantages
|
||||
- Detailed visibility into mathematical operations
|
||||
- Type-safe debugging
|
||||
- Stack trace integration
|
||||
|
||||
#### Drawbacks
|
||||
- Increased binary size
|
||||
- Runtime overhead when enabled
|
||||
- Additional complexity in template code
|
||||
|
||||
### Preprocessor Output
|
||||
#### Advantages
|
||||
- Clear visibility into macro expansion
|
||||
- Helps debug complex macro interactions
|
||||
- Useful for template debugging
|
||||
- Early exit prevents unnecessary compilation steps
|
||||
|
||||
#### Drawbacks
|
||||
- Increases build complexity
|
||||
- Additional disk space requirements
|
||||
- Requires separate build run for preprocessing
|
||||
- Cannot be combined with normal build flow
|
||||
|
||||
## References
|
||||
- platformmath.h
|
||||
- templatemath.h
|
||||
- CMakeLists.txt configuration
|
||||
- pom.xml configuration
|
||||
- build-dl4j-natives.sh script
|
||||
- ReduceBoolFunction implementation
|
||||
@@ -0,0 +1,170 @@
|
||||
# ADR 0033: Shape Buffer Trie Implementation
|
||||
|
||||
## Status
|
||||
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (19-12-2024)
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
## Context
|
||||
The libnd4j library requires efficient storage and lookup of shape
|
||||
information for neural network operations. Shape information is
|
||||
usually calculated multilple times per operation and can be expensive
|
||||
to maintain.
|
||||
One goal is to reduce the overhead by getting rid of ShapeDescriptors
|
||||
which were unnecessary extra allocations rather than just using only the shape
|
||||
buffers.
|
||||
This was all stored in a ShapeDescriptor-based cache with an unordered map,
|
||||
|
||||
The primary challenges include:
|
||||
|
||||
1. Frequent shape buffer allocations and deallocations during neural
|
||||
network operations
|
||||
2. Need for fast shape lookup during computation
|
||||
3. Memory management of redundant shape information
|
||||
4. Thread safety requirements for parallel execution
|
||||
5. Overhead from ShapeDescriptor creation for cache lookups
|
||||
6. Memory overhead from unordered map storage
|
||||
|
||||
## Decision
|
||||
We implement a shape buffer trie data structure (`DirectShapeTrie`) to manage and
|
||||
cache shape information, replacing the previous unordered map implementation.
|
||||
The trie structure is chosen for the following characteristics:
|
||||
|
||||
### Key Components
|
||||
- A trie node structure containing:
|
||||
- Shape buffer pointer
|
||||
- Child node pointers
|
||||
- Reference counting mechanism
|
||||
- [Striped thread safety](https://www.baeldung.com/java-lock-stripping) using an array of mutexes
|
||||
- Direct memory management of shape buffers
|
||||
- Sequential shape information exploitation similar to word tries
|
||||
|
||||
### Implementation Details
|
||||
1. The trie stores shape buffers based on their content as sequential paths
|
||||
2. Each unique shape path in the trie represents a unique shape configuration
|
||||
3. Reference counting is used to manage memory lifecycle
|
||||
4. Thread safety is ensured through striped mutex locks
|
||||
5. Direct memory allocation is used instead of standard containers
|
||||
6. Removed ShapeDescriptor creation requirement for lookups
|
||||
7. Shape values are stored sequentially in the trie, similar to characters in a word trie
|
||||
|
||||
### Visual Example
|
||||
```
|
||||
Root
|
||||
├── 2 (rank)
|
||||
│ ├── 3,4 (shape values)
|
||||
│ │ └── [ptr: shape_buffer_1]
|
||||
│ └── 5,6 (shape values)
|
||||
│ └── [ptr: shape_buffer_2]
|
||||
└── 3 (rank)
|
||||
├── 2,3,4 (shape values)
|
||||
│ │ └── [ptr: shape_buffer_3]
|
||||
└── 4,5,6 (shape values)
|
||||
└── [ptr: shape_buffer_4]
|
||||
```
|
||||
|
||||
In this example:
|
||||
- Each level represents a component of the shape
|
||||
- First level: rank of the array
|
||||
- Subsequent levels: actual shape values
|
||||
- Leaf nodes contain pointers to the actual shape buffers
|
||||
- Multiple shapes can share common prefixes, saving memory
|
||||
|
||||
## Thread Safety Implementation
|
||||
|
||||
The shape buffer cache implements striped locking using an array of mutexes:
|
||||
```cpp
|
||||
mutable std::array<MUTEX_TYPE, NUM_STRIPES> _mutexes;
|
||||
```
|
||||
|
||||
This design provides:
|
||||
1. Reduced contention through multiple lock stripes
|
||||
2. Better concurrency than a single global mutex
|
||||
3. Lower memory overhead than per-node locking
|
||||
4. Const-correctness through mutable mutex array
|
||||
|
||||
The striping mechanism:
|
||||
1. Distributes shapes across multiple mutexes based on their characteristics
|
||||
2. Allows concurrent operations on shapes in different stripes
|
||||
3. Balances between fine-grained locking and implementation complexity
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
1. Memory Efficiency:
|
||||
- Eliminates redundant shape buffer storage
|
||||
- Automatic cleanup of unused shapes through reference counting
|
||||
- Shared shape buffers across operations
|
||||
- Removal of ShapeDescriptor allocation overhead
|
||||
- Better memory locality due to trie structure
|
||||
|
||||
2. Performance:
|
||||
- O(n) lookup time where n is the shape length
|
||||
- Efficient shape comparison through pointer equality
|
||||
- Reduced memory allocation overhead
|
||||
- No ShapeDescriptor creation cost for lookups
|
||||
- Sequential access patterns for shape values
|
||||
|
||||
3. Thread Safety:
|
||||
- Safe concurrent access through striped mutex protection
|
||||
- Atomic reference counting operations
|
||||
- Protected shape buffer lifecycle management
|
||||
|
||||
4. Concurrency:
|
||||
- Striped locking enables parallel access to different shape regions
|
||||
- Better scaling under high concurrency than single mutex
|
||||
- Maintains simplicity compared to per-node locking
|
||||
|
||||
### Disadvantages
|
||||
1. Implementation Complexity:
|
||||
- Manual memory management requires careful implementation
|
||||
- Reference counting edge cases need careful handling
|
||||
- Thread synchronization adds complexity
|
||||
- More complex trie traversal logic
|
||||
|
||||
2. Memory Overhead:
|
||||
- Trie structure itself introduces memory overhead
|
||||
- Additional pointers for trie navigation
|
||||
|
||||
3. Performance Trade-offs:
|
||||
- Stripe selection adds minor overhead
|
||||
- Multiple shapes might still hash to same stripe
|
||||
- Reference counting operations add CPU overhead
|
||||
- Fixed number of stripes limits maximum parallelism
|
||||
|
||||
## Technical Details
|
||||
|
||||
### Memory Management
|
||||
```cpp
|
||||
sd::LongType* createBuffer(int length);
|
||||
void deleteBuffer(sd::LongType* buffer);
|
||||
```
|
||||
|
||||
### API Design
|
||||
```cpp
|
||||
sd::LongType* lookupBuffer(const sd::LongType* shape, int length);
|
||||
void registerBuffer(const sd::LongType* shape, int length);
|
||||
void decrementRef(const sd::LongType* buffer);
|
||||
```
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. Hash Table Implementation (Previous Approach):
|
||||
- Pros: Simpler implementation, O(1) average lookup
|
||||
- Cons: More memory usage, ShapeDescriptor overhead, potential hash collisions
|
||||
- Remov
|
||||
2. Alternative Locking Strategies:
|
||||
- Single Global Mutex:
|
||||
- Pros: Simplest implementation
|
||||
- Cons: High contention under load
|
||||
- Per-Node Locking:
|
||||
- Pros: Maximum concurrency
|
||||
- Cons: High memory overhead, complex synchronization
|
||||
- Lock-Free Design:
|
||||
- Pros: No lock contention
|
||||
- Cons: Extremely complex implementation, harder to verify correctness
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
# ADR 0034: FlatBuffers Modernization
|
||||
|
||||
## Status
|
||||
|
||||
Implemented
|
||||
|
||||
Proposed by: Assistant (14-04-2025)
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
## Context
|
||||
|
||||
The libnd4j library uses FlatBuffers for serialization of neural network graphs and related data structures. The current implementation uses FlatBuffers 1.12.0 syntax and conventions, particularly for handling sequences and arrays. With the upgrade to newer versions of FlatBuffers, we need to modernize our schema definitions and code generation.
|
||||
|
||||
The primary challenges include:
|
||||
|
||||
1. Migration from legacy Sequence/SequenceItem patterns to modern vector types
|
||||
2. Proper generation of Java files with correct package structures
|
||||
3. Integration with build system for consistent compilation
|
||||
4. Maintaining backward compatibility where possible
|
||||
5. Ensuring proper environment variable passing for code generation
|
||||
6. Proper schema namespace management
|
||||
|
||||
## Decision
|
||||
|
||||
We implement a modernized FlatBuffers integration that uses current vector syntax and build processes. This includes:
|
||||
|
||||
### Key Components
|
||||
- Schema files using modern vector syntax (`[Type]` instead of Sequence)
|
||||
- Direct build process integration in CMake
|
||||
- Explicit environment variable handling for flatc compiler
|
||||
- Namespace standardization across schema files
|
||||
|
||||
### Implementation Details
|
||||
1. Schemas use vector syntax for array types (e.g., `[FlatArray]` instead of `Sequence<FlatArray>`)
|
||||
2. CMake executes flatc compilation directly instead of using custom targets
|
||||
3. Build process ensures flatc is compiled before schema generation
|
||||
4. Java package structure matches expected nd4j-api layout
|
||||
5. Schema namespaces standardized to `graph` without `sd` prefix
|
||||
|
||||
### Schema Example
|
||||
```fbs
|
||||
namespace graph;
|
||||
|
||||
table SequenceItem {
|
||||
name:string;
|
||||
associated_variable:[FlatArray];
|
||||
}
|
||||
|
||||
table SequenceItemRoot {
|
||||
sequence_items:[SequenceItem];
|
||||
}
|
||||
|
||||
root_type SequenceItemRoot;
|
||||
```
|
||||
|
||||
### Build Integration
|
||||
```cmake
|
||||
execute_process(
|
||||
COMMAND ${CMAKE_COMMAND} -E env "FLATC_PATH=${FLATC_EXECUTABLE}"
|
||||
bash ${CMAKE_CURRENT_SOURCE_DIR}/flatc-generate.sh
|
||||
RESULT_VARIABLE FLATC_RESULT
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
)
|
||||
```
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
1. Modern FlatBuffers Integration:
|
||||
- Cleaner schema definitions
|
||||
- Better type safety through vector syntax
|
||||
- More maintainable code generation
|
||||
- Consistent with current FlatBuffers best practices
|
||||
|
||||
2. Build System Integration:
|
||||
- Reliable flatc compilation
|
||||
- Proper environment variable handling
|
||||
- Direct process execution instead of custom targets
|
||||
- Better error handling and reporting
|
||||
|
||||
3. Code Organization:
|
||||
- Correct Java package structure
|
||||
- Standardized namespaces
|
||||
- Clear separation of generated code
|
||||
- Better integration with existing nd4j structure
|
||||
|
||||
### Disadvantages
|
||||
1. Implementation Requirements:
|
||||
- Need to update existing schema files
|
||||
- Must ensure build process compatibility
|
||||
- Potential for temporary build issues during transition
|
||||
|
||||
2. Migration Effort:
|
||||
- Updates needed for existing code using legacy patterns
|
||||
- Need to verify all schema files
|
||||
- Testing required for all serialization paths
|
||||
|
||||
## Technical Details
|
||||
|
||||
### Schema Location
|
||||
```
|
||||
/libnd4j/include/graph/scheme/*.fbs
|
||||
```
|
||||
|
||||
### Generated Code Location
|
||||
```
|
||||
/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/graph/
|
||||
```
|
||||
|
||||
### Build Process
|
||||
1. CMake configures build
|
||||
2. flatc compiler is built
|
||||
3. Schema generation script runs
|
||||
4. Java files are copied to appropriate location
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. Custom Target Approach:
|
||||
- Pros: More traditional CMake integration
|
||||
- Cons: Less direct control over process, harder to debug
|
||||
|
||||
2. Manual File Generation:
|
||||
- Pros: Simpler build process
|
||||
- Cons: Error-prone, harder to maintain
|
||||
|
||||
3. Separate Build Step:
|
||||
- Pros: Cleaner separation of concerns
|
||||
- Cons: More complex build process, potential for synchronization issues
|
||||
@@ -0,0 +1,325 @@
|
||||
# ADR 0035: SameDiff Unified Container Format
|
||||
|
||||
## Status
|
||||
|
||||
Implemented
|
||||
|
||||
Proposed by: Adam Gibson (15-04-2025)
|
||||
|
||||
|
||||
## Context
|
||||
|
||||
The current SameDiff serialization relies on FlatBuffers for graph representation and handles large arrays (>2GB) using a chunking mechanism. However, this approach has several limitations:
|
||||
|
||||
1. **Single File Deployment**: Current format often requires multiple files when externalizing large arrays
|
||||
2. **Large Model Support**: Limited efficiency when dealing with very large models
|
||||
3. **Metadata Management**: Lack of standardized metadata for model tracking and versioning
|
||||
4. **Model Sharding**: Limited explicit support for sharding large models
|
||||
5. **Compatibility**: Each format change risks breaking backward compatibility
|
||||
|
||||
We need a more robust serialization format that addresses these challenges while maintaining compatibility with existing systems.
|
||||
|
||||
## Decision
|
||||
|
||||
We have implemented a unified container format for SameDiff that encapsulates both graph structure and arrays in a single file, with support for optional externalization and sharding when needed. This format maintains full backward compatibility with the original serialization approach.
|
||||
|
||||
### Key Components
|
||||
|
||||
1. **Multi-Format Support**:
|
||||
- SDNB Format: Single-file internal format (.sdnb)
|
||||
- SDZ Format: ZIP-based container format (.sdz)
|
||||
- Sharded formats for both SDNB and SDZ
|
||||
|
||||
2. **SDNB Format**:
|
||||
- Section-based container with header, metadata, graph, and arrays
|
||||
- Efficient memory mapping for large arrays
|
||||
- Optimized for performance with direct I/O
|
||||
- Compatible with 32-bit FlatBuffers limitations
|
||||
|
||||
3. **SDZ Format**:
|
||||
- Standard ZIP archive containing internal .sdnb files
|
||||
- Compressed storage to reduce file size
|
||||
- Standard tools compatibility for inspection and extraction
|
||||
- Single file deployment for complex models
|
||||
- Simplicity of implementation using standard ZIP libraries
|
||||
|
||||
4. **Metadata Management**:
|
||||
- Standardized keys for common model attributes
|
||||
- Support for custom metadata
|
||||
- Versioning and provenance information
|
||||
- Extensible metadata system similar to GGUF (General GPU Unified Format)
|
||||
- Ability to add metadata later without reserializing model parameters
|
||||
|
||||
5. **Sharding Support**:
|
||||
- Explicit first-class support for model sharding in both formats
|
||||
- Smart distribution of variables across shards
|
||||
- Automatic shard count determination based on model size
|
||||
- Consistent naming convention for shards
|
||||
- Support for NDArrays of any size through intelligent sharding
|
||||
|
||||
6. **Backward Compatibility**:
|
||||
- Automatic format detection between SDNB and SDZ formats
|
||||
- Support for loading both internal and externalized original formats
|
||||
- Legacy model conversion utilities
|
||||
|
||||
### Implementation Details
|
||||
|
||||
1. **SDNB Format Structure**:
|
||||
```
|
||||
MAGIC_BYTES (4 bytes: "SDNB")
|
||||
VERSION (4 bytes)
|
||||
MANIFEST_OFFSET (8 bytes)
|
||||
MANIFEST_LENGTH (8 bytes)
|
||||
METADATA_OFFSET (8 bytes)
|
||||
[FLATBUFFER_GRAPH_DATA]
|
||||
[APPENDED_ARRAYS_DATA]
|
||||
[SERIALIZED_MANIFEST]
|
||||
```
|
||||
|
||||
2. **SDZ Format Structure**:
|
||||
```
|
||||
ZIP_HEADER
|
||||
[ENTRY: model.sdnb] # Graph structure shard
|
||||
[ENTRY: model.shard0-of-N.sdnb] # Alternative naming for graph shard
|
||||
[ENTRY: model.shard1-of-N.sdnb] # Variable shard 1
|
||||
[ENTRY: model.shard2-of-N.sdnb] # Variable shard 2
|
||||
...
|
||||
[ENTRY: model.shardM-of-N.sdnb] # Variable shard M
|
||||
ZIP_DIRECTORY
|
||||
ZIP_END
|
||||
```
|
||||
|
||||
3. **Sharding Strategy**:
|
||||
- Graph structure in shard 0
|
||||
- Variables distributed across remaining shards
|
||||
- Dynamic shard count calculation based on variable sizes
|
||||
- Maximum shard size limit of 1GB per shard
|
||||
- Smart variable grouping to minimize cross-shard dependencies
|
||||
|
||||
4. **API Design**:
|
||||
```java
|
||||
// SDNB Format API
|
||||
SameDiffSerializer.save(sameDiff, file, saveUpdaterState, metadata);
|
||||
SameDiffSerializer.saveAutoShard(sameDiff, baseFile, saveUpdaterState, metadata);
|
||||
SameDiffSerializer.saveSharded(sameDiff, baseFile, saveUpdaterState, estimatedShards, metadata);
|
||||
SameDiff model = SameDiffSerializer.load(file, loadUpdaterState);
|
||||
SameDiff model = SameDiffSerializer.loadSharded(baseFile, loadUpdaterState);
|
||||
|
||||
// SDZ Format API
|
||||
SDZSerializer.save(sameDiff, outputZipFile, saveUpdaterState, metadata);
|
||||
SameDiff model = SDZSerializer.load(modelZipFile, loadUpdaterState);
|
||||
```
|
||||
|
||||
## Implementation
|
||||
|
||||
### SDZ Format Details
|
||||
|
||||
The SDZ format addresses the need for single-file distribution of large models through the following implementation:
|
||||
|
||||
1. **ZIP Container**: The SDZ format uses a standard ZIP archive as its container, enabling compatibility with standard zip tools for inspection and extraction.
|
||||
|
||||
2. **Internal Structure**:
|
||||
- The ZIP archive contains one or more SDNB format files
|
||||
- The first file (shard0) contains the graph structure
|
||||
- Subsequent files contain variables distributed across shards
|
||||
- Consistent naming convention ensures proper loading sequence
|
||||
|
||||
3. **Sharding Implementation**:
|
||||
- `SDZSerializer.save()` internally calls `SameDiffSerializer.saveAutoShard()` to create SDNB files
|
||||
- These files are then compressed and packaged into the ZIP archive
|
||||
- Automatic cleanup of temporary files after ZIP creation
|
||||
- Distributed variable serialization across shards based on size
|
||||
|
||||
4. **Loading Process*``*:
|
||||
- `SDZSerializer.load()` extracts all SDNB files to a temporary directory
|
||||
- Loads shard 0 first to establish graph structure
|
||||
- Loads variable data from remaining shards
|
||||
- Ensures temporary directory cleanup
|
||||
- Returns fully reconstituted SameDiff instance
|
||||
|
||||
5. **ZIP Operations**:
|
||||
- Uses standard Java ZIP APIs for maximum compatibility
|
||||
- Implements efficient I/O with buffering for large file handling
|
||||
- Security measures against zip slip vulnerabilities
|
||||
- Validation of ZIP structure integrity
|
||||
|
||||
6. **Optimizations**:
|
||||
- Manifest-based array lookup for efficient loading
|
||||
- Smart buffer management to minimize memory pressure
|
||||
- Native byte order handling for cross-platform compatibility
|
||||
- Verification steps to validate loaded model integrity
|
||||
|
||||
### Performance Considerations
|
||||
|
||||
The SDZ format balances compression benefits against performance requirements:
|
||||
|
||||
1. **Serialization Performance**:
|
||||
- Slight additional overhead for ZIP compression
|
||||
- Parallelized compression when possible
|
||||
- Progressive ZIP writing to avoid memory spikes
|
||||
|
||||
2. **Deserialization Performance**:
|
||||
- Sequential extraction for predictable memory usage
|
||||
- Lazy loading strategies for large variables
|
||||
- Efficient memory mapping for large arrays when possible
|
||||
- Verification during loading to ensure data integrity
|
||||
|
||||
3. **Storage Efficiency**:
|
||||
- Typically 30-50% size reduction through compression
|
||||
- Optimal balance between compression level and performance
|
||||
- Compression ratio varies based on parameter data patterns
|
||||
|
||||
## Trade-offs and Consequences
|
||||
|
||||
### Design Trade-offs
|
||||
|
||||
1. **FlatBuffers Compatibility vs. Unlimited Model Size**:
|
||||
- We maintain compatibility with 32-bit FlatBuffers for graph structure
|
||||
- We overcome FlatBuffers' 2GB size limitation through our sharding approach
|
||||
- This allows us to leverage FlatBuffers' efficiency for small graph structures while supporting NDArrays of any size
|
||||
|
||||
2. **Single File Format vs. Performance**:
|
||||
- We chose ZIP for its ubiquity, tooling support, and single-file deployment benefits
|
||||
- ZIP allows self-contained distribution while accepting some performance overhead during compression/decompression
|
||||
- This trades some loading speed for better deployment experience and reduced operational complexity
|
||||
|
||||
3. **Metadata Extensibility vs. Format Complexity**:
|
||||
- We implement an extensible metadata system similar to GGUF
|
||||
- This allows adding/updating metadata without reserializing the entire model
|
||||
- The increased format complexity is justified by the flexibility to evolve models over time
|
||||
|
||||
4. **Cross-Platform Support vs. Optimization**:
|
||||
- We prioritize cross-platform compatibility over platform-specific optimizations
|
||||
- This ensures models can be shared across environments but may not achieve maximum performance on specialized hardware
|
||||
|
||||
### Advantages
|
||||
|
||||
1. **Simplified Deployment**:
|
||||
- Single file deployment with SDZ format
|
||||
- Easier distribution and management
|
||||
- Reduced risk of missing files or shard mismatches
|
||||
|
||||
2. **Enhanced Model Storage**:
|
||||
- Support for NDArrays and models of any size
|
||||
- Efficient storage with ZIP compression
|
||||
- Selective loading of model components
|
||||
|
||||
3. **Better Metadata Management**:
|
||||
- Standardized tracking of model attributes
|
||||
- Version management for compatibility
|
||||
- Custom metadata for specific requirements
|
||||
- Post-training metadata additions without parameter reserializing
|
||||
|
||||
4. **First-Class Sharding**:
|
||||
- Explicit support for very large models
|
||||
- Intelligent variable distribution
|
||||
- Efficient loading of sharded models
|
||||
|
||||
5. **Complete Backward Compatibility**:
|
||||
- Seamless support for reading existing formats
|
||||
- Automatic format detection and handling
|
||||
- No disruption to existing workflows
|
||||
- Migration path for older models
|
||||
|
||||
### Disadvantages
|
||||
|
||||
1. **Implementation Complexity**:
|
||||
- More complex than previous FlatBuffers-only approach
|
||||
- Additional code paths for format handling
|
||||
- Need for comprehensive testing across formats
|
||||
|
||||
2. **Performance Considerations**:
|
||||
- Compression/decompression time with SDZ format
|
||||
- Temporary storage requirements during extraction
|
||||
- Slight overhead for small models
|
||||
|
||||
3. **Tool Ecosystem**:
|
||||
- Need for updates to existing tooling
|
||||
- Additional format documentation requirements
|
||||
- Migration guidance for existing models
|
||||
|
||||
## Technical Implementation
|
||||
|
||||
### Format Detection Algorithm
|
||||
```java
|
||||
public static SameDiff load(File file, boolean loadUpdaterState) throws IOException {
|
||||
// Check if it's a ZIP file first (SDZ format)
|
||||
if (isZipFile(file)) {
|
||||
return SDZSerializer.load(file, loadUpdaterState);
|
||||
}
|
||||
|
||||
// Not a ZIP, check if it's a native SDNB file
|
||||
if (isValidSdnbFile(file)) {
|
||||
return SameDiffSerializer.load(file, loadUpdaterState);
|
||||
}
|
||||
|
||||
// Check if it's a base name for sharded files
|
||||
if (hasShardedFiles(file)) {
|
||||
return SameDiffSerializer.loadSharded(file, loadUpdaterState);
|
||||
}
|
||||
|
||||
// Unsupported format
|
||||
throw new UnsupportedOperationException("Unrecognized model format");
|
||||
}
|
||||
```
|
||||
|
||||
### SDZ Implementation
|
||||
```java
|
||||
public static void save(SameDiff sameDiff, File outputZipFile, boolean saveUpdaterState,
|
||||
Map<String, String> metadata) throws IOException {
|
||||
// Create temporary directory for SDNB files
|
||||
Path tempDir = Files.createTempDirectory("sdz-serializer-save-");
|
||||
|
||||
try {
|
||||
// Save using SDNB serializer to temporary directory
|
||||
File internalSavePath = new File(tempDir.toFile(), "model");
|
||||
SameDiffSerializer.saveAutoShard(sameDiff, internalSavePath, saveUpdaterState, metadata);
|
||||
|
||||
// Collect all files to add to ZIP
|
||||
List<File> filesToZip = new ArrayList<>();
|
||||
findAllFilesRecursively(tempDir.toFile(), filesToZip);
|
||||
|
||||
// Create ZIP archive
|
||||
createZipArchive(outputZipFile, filesToZip);
|
||||
} finally {
|
||||
// Clean up temporary directory
|
||||
FileUtils.deleteDirectory(tempDir.toFile());
|
||||
}
|
||||
}
|
||||
|
||||
public static SameDiff load(File modelZipFile, boolean loadUpdaterState) throws IOException {
|
||||
// Extract ZIP to temporary directory
|
||||
Path tempDir = Files.createTempDirectory("sdz-serializer-load-");
|
||||
|
||||
try {
|
||||
// Extract ZIP contents
|
||||
extractZip(modelZipFile, tempDir.toFile());
|
||||
|
||||
// Determine the path to load from
|
||||
File loadPath = determineLoadPath(tempDir.toFile());
|
||||
|
||||
// Load using SDNB serializer
|
||||
return SameDiffSerializer.load(loadPath, loadUpdaterState);
|
||||
} finally {
|
||||
// Clean up temporary directory
|
||||
FileUtils.deleteDirectory(tempDir.toFile());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Migration Guidelines
|
||||
|
||||
For existing users:
|
||||
|
||||
1. **Loading Existing Models**:
|
||||
- No changes needed, automatic format detection handles existing models
|
||||
|
||||
2. **Converting to SDZ Format**:
|
||||
- Use `SDZSerializer.save()` with existing SameDiff instances
|
||||
- Alternatively, load existing models and save in SDZ format
|
||||
|
||||
3. **When to Use Each Format**:
|
||||
- SDNB: For highest performance, particularly during training
|
||||
- SDZ: For deployment, storage efficiency, and single-file distribution
|
||||
- Sharded formats: For very large models exceeding memory limits
|
||||
@@ -0,0 +1,262 @@
|
||||
# ADR: Migrate Project Namespaces to org.eclipse.deeplearning4j using OpenRewrite
|
||||
|
||||
## Status
|
||||
|
||||
Proposed
|
||||
|
||||
Proposed by: Adam Gibson (May 8, 2025)
|
||||
|
||||
Discussed with: Paul Dubs
|
||||
|
||||
## Context
|
||||
|
||||
The Deeplearning4j project ecosystem currently utilizes multiple root Java package namespaces, primarily `org.nd4j`, `org.deeplearning4j`, and `org.datavec`, along with existing code under `org.eclipse.deeplearning4j`, bindings to
|
||||
external libraries (like FlatBuffers, ONNX, TensorFlow, Python), and various `contrib` and `codegen` modules. This distributed namespace structure can make navigation, maintenance, and
|
||||
understanding component relationships more challenging than necessary.
|
||||
|
||||
The goal is to consolidate the core project codebase under a single, unified root namespace: `org.eclipse.deeplearning4j`. This aligns with the project's stewardship under the Eclipse Foundation and aims to create a clearer,
|
||||
more consistent, and maintainable structure. Given the project's large scale, an automated refactoring approach using [OpenRewrite](https://docs.openrewrite.org/reference/rewrite-maven-plugin) is optimal for feasibility. This ADR proposes the strategy and
|
||||
specific rules for this migration.
|
||||
|
||||
This is part of a 2 phase release plan where a milestone release that has the old package names is performed followed by the major renamespacing.
|
||||
|
||||
|
||||
## Proposal
|
||||
|
||||
This ADR proposes using a comprehensive [OpenRewrite recipe](https://docs.openrewrite.org/reference/rewrite-maven-plugin) to automatically refactor the project's primary Java packages into the `org.eclipse.deeplearning4j` namespace. The refactoring follows a two-phase conceptual approach:
|
||||
|
||||
1. **Foundational Re-basing:** Map the main existing root packages (`org.nd4j`, `org.deeplearning4j`, `org.datavec`) to logical sub-packages under the new root (`.nd4j`, `.dl4jcore`, `.datavec`).
|
||||
2. **Architectural Refinement:** Apply more specific rules to achieve a clearer target structure, including:
|
||||
* Elevating the UI components to a top-level `org.eclipse.deeplearning4j.ui` package.
|
||||
* Consolidating ND4J backend-specific code (including former `jita` and `jcublas` into `.cuda`) under `org.eclipse.deeplearning4j.nd4j.backend.[type]`.
|
||||
* Structuring DataVec components functionally under `org.eclipse.deeplearning4j.datavec.*`.
|
||||
* Isolating runtime bindings for external libraries under `org.eclipse.deeplearning4j.bindings.*`.
|
||||
* Providing clear namespaces for `contrib` and `codegen` modules.
|
||||
* Establishing top-level `common` and `resources` packages (though full consolidation requires further steps).
|
||||
|
||||
The core of the proposal is the following OpenRewrite YAML recipe, which implements the automated parts of this strategy:
|
||||
|
||||
|
||||
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
|
||||
# OpenRewrite Recipe for Deeplearning4j Namespace Migration #
|
||||
# Target Root: org.eclipse.deeplearning4j #
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
|
||||
type: org.openrewrite.Recipe
|
||||
name: org.eclipse.deeplearning4j.refactor.NamespaceMigration
|
||||
displayName: Migrate DL4J Project to org.eclipse.deeplearning4j Namespace
|
||||
description: Comprehensive refactoring of core DL4J, ND4J, and DataVec packages under the org.eclipse.deeplearning4j root, including backend consolidation and UI elevation.
|
||||
|
||||
recipeList:
|
||||
#--------------------------------------------------------------------------#
|
||||
# Step 1: Handle Specific Component Moves (More Specific Rules First) #
|
||||
#--------------------------------------------------------------------------#
|
||||
|
||||
# --- UI Components -> o.e.d.ui.* ---
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.deeplearning4j.ui
|
||||
newPackageName: org.eclipse.deeplearning4j.ui
|
||||
recursive: true
|
||||
# Note: This rule assumes org.deeplearning4j.ui.* should be elevated.
|
||||
# It must run before the general org.deeplearning4j -> o.e.d.dl4jcore rule.
|
||||
|
||||
# --- ND4J Backends -> o.e.d.nd4j.backend.[type].* ---
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.linalg.jcublas # Maps to cuda backend
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j.backend.cuda
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.jita # JITA components also map to cuda backend
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j.backend.cuda # Target same new package
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.linalg.cpu.nativecpu # Maps to cpu backend
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j.backend.cpu
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.presets.cuda # CUDA presets
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j.backend.cuda.presets
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.presets.cpu # CPU presets
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j.backend.cpu.presets
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage: # Minimal backend?
|
||||
oldPackageName: org.nd4j.linalg.minimal
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j.backend.minimal
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.presets.minimal
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j.backend.minimal.presets
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage: # Common presets utils
|
||||
oldPackageName: org.nd4j.presets
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j.backend.common.presets
|
||||
recursive: true
|
||||
|
||||
# --- Native Ops / Bindings Facades ---
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.nativeblas # Common native facade classes like NativeOpsHolder
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j.nativeops.common
|
||||
recursive: true
|
||||
|
||||
# --- Runtime Bindings (Wrappers/Runners for external libs/runtimes) ---
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.onnxruntime # ONNX Runtime Runner
|
||||
newPackageName: org.eclipse.deeplearning4j.bindings.onnx.runtime
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.tensorflow.conversion # TF Runtime Runner/Converter
|
||||
newPackageName: org.eclipse.deeplearning4j.bindings.tensorflow.runtime
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.tensorflowlite # TF Lite Runner
|
||||
newPackageName: org.eclipse.deeplearning4j.bindings.tensorflowlite.runtime
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.tvm # TVM Runner
|
||||
newPackageName: org.eclipse.deeplearning4j.bindings.tvm.runtime
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.python4j # Python Bindings
|
||||
newPackageName: org.eclipse.deeplearning4j.bindings.python.api # Map core to .api
|
||||
recursive: true # Handles subpackages like .numpy unless more specific rules added
|
||||
|
||||
# --- Specific Contrib Modules (Using explicit contrib namespacing) ---
|
||||
# Note: fileMatcher might be needed in practice to limit these rules to specific module paths
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j # For classes directly in org.nd4j within nd4j-benchmark
|
||||
newPackageName: org.eclipse.deeplearning4j.contrib.benchmark.nd4j
|
||||
recursive: false # Avoid affecting subpackages handled below
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.bypass # Sub-package in nd4j-benchmark
|
||||
newPackageName: org.eclipse.deeplearning4j.contrib.benchmark.nd4j.bypass
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.memorypressure # Sub-package in nd4j-benchmark
|
||||
newPackageName: org.eclipse.deeplearning4j.contrib.benchmark.nd4j.memorypressure
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j # For classes directly in org.nd4j within version-updater
|
||||
newPackageName: org.eclipse.deeplearning4j.contrib.versionupdater
|
||||
recursive: false
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j.fileupdater # Sub-package in version-updater
|
||||
newPackageName: org.eclipse.deeplearning4j.contrib.versionupdater.fileupdater
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage: # For contrib blas-lapack-gen
|
||||
oldPackageName: org.deeplearning4j
|
||||
newPackageName: org.eclipse.deeplearning4j.contrib.blaslapackgen
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage: # For contrib nd4j-log-analyzer
|
||||
oldPackageName: org.nd4j.interceptor
|
||||
newPackageName: org.eclipse.deeplearning4j.contrib.nd4jloganalyzer.interceptor
|
||||
recursive: true
|
||||
|
||||
# --- Specific Codegen Modules ---
|
||||
# Note: fileMatcher might be needed in practice
|
||||
- org.openrewrite.java.ChangePackage: # For libnd4j-gen
|
||||
oldPackageName: org.nd4j.descriptor
|
||||
newPackageName: org.eclipse.deeplearning4j.codegen.descriptor
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage: # For op-codegen
|
||||
oldPackageName: org.nd4j.codegen
|
||||
newPackageName: org.eclipse.deeplearning4j.codegen.op
|
||||
recursive: true
|
||||
- org.openrewrite.java.ChangePackage: # For codegen blas-lapack-generator
|
||||
oldPackageName: org.deeplearning4j
|
||||
newPackageName: org.eclipse.deeplearning4j.codegen.blaslapack
|
||||
recursive: true
|
||||
|
||||
#--------------------------------------------------------------------------#
|
||||
# Step 2: Apply General Re-basing Rules (Less Specific Rules Last) #
|
||||
#--------------------------------------------------------------------------#
|
||||
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.nd4j # General ND4J code not caught by specific rules above
|
||||
newPackageName: org.eclipse.deeplearning4j.nd4j
|
||||
recursive: true
|
||||
# This will map remaining org.nd4j.* like linalg.api.*, samediff.*, common.*, etc.
|
||||
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.deeplearning4j # General DL4J Core code (excluding UI already handled)
|
||||
newPackageName: org.eclipse.deeplearning4j.dl4jcore
|
||||
recursive: true
|
||||
# This will map nn.*, models.*, datasets.*, eval.*, nlp.*, etc.
|
||||
|
||||
- org.openrewrite.java.ChangePackage:
|
||||
oldPackageName: org.datavec # General DataVec code
|
||||
newPackageName: org.eclipse.deeplearning4j.datavec
|
||||
recursive: true
|
||||
# This will map api.*, transform.*, image.*, arrow.*, etc.
|
||||
|
||||
#--------------------------------------------------------------------------#
|
||||
# Step 3: Exclusions / Manual Handling (Commented Out for Safety) #
|
||||
#--------------------------------------------------------------------------#
|
||||
|
||||
# --- DO NOT AUTOMATICALLY RUN THESE for generated binding specs ---
|
||||
# --- Prefer changing generator configurations ---
|
||||
# - org.openrewrite.java.ChangePackage:
|
||||
# oldPackageName: com.google.flatbuffers
|
||||
# newPackageName: org.eclipse.deeplearning4j.bindings.flatbuffers.spec # Or .core
|
||||
# recursive: true
|
||||
# - org.openrewrite.java.ChangePackage:
|
||||
# oldPackageName: onnx
|
||||
# newPackageName: org.eclipse.deeplearning4j.bindings.onnx.spec
|
||||
# recursive: true
|
||||
# - org.openrewrite.java.ChangePackage:
|
||||
# oldPackageName: tensorflow # Includes tensorflow.framework, tensorflow.eager etc
|
||||
# newPackageName: org.eclipse.deeplearning4j.bindings.tensorflow.spec
|
||||
# recursive: true
|
||||
|
||||
#--------------------------------------------------------------------------#
|
||||
# Step 4: Potential Cleanup (Run Separately After Migration) #
|
||||
#--------------------------------------------------------------------------#
|
||||
|
||||
# - org.openrewrite.java.cleanup.OrganizeImports
|
||||
# - org.openrewrite.java.cleanup.RemoveUnusedImports
|
||||
# - org.openrewrite.java.format.AutoFormat
|
||||
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
|
||||
# End of Recipe #
|
||||
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
|
||||
|
||||
|
||||
## Consequences
|
||||
|
||||
### Advantages
|
||||
|
||||
* **Namespace Consistency:** Provides a single, unified root namespace (`org.eclipse.deeplearning4j`) for the entire project, aligning with Eclipse Foundation stewardship.
|
||||
* **Improved Structure:** The refined Phase 2 structure (explicit backends, UI, common components, bindings) enhances architectural clarity and modularity compared to the original scattered namespaces.
|
||||
* **Easier Navigation & Maintenance:** A consistent structure makes it easier for developers to find code, understand component relationships, and perform maintenance.
|
||||
* **Foundation for Future Work:** Creates a cleaner base for developing new features and potentially consolidating shared functionality (e.g., in common utilities, resource management).
|
||||
|
||||
### Disadvantages
|
||||
|
||||
* **Significant Initial Effort:** Running this large-scale refactoring requires careful setup, dry runs, review, and validation.
|
||||
* **Potential for Breakage:** Automated refactoring on this scale carries inherent risks. Build script issues, reflection usage, non-Java file references (like in XML or properties), and complex type resolution might lead to compilation errors or runtime issues requiring manual fixes.
|
||||
* **Testing Burden:** Requires executing the full test suite (unit, integration, platform-specific) to ensure correctness after refactoring. Test code itself will also be refactored and needs verification.
|
||||
* **External Bindings Complexity:** Handling generated code or bindings for libraries like FlatBuffers, ONNX, and TensorFlow requires careful consideration beyond simple package renaming; modifying generator configurations is preferred.
|
||||
* **Learning Curve:** Teams unfamiliar with OpenRewrite will need to learn its usage and recipe development, especially for any necessary follow-up complex refactorings.
|
||||
* **Merge Conflicts:** This constitutes a large change, potentially creating significant merge conflicts for ongoing feature branches. Coordination is essential.
|
||||
|
||||
### Technical details about the Namespace Refactoring
|
||||
|
||||
* **Tooling:** The proposal relies on OpenRewrite and its `org.openrewrite.java.ChangePackage` recipe.
|
||||
* **Recipe Structure:** The YAML recipe uses a `recipeList` and applies more specific package mapping rules *before* more general ones to handle structural changes like backend separation and UI elevation correctly.
|
||||
* **`ChangePackage` Limitations:** This recipe primarily uses `ChangePackage`. Achieving the full Phase 2 vision, especially consolidating classes from multiple old locations into a single new package (e.g., common utilities), may require additional recipes using `MoveClass`, `MovePackage`, or custom Java-based recipes after this initial migration.
|
||||
* **Exclusions:** Generated binding specifications (`com.google.flatbuffers`, `onnx.*`, `tensorflow.*`) are intentionally excluded from automated changes due to high risk. Runtime wrappers/runners for these *are* included in the refactoring.
|
||||
* **Process:** The recommended process involves: backup -> dry run -> review -> iterative recipe refinement (if needed) -> apply changes -> compile -> test -> manual code review -> commit.
|
||||
|
||||
## Discussion
|
||||
|
||||
The proposed structure aims for a balance between respecting the original component boundaries (ND4J, DL4J Core, DataVec) and creating a more modern, cohesive structure under the Eclipse Foundation namespace. Key decisions reflected in the proposal:
|
||||
|
||||
* Explicitly separating backend implementations (`.nd4j.backend.[type]`).
|
||||
* Elevating UI to a top-level component (`.ui`).
|
||||
* Designating clear homes for bindings (`.bindings`), contrib (`.contrib`), and codegen (`.codegen`).
|
||||
* Establishing placeholders for consolidated common code (`.common`) and resource management (`.resources`), acknowledging full consolidation is a subsequent step.
|
||||
* Using `.dl4jcore` to house the bulk of the original `org.deeplearning4j` code to distinguish it clearly within the new structure.
|
||||
|
||||
Further discussion within the development team is needed to confirm these target structures and plan the implementation, particularly the steps required beyond the initial automated recipe application for deeper Phase 2 refinements.
|
||||
@@ -0,0 +1,51 @@
|
||||
# Contributing to Deeplearning4j
|
||||
|
||||
Thanks for your interest in DL4J. Our goal is to bring fast, open-source deep learning to all JVM-based communities.
|
||||
|
||||
|
||||
## Getting Started
|
||||
|
||||
Deeplearning4j's [open issues are here](https://github.com/eclipse/deeplearning4j/issues). In time, we'll tag issues that would make a good first pull request for new contributors. An easy way to get started helping the project is to *file an issue*. You can do that on the Deeplearning4j issues page by clicking on the green button at the right. Issues can include bugs to fix, features to add, or documentation that looks outdated.
|
||||
|
||||
Note that you will need to [build dl4j from source](https://deeplearning4j.konduit.ai/multi-project/how-to-guides/build-from-source)
|
||||
|
||||
For some tips on contributing to open source, this [post is helpful](https://smartbear.com/blog/test-and-monitor/14-ways-to-contribute-to-open-source-without-being/).
|
||||
|
||||
## Contributions
|
||||
|
||||
Deeplearning4j welcomes contributions from everyone.
|
||||
|
||||
Contributions to Deeplearning4j should be made in the form of GitHub pull requests. Each pull request will
|
||||
be reviewed by a core contributor (someone with permission to land patches) and either landed in the
|
||||
main tree or given feedback for changes that would be required.
|
||||
|
||||
## Pull Request Checklist
|
||||
|
||||
- Branch from the master branch and, if needed, rebase to the current master
|
||||
branch before submitting your pull request. If it doesn't merge cleanly with
|
||||
master you may be asked to rebase your changes.
|
||||
|
||||
- Commits should be as small as possible, while ensuring that each commit is
|
||||
correct independently (i.e., each commit should compile and pass tests).
|
||||
|
||||
- Don't put submodule updates in your pull request unless they are to landed
|
||||
commits.
|
||||
|
||||
- If your patch is not getting reviewed or you need a specific person to review
|
||||
it, you can @-reply a reviewer asking for a review in the pull request or a
|
||||
comment.
|
||||
|
||||
- Work-in-progress pull requests are welcome. Please prefix them with `[WIP]` to tell the continuous integration (CI) backend not to run tests/checks on them (until that tag is removed and another commit is pushed up).
|
||||
|
||||
- Add tests relevant to the fixed bug or new feature.
|
||||
|
||||
## Conduct & License
|
||||
|
||||
We follow the [Rust Code of Conduct](http://www.rust-lang.org/conduct.html).
|
||||
|
||||
All code in this repository is released under the Apache Software Foundation License, 2.0, and by contributing to this repository, you agree to release that contribution under that same license.
|
||||
|
||||
|
||||
## Eclipse Contributor Agreement and Commit Signing
|
||||
|
||||
Please see the following page for details: [https://deeplearning4j.konduit.ai/multi-project/how-to-guides/contribute/eclipse-contributors](https://deeplearning4j.konduit.ai/multi-project/how-to-guides/contribute/eclipse-contributors)
|
||||
@@ -0,0 +1,384 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
##########################
|
||||
|
||||
Keras code
|
||||
|
||||
Auto-generated documentation: https://github.com/deeplearning4j/deeplearning4j/blob/master/docs/doc_generator.py
|
||||
|
||||
COPYRIGHT
|
||||
|
||||
All contributions by François Chollet:
|
||||
Copyright (c) 2015 - 2018, François Chollet.
|
||||
All rights reserved.
|
||||
|
||||
All contributions by Google:
|
||||
Copyright (c) 2015 - 2018, Google, Inc.
|
||||
All rights reserved.
|
||||
|
||||
All contributions by Microsoft:
|
||||
Copyright (c) 2017 - 2018, Microsoft, Inc.
|
||||
All rights reserved.
|
||||
|
||||
All other contributions:
|
||||
Copyright (c) 2015 - 2018, the respective contributors.
|
||||
All rights reserved.
|
||||
|
||||
Each contributor holds copyright over their respective contributions.
|
||||
The project versioning (Git) records all such contribution source information.
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
|
||||
##########################
|
||||
|
||||
OpenCSV Code
|
||||
|
||||
CSVParser: https://github.com/deeplearning4j/deeplearning4j/blob/master/datavec/datavec-api/src/main/java/org/datavec/api/records/reader/impl/csv/SerializableCSVParser.java
|
||||
|
||||
Apache 2.0 License
|
||||
|
||||
All contributions by Bytecode Pty Ltd.
|
||||
Copyright 2005 Bytecode Pty Ltd.
|
||||
All rights reserved.
|
||||
|
||||
|
||||
##########################
|
||||
|
||||
Aeron Code
|
||||
|
||||
Modifed Code: nd4j/nd4j-serde/nd4j-aeron/src/main/java/org/nd4j/aeron/ipc/AeronUtil.java
|
||||
|
||||
Copyright 2014 - 2016 Real Logic Ltd. All rights reserved.
|
||||
|
||||
Apache License, Version 2.0
|
||||
|
||||
|
||||
##########################
|
||||
|
||||
cnpy Code
|
||||
|
||||
Forked Code: libnd4j/include/cnpy/
|
||||
|
||||
The MIT License
|
||||
|
||||
Copyright (c) Carl Rogers, 2011
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
|
||||
##########################
|
||||
|
||||
Protocol Buffers Code
|
||||
|
||||
Codebase: nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/google/protobuf/
|
||||
|
||||
Protocol Buffers - Google's data interchange format
|
||||
Copyright 2008 Google Inc. All rights reserved.
|
||||
https://developers.google.com/protocol-buffers/
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
##########################
|
||||
|
||||
ONNX Code
|
||||
|
||||
Protocol Buffers: nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/onnx/
|
||||
|
||||
Copyright (c) Facebook Inc. and Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT license.
|
||||
|
||||
|
||||
##########################
|
||||
|
||||
TensorFlow Code
|
||||
|
||||
Protocol Buffers: nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/protobuf/tf/tensorflow/core/
|
||||
Operations: libnd4j/include/ops/declarable/generic/parity_ops/
|
||||
|
||||
Copyright 2015-2017 The TensorFlow Authors. All rights reserved.
|
||||
|
||||
Apache License, Version 2.0
|
||||
|
||||
|
||||
##########################
|
||||
|
||||
Ansj Code
|
||||
|
||||
Codebase: deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp-chinese/src/main/java/org/ansj/
|
||||
Resources: deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp-chinese/src/main/resources/
|
||||
|
||||
Copyright 2011-2016 ansj_seg. All rights reserved.
|
||||
|
||||
Apache License, Version 2.0
|
||||
|
||||
|
||||
##########################
|
||||
|
||||
Kuromoji Code
|
||||
|
||||
Codebase: deeplearning4j/deeplearning4j-nlp-parent/deeplearning4j-nlp-japanese/src/main/java/com/atilika/kuromoji/
|
||||
|
||||
Copyright (c) 2010-2015 Atilika Inc. and contributors. All rights reserved.
|
||||
|
||||
Apache License, Version 2.0
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
Eclipse Deeplearning4j
|
||||
Copyright 2021 Eclipse Deeplearning4j Contributors
|
||||
|
||||
This product includes software developed at
|
||||
The Apache Software Foundation (http://www.apache.org/).
|
||||
|
||||
This product includes software developed at
|
||||
* Skymind Inc (Apache 2.0). Copyright (C) 2015-2018 Skymind Inc .
|
||||
|
||||
This product includes software developed at
|
||||
* Konduit KK (Apache 2.0). Copyright (C) 2020.
|
||||
|
||||
|
||||
This product includes software from the Tensorflow Project (Apache 2.0).
|
||||
* Copyright (C) 2015-2018 Tensorflow Authors.
|
||||
|
||||
# https://github.com/onnx/onnx
|
||||
|
||||
This product includes software from the Onnx Project project (Apache 2.0).
|
||||
* Copyright (C) 2020 Onnx Contributors (https://github.com/onnx/onnx)
|
||||
@@ -0,0 +1,110 @@
|
||||
<p align="center">
|
||||
<img src="https://www.zeljkoobrenovic.com/tools/tech/images/eclipse_deeplearning4j.png">
|
||||
</p>
|
||||
|
||||
[](https://deeplearning4j.konduit.ai/)
|
||||
[](https://community.konduit.ai/)
|
||||
[](https://javadoc.io/doc/org.deeplearning4j/deeplearning4j-nn)
|
||||
[](https://javadoc.io/doc/org.nd4j/nd4j-api)
|
||||
[](LICENSE)
|
||||
|
||||
|
||||
|
||||
The **[Eclipse Deeplearning4J](https://deeplearning4j.konduit.ai/)** (DL4J) ecosystem is a set of projects intended to support all the needs of a JVM based deep learning application. This means starting with the raw data, loading and preprocessing it from wherever and whatever format it is in to building and tuning a wide variety of simple and complex deep learning networks.
|
||||
|
||||
Because Deeplearning4J runs on the JVM you can use it with a wide variety of JVM based languages other than Java, like Scala, Kotlin, Clojure and many more.
|
||||
|
||||
The DL4J stack comprises of:
|
||||
- **DL4J**: High level API to build MultiLayerNetworks and ComputationGraphs with a variety of layers, including custom ones. Supports importing Keras models from h5, including tf.keras models (as of 1.0.0-beta7) and also supports distributed training on Apache Spark
|
||||
- **ND4J**: General purpose linear algebra library with over 500 mathematical, linear algebra and deep learning operations. ND4J is based on the highly-optimized C++ codebase LibND4J that provides CPU (AVX2/512) and GPU (CUDA) support and acceleration by libraries such as OpenBLAS, OneDNN (MKL-DNN), cuDNN, cuBLAS, etc
|
||||
- **SameDiff** : Part of the ND4J library, SameDiff is our automatic differentiation / deep learning framework. SameDiff uses a graph-based (define then run) approach, similar to TensorFlow graph mode. Eager graph (TensorFlow 2.x eager/PyTorch) graph execution is planned. SameDiff supports importing TensorFlow frozen model format .pb (protobuf) models. Import for ONNX, TensorFlow SavedModel and Keras models are planned. Deeplearning4j also has full SameDiff support for easily writing custom layers and loss functions.
|
||||
- **DataVec**: ETL for machine learning data in a wide variety of formats and files (HDFS, Spark, Images, Video, Audio, CSV, Excel etc)
|
||||
- **LibND4J** : C++ library that underpins everything. For more information on how the JVM acceses native arrays and operations refer to [JavaCPP](https://github.com/bytedeco/javacpp)
|
||||
- **Python4J**: Bundled cpython execution for the JVM
|
||||
|
||||
All projects in the DL4J ecosystem support Windows, Linux and macOS. Hardware support includes CUDA GPUs (10.0, 10.1, 10.2 except OSX), x86 CPU (x86_64, avx2, avx512), ARM CPU (arm, arm64, armhf) and PowerPC (ppc64le).
|
||||
|
||||
## Community Support
|
||||
For support for the project, please go over to https://community.konduit.ai/
|
||||
|
||||
## Using Eclipse Deeplearning4J in your project
|
||||
|
||||
Deeplearning4J has quite a few dependencies. For this reason we only support usage with a build tool.
|
||||
|
||||
```xml
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>deeplearning4j-core</artifactId>
|
||||
<version>1.0.0-M2.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-native-platform</artifactId>
|
||||
<version>1.0.0-M2.1</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
```
|
||||
|
||||
Add these dependencies to your pom.xml file to use Deeplearning4J with the CPU backend. A full standalone project example is [available in the example repository](https://github.com/eclipse/deeplearning4j-examples), if you want to start a new Maven project from scratch.
|
||||
|
||||
## Code samples
|
||||
|
||||
Due to DL4J being a multi faceted project
|
||||
with several modules in the mono repo, we recommend looking at the examples
|
||||
for a taste of different usages of the different modules. Below
|
||||
we'll link to examples for each module.
|
||||
|
||||
1. ND4J: https://github.com/deeplearning4j/deeplearning4j-examples/tree/master/nd4j-ndarray-examples
|
||||
2. DL4J: https://github.com/deeplearning4j/deeplearning4j-examples/tree/master/dl4j-examples
|
||||
3. Samediff: https://github.com/deeplearning4j/deeplearning4j-examples/tree/master/samediff-examples
|
||||
4. Datavec: https://github.com/deeplearning4j/deeplearning4j-examples/tree/master/data-pipeline-examples
|
||||
5. Python4j: https://deeplearning4j.konduit.ai/python4j/tutorials/quickstart
|
||||
|
||||
|
||||
For users looking for being able to run models from other frameworks, see:
|
||||
1. Onnx: https://github.com/deeplearning4j/deeplearning4j-examples/tree/master/onnx-import-examples
|
||||
2. Tensorflow/Keras: https://github.com/deeplearning4j/deeplearning4j-examples/tree/master/tensorflow-keras-import-examples
|
||||
|
||||
|
||||
## Documentation, Guides and Tutorials
|
||||
You can find the official documentation for Deeplearning4J and the other libraries of its ecosystem at http://deeplearning4j.konduit.ai/.
|
||||
|
||||
## Want some examples?
|
||||
We have separate repository with various examples available: https://github.com/eclipse/deeplearning4j-examples
|
||||
|
||||
## Building from source
|
||||
It is preferred to use the official pre-compiled releases (see above). But if you want to build from source, first take a look at the prerequisites for building from source here: https://deeplearning4j.konduit.ai/multi-project/how-to-guides/build-from-source. Various instructions for cpu and gpu builds can be found there. Please go to our [forums](https://community.konduit.ai/) for further help.
|
||||
|
||||
|
||||
## Running tests
|
||||
|
||||
In order to run tests, please see the platform-tests module.
|
||||
This module only runs on jdk 11 (mostly due to spark and bugs with older scala versions + JDK 17)
|
||||
|
||||
platform-tests allows you to run dl4j for different backends.
|
||||
There are a few properties you can specify on the command line:
|
||||
1. backend.artifactId: this defaults to nd4j-native and will run tests on cpu,you can specify other backends like nd4j-cuda-11.6
|
||||
2. dl4j.version: You can change the dl4j version that the tests run against. This defaults to 1.0.0-SNAPSHOT.
|
||||
|
||||
More parameters can be found here:
|
||||
https://github.com/deeplearning4j/deeplearning4j/blob/c1bf8717e4839c8930e9c43183bf7b94d0cf84dc/platform-tests/pom.xml#L47
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Running project in Intellij IDEA:
|
||||
1. Ensure you follow https://stackoverflow.com/questions/45370178/exporting-a-package-from-system-module-is-not-allowed-with-release on jdk 9 or later
|
||||
2. Ignore all nd4j-shade submodules. Right click on each folder and click: Maven -> Ignore project
|
||||
|
||||
|
||||
## License
|
||||
|
||||
[Apache License 2.0](LICENSE)
|
||||
|
||||
|
||||
## Commercial Support
|
||||
Deeplearning4J is actively developed by the team at [Konduit K.K.](https://konduit.ai).
|
||||
|
||||
[If you need any commercial support feel free to reach out to us. at [support@konduit.ai](mailto:support@konduit.ai)
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`deeplearning4j/deeplearning4j`
|
||||
- 原始仓库：https://github.com/deeplearning4j/deeplearning4j
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
@@ -0,0 +1,122 @@
|
||||
#!/bin/bash
|
||||
|
||||
file_name=""
|
||||
|
||||
|
||||
|
||||
function append_args {
|
||||
if [ -n "$1" ]; then
|
||||
file_name="${file_name}-$1"
|
||||
fi
|
||||
}
|
||||
|
||||
function create_config {
|
||||
if [ "$1" == "cuda" ]; then
|
||||
engine="ENGINE_CUDA"
|
||||
elif [ "$1" == "aurora" ]; then
|
||||
engine="ENGINE_AURORA"
|
||||
else
|
||||
engine="ENGINE_CPU"
|
||||
|
||||
fi
|
||||
|
||||
config="${GITHUB_WORKSPACE}/libnd4j/blasbuild/$1/include/config.h"
|
||||
config_copy="${GITHUB_WORKSPACE}/libnd4j/include/config.h"
|
||||
if test -f "${config_copy}"; then
|
||||
rm -f "${config_copy}"
|
||||
fi
|
||||
|
||||
if test -f "${config}"; then
|
||||
rm -f "${config}"
|
||||
fi
|
||||
|
||||
print_config "${config_copy}" "${engine}"
|
||||
print_config "${config}" "${engine}"
|
||||
|
||||
|
||||
}
|
||||
|
||||
function print_config {
|
||||
echo "Generating config.h $1"
|
||||
echo "#ifndef LIBND4J_CONFIG_H" >> "$1"
|
||||
echo "#define LIBND4J_CONFIG_H" >> "$1"
|
||||
echo "#define DEFAULT_ENGINE samediff::$2" >> "$1"
|
||||
echo "#endif" >> "$1"
|
||||
echo "Generated config.h at $1"
|
||||
cat "$1"
|
||||
}
|
||||
|
||||
function copy_lib {
|
||||
cp -rf "libnd4j/blasbuild/"* "${GITHUB_WORKSPACE}/libnd4j/blasbuild/"
|
||||
echo "Copied libraries in libnd4j/blasbuild to ${GITHUB_WORKSPACE}/libnd4j/blasbuild"
|
||||
echo "libraries in libnd4j/blasbuild were"
|
||||
ls "libnd4j/blasbuild"
|
||||
echo "libraries in target ${GITHUB_WORKSPACE}/libnd4j/blasbuild were"
|
||||
ls "${GITHUB_WORKSPACE}/libnd4j/blasbuild"
|
||||
echo "libraries in device directory ${GITHUB_WORKSPACE}/libnd4j/blasbuild/$1 were"
|
||||
ls "${GITHUB_WORKSPACE}/libnd4j/blasbuild/$1"
|
||||
if [ "$1" == "cpu" ]; then
|
||||
echo "Copying libnd4j cpu to ${OPENBLAS_PATH}/lib"
|
||||
cp -rf "${GITHUB_WORKSPACE}/libnd4j/blasbuild/$1/blas/"* "${OPENBLAS_PATH}/lib"
|
||||
echo "Contents of ${OPENBLAS_PATH}/lib are"
|
||||
ls "${OPENBLAS_PATH}/lib"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
for var in "$@"
|
||||
do
|
||||
append_args "${var}"
|
||||
done
|
||||
|
||||
# Get rid of the first character
|
||||
file_name_2="${file_name:1:${#file_name}}"
|
||||
|
||||
|
||||
if ! [[ -z "$LIBND4J_FILE_NAME" ]]; then
|
||||
echo "Downloading file with url at $LIBND4J_FILE_NAME/${file_name_2}"
|
||||
curl "${LIBND4J_FILE_NAME}/${file_name_2}" -o file_url.txt
|
||||
# shellcheck disable=SC2006
|
||||
export LIBND4J_URL=`cat file_url.txt`
|
||||
echo "Setup LIBND4J_URL to $LIBND4J_URL"
|
||||
fi
|
||||
|
||||
if ! [[ -z "$LIBND4J_URL" ]]; then
|
||||
mkdir libnd4j_home
|
||||
cd libnd4j_home
|
||||
# Set a suffix for the downloaded libnd4j directory
|
||||
if [ -z "${LIBND4J_HOME_SUFFIX}" ]; then export LIBND4J_HOME_SUFFIX="cpu"; fi
|
||||
wget "${LIBND4J_URL}" -O libnd4j.zip
|
||||
unzip libnd4j.zip
|
||||
echo "Files in directory $(pwd) are "
|
||||
ls
|
||||
echo "Copying files to libnd4j directory"
|
||||
# Note: for builds, the whole source directory is uploaded, but a valid libnd4j home is actually only the compiled output
|
||||
cp -rf libnd4j/blasbuild/ "../libnd4j"
|
||||
echo "Files in ${GITHUB_WORKSPACE} are"
|
||||
ls "$GITHUB_WORKSPACE"
|
||||
echo "Files in libnd4j directory are ${GITHUB_WORKSPACE}/libnd4j"
|
||||
ls "${GITHUB_WORKSPACE}/libnd4j"
|
||||
cd ..
|
||||
# generated files may not exist, use in javacpp compilation
|
||||
mkdir -p ${GITHUB_WORKSPACE}/libnd4j/include/generated
|
||||
touch ${GITHUB_WORKSPACE}/libnd4j/include/generated/include_ops.h
|
||||
# Add flatbuffers include files to libnd4j directory
|
||||
git clone https://github.com/KonduitAI/flatbuffers.git
|
||||
mv flatbuffers flatbuffers-src
|
||||
cp -rf flatbuffers-src/include ${GITHUB_WORKSPACE}/libnd4j/
|
||||
echo "Copied flatbuffers to ${GITHUB_WORKSPACE}/libnd4j/include"
|
||||
if [ "$#" -gt 1 ]; then
|
||||
if [ "$2" == "cuda" ]; then
|
||||
create_config "cuda"
|
||||
copy_lib "cuda"
|
||||
elif [ "$2" == "aurora" ]; then
|
||||
create_config "aurora"
|
||||
copy_lib "aurora"
|
||||
else
|
||||
create_config "cpu"
|
||||
copy_lib "cpu"
|
||||
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcpu clean install -DskipTests -pl :libnd4j,:nd4j-native-preset,:nd4j-native -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=address,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcpu clean install -Dlibnd4j.calltrace=ON -Dlibnd4j.build=debug -DskipTests -pl :libnd4j,:nd4j-native-preset,:nd4j-native -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=address,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcpu clean install -Dlibnd4j.calltrace=ON -Dlibnd4j.build=debug -DskipTests -pl :libnd4j,:nd4j-native-preset,:nd4j-native
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcpu clean install -DskipTests -Dlibnd4j.helper=onednn -pl :libnd4j,:nd4j-native-preset,:nd4j-native -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=address,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcpu clean install -DskipTests -Dlibnd4j.calltrace=ON -Dlibnd4j.helper=onednn -Dlibnd4j.build=debug -pl :libnd4j,:nd4j-native-preset,:nd4j-native
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcpu clean install -DskipTests -Dlibnd4j.helper=onednn -pl :libnd4j,:nd4j-native-preset,:nd4j-native -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=thread,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcpu clean install -DskipTests -Dlibnd4j.helper=onednn -pl :libnd4j,:nd4j-native-preset,:nd4j-native
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcpu clean install -DskipTests -pl :libnd4j,:nd4j-native-preset,:nd4j-native
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.chip=cuda clean install -DskipTests -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1 -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=address,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.compute=86 -Dlibnd4j.chip=cuda clean install -DskipTests -Dlibnd4j.helper=cudnn -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1 -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=address,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.compute=86 -Dlibnd4j.chip=cuda -Dlibnd4j.helper=cudnn clean install -Dlibnd4j.build=debug -Dlibnd4j.calltrace=ON -DskipTests -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1 -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=address,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.compute=86 -Dlibnd4j.chip=cuda -Dlibnd4j.helper=cudnn clean install -Dlibnd4j.build=debug -Dlibnd4j.calltrace=ON -DskipTests -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1 -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=thread,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.compute=86 -Dlibnd4j.chip=cuda -Dlibnd4j.helper=cudnn clean install -Dlibnd4j.build=debug -Dlibnd4j.calltrace=ON -DskipTests -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.compute=86 -Dlibnd4j.chip=cuda clean install -DskipTests -Dlibnd4j.helper=cudnn -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1 -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=thread,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.compute=86 -Dlibnd4j.chip=cuda clean install -DskipTests -Dlibnd4j.helper=cudnn -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.compute=86 -Dlibnd4j.chip=cuda clean install -Dlibnd4j.build=debug -Dlibnd4j.calltrace=ON -DskipTests -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.chip=cuda clean install -DskipTests -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1 -Dlibnd4j.sanitize=ON -Dlibnd4j.sanitizers=thread,undefined,float-divide-by-zero,float-cast-overflow
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
cd ..
|
||||
mvn -Pcuda -Dlibnd4j.chip=cuda clean install -DskipTests -pl :libnd4j,:nd4j-cuda-12.1-preset,:nd4j-cuda-12.1
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#
|
||||
# /* ******************************************************************************
|
||||
# *
|
||||
# *
|
||||
# * This program and the accompanying materials are made available under the
|
||||
# * terms of the Apache License, Version 2.0 which is available at
|
||||
# * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
# *
|
||||
# * See the NOTICE file distributed with this work for additional
|
||||
# * information regarding copyright ownership.
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# * License for the specific language governing permissions and limitations
|
||||
# * under the License.
|
||||
# *
|
||||
# * SPDX-License-Identifier: Apache-2.0
|
||||
# ******************************************************************************/
|
||||
#
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
set -e
|
||||
|
||||
VALID_VERSIONS=( 9.2 10.0 10.1 10.2 11.0 11.1 11.2 11.4 11.6 11.8 12.1 12.3 12.6)
|
||||
|
||||
usage() {
|
||||
echo "Usage: $(basename $0) [-h|--help] <cuda version to be used>
|
||||
where :
|
||||
-h| --help Display this help text
|
||||
valid cuda version values : ${VALID_VERSIONS[*]}
|
||||
" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ ($# -ne 1) || ( $1 == "--help") || $1 == "-h" ]]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
VERSION=$1
|
||||
|
||||
check_cuda_version() {
|
||||
for i in ${VALID_VERSIONS[*]}; do [ $i = "$1" ] && return 0; done
|
||||
echo "Invalid CUDA version: $1. Valid versions: ${VALID_VERSIONS[*]}" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
check_cuda_version "$VERSION"
|
||||
|
||||
case $VERSION in
|
||||
12.6)
|
||||
VERSION2="9.5"
|
||||
VERSION3="1.5.11"
|
||||
;;
|
||||
12.3)
|
||||
VERSION2="8.9"
|
||||
VERSION3="1.5.10"
|
||||
;;
|
||||
12.1)
|
||||
VERSION2="8.9"
|
||||
VERSION3="1.5.9"
|
||||
;;
|
||||
11.8)
|
||||
VERSION2="8.6"
|
||||
VERSION3="1.5.8"
|
||||
;;
|
||||
11.6)
|
||||
VERSION2="8.3"
|
||||
VERSION3="1.5.7"
|
||||
;;
|
||||
11.4)
|
||||
VERSION2="8.2"
|
||||
VERSION3="1.5.6"
|
||||
;;
|
||||
11.2)
|
||||
VERSION2="8.1"
|
||||
VERSION3="1.5.5"
|
||||
;;
|
||||
11.1)
|
||||
VERSION2="8.0"
|
||||
VERSION3="1.5.5"
|
||||
;;
|
||||
11.0)
|
||||
VERSION2="8.0"
|
||||
VERSION3="1.5.4"
|
||||
;;
|
||||
10.2)
|
||||
VERSION2="8.2"
|
||||
VERSION3="1.5.6"
|
||||
;;
|
||||
10.1)
|
||||
VERSION2="7.6"
|
||||
VERSION3="1.5.2"
|
||||
;;
|
||||
10.0)
|
||||
VERSION2="7.4"
|
||||
VERSION3="1.5"
|
||||
;;
|
||||
9.2)
|
||||
VERSION2="7.2"
|
||||
VERSION3="1.5"
|
||||
;;
|
||||
esac
|
||||
|
||||
sed_i() {
|
||||
if test -f "$2" && test -f "$1"; then
|
||||
sed -i "" -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2"
|
||||
fi
|
||||
}
|
||||
|
||||
export -f sed_i
|
||||
|
||||
echo "Updating CUDA versions in pom.xml files to CUDA $1";
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
|
||||
SOURCE="$(readlink "$SOURCE")"
|
||||
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
|
||||
done
|
||||
BASEDIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
|
||||
|
||||
cd "${BASEDIR}/"contrib/version-updater
|
||||
mvn clean compile
|
||||
mvn exec:java -Dexec.args="--root-dir=${BASEDIR} --cuda-version=${VERSION} --cudnn-version=${VERSION2} --javacpp-version=${VERSION3} --update-type=cuda"
|
||||
|
||||
echo "Done updating CUDA versions.";
|
||||
@@ -0,0 +1,69 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>blas-lapack-generator</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
|
||||
<parent>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>codegen</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<name>blas-lapack-generator</name>
|
||||
|
||||
<properties>
|
||||
<javaparser.version>3.24.4</javaparser.version>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<javapoet.version>1.13.0</javapoet.version>
|
||||
<openblas.version>0.3.28</openblas.version>
|
||||
<javacpp.version>1.5.11</javacpp.version>
|
||||
<openblas.javacpp.version>${openblas.version}-${javacpp.version}</openblas.javacpp.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-cpu-backend-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.deeplearning4j</groupId>
|
||||
<artifactId>nd4j-native</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.reflections</groupId>
|
||||
<artifactId>reflections</artifactId>
|
||||
<version>0.10.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.bytedeco</groupId>
|
||||
<artifactId>openblas</artifactId>
|
||||
<version>${openblas.javacpp.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.squareup</groupId>
|
||||
<artifactId>javapoet</artifactId>
|
||||
<version>${javapoet.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.github.javaparser</groupId>
|
||||
<artifactId>javaparser-core-serialization</artifactId>
|
||||
<version>${javaparser.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.javaparser</groupId>
|
||||
<artifactId>javaparser-symbol-solver-core</artifactId>
|
||||
<version>${javaparser.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
package org.deeplearning4j;
|
||||
|
||||
import com.github.javaparser.ParserConfiguration;
|
||||
import com.github.javaparser.StaticJavaParser;
|
||||
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver;
|
||||
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
|
||||
import com.github.javaparser.utils.SourceRoot;
|
||||
import com.squareup.javapoet.*;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.bytedeco.javacpp.Pointer;
|
||||
import org.bytedeco.openblas.global.openblas;
|
||||
|
||||
import javax.lang.model.element.Modifier;
|
||||
import java.io.File;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public class BlasLapackGenerator {
|
||||
|
||||
private SourceRoot sourceRoot;
|
||||
private File rootDir;
|
||||
private File targetFile;
|
||||
|
||||
private static String copyright =
|
||||
"/*\n" +
|
||||
" * ******************************************************************************\n" +
|
||||
" * *\n" +
|
||||
" * *\n" +
|
||||
" * * This program and the accompanying materials are made available under the\n" +
|
||||
" * * terms of the Apache License, Version 2.0 which is available at\n" +
|
||||
" * * https://www.apache.org/licenses/LICENSE-2.0.\n" +
|
||||
" * *\n" +
|
||||
" * * See the NOTICE file distributed with this work for additional\n" +
|
||||
" * * information regarding copyright ownership.\n" +
|
||||
" * * Unless required by applicable law or agreed to in writing, software\n" +
|
||||
" * * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n" +
|
||||
" * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n" +
|
||||
" * * License for the specific language governing permissions and limitations\n" +
|
||||
" * * under the License.\n" +
|
||||
" * *\n" +
|
||||
" * * SPDX-License-Identifier: Apache-2.0\n" +
|
||||
" * *****************************************************************************\n" +
|
||||
" */\n";
|
||||
private static String codeGenWarning =
|
||||
"\n//================== GENERATED CODE - DO NOT MODIFY THIS FILE ==================\n\n";
|
||||
|
||||
|
||||
private static Set<String> methodsToSkip = new HashSet<>(Arrays.asList(
|
||||
"LAPACK_zlangb_base",
|
||||
"LAPACK_lsame_base",
|
||||
"LAPACK_clangb_base",
|
||||
"LAPACK_lsame_base",
|
||||
"LAPACK_dlangb_base",
|
||||
"LAPACK_slangb_base"
|
||||
|
||||
));
|
||||
|
||||
|
||||
public BlasLapackGenerator(File nd4jApiRootDir) {
|
||||
this.sourceRoot = initSourceRoot(nd4jApiRootDir);
|
||||
this.rootDir = nd4jApiRootDir;
|
||||
}
|
||||
|
||||
public SourceRoot getSourceRoot() {
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
public void setSourceRoot(SourceRoot sourceRoot) {
|
||||
this.sourceRoot = sourceRoot;
|
||||
}
|
||||
|
||||
public File getTargetFile() {
|
||||
return targetFile;
|
||||
}
|
||||
|
||||
public void setTargetFile(File targetFile) {
|
||||
this.targetFile = targetFile;
|
||||
}
|
||||
|
||||
public void parse() throws Exception {
|
||||
targetFile = new File(rootDir,"org/nd4j/linalg/api/blas/BLASLapackDelegator.java");
|
||||
String packageName = "org.nd4j.linalg.api.blas";
|
||||
TypeSpec.Builder openblasLapackDelegator = TypeSpec.interfaceBuilder("BLASLapackDelegator");
|
||||
openblasLapackDelegator.addModifiers(Modifier.PUBLIC);
|
||||
Class<openblas> clazz = openblas.class;
|
||||
List<Method> objectMethods = Arrays.asList(Object.class.getMethods());
|
||||
Arrays.stream(clazz.getMethods())
|
||||
.filter(input -> !objectMethods.contains(input))
|
||||
.filter(input -> !input.getName().equals("map") && !input.getName().equals("init"))
|
||||
.filter(input -> !methodsToSkip.contains(input.getName()))
|
||||
.forEach(method -> {
|
||||
MethodSpec.Builder builder = MethodSpec.methodBuilder(
|
||||
method.getName()
|
||||
).returns(method.getReturnType())
|
||||
.addModifiers(Modifier.DEFAULT,Modifier.PUBLIC);
|
||||
Arrays.stream(method.getParameters()).forEach(param -> {
|
||||
builder.addParameter(ParameterSpec.builder(
|
||||
!lapackType(param.getType()) ?
|
||||
TypeName.get(param.getType()) :
|
||||
TypeName.get(Pointer.class),
|
||||
param.getName()
|
||||
).build());
|
||||
});
|
||||
|
||||
openblasLapackDelegator.addMethod(builder.build());
|
||||
});
|
||||
|
||||
JavaFile finalFile = JavaFile.builder(packageName, openblasLapackDelegator.build())
|
||||
.addFileComment(copyright)
|
||||
.build();
|
||||
finalFile
|
||||
.writeTo(rootDir);
|
||||
}
|
||||
|
||||
private boolean lapackType(Class<?> clazz) {
|
||||
return clazz.equals(openblas.LAPACK_C_SELECT1.class) ||
|
||||
clazz.equals(openblas.LAPACK_C_SELECT2.class) ||
|
||||
clazz.equals(openblas.LAPACK_D_SELECT2.class) ||
|
||||
clazz.equals(openblas.LAPACK_S_SELECT2.class) ||
|
||||
clazz.equals(openblas.LAPACK_Z_SELECT1.class)
|
||||
|| clazz.equals(openblas.LAPACK_Z_SELECT2.class) ||
|
||||
clazz.equals(openblas.LAPACK_D_SELECT3.class) ||
|
||||
clazz.equals(openblas.LAPACK_S_SELECT3.class);
|
||||
}
|
||||
|
||||
|
||||
private SourceRoot initSourceRoot(File nd4jApiRootDir) {
|
||||
CombinedTypeSolver typeSolver = new CombinedTypeSolver();
|
||||
typeSolver.add(new ReflectionTypeSolver(false));
|
||||
typeSolver.add(new JavaParserTypeSolver(nd4jApiRootDir));
|
||||
JavaSymbolSolver symbolSolver = new JavaSymbolSolver(typeSolver);
|
||||
StaticJavaParser.getConfiguration().setSymbolResolver(symbolSolver);
|
||||
SourceRoot sourceRoot = new SourceRoot(nd4jApiRootDir.toPath(),new ParserConfiguration().setSymbolResolver(symbolSolver));
|
||||
return sourceRoot;
|
||||
}
|
||||
|
||||
|
||||
public static void main(String...args) throws Exception {
|
||||
BlasLapackGenerator blasLapackGenerator = new BlasLapackGenerator(new File("../../nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/"));
|
||||
blasLapackGenerator.parse();
|
||||
String generated = FileUtils.readFileToString(blasLapackGenerator.getTargetFile(), Charset.defaultCharset());
|
||||
generated = generated.replaceAll("\\{\\s+\\}",";");
|
||||
generated = generated.replace("default","");
|
||||
FileUtils.write(blasLapackGenerator.getTargetFile(),generated,Charset.defaultCharset());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user