Files
deeplearning4j--deeplearning4j/.github/workflows/build-deploy-android-x86_64.yml
T
2026-07-13 12:47:05 +08:00

529 lines
20 KiB
YAML

# GitHub Actions Workflow for android-x86_64
on:
workflow_dispatch:
inputs:
buildThreads:
description: 'Build threads for libnd4j. Used to control memory usage of builds.'
required: true
default: 2
deployToReleaseStaging:
description: 'Whether to deploy to release staging or not.'
required: false
default: 0
releaseVersion:
description: 'Release version target'
required: false
default: 1.0.0-M3
snapshotVersion:
description: 'Snapshot version target'
required: false
default: 1.0.0-SNAPSHOT
releaseRepoId:
description: 'Release repository id'
required: false
default: ""
serverId:
description: 'Server id to publish to'
required: false
default: central
mvnFlags:
description: "Extra maven flags (must escape input yourself if used)"
required: false
default: ""
libnd4jUrl:
description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.'
required: false
default: ""
runsOn:
description: 'System to run on'
required: false
default: ubuntu-latest
debug_enabled:
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: false
jobs:
android-x86_64:
strategy:
fail-fast: false
matrix:
helper: [onednn, ""]
include:
- mvn_ext: ${{ github.event.inputs.mvnFlags }}
experimental: true
name: Extra maven flags
- debug_enabled: ${{ github.event.inputs.debug_enabled }}
experimental: true
name: Debug enabled
- deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }}
experimental: true
name: Whether to deploy to release staging or not
- release_version: ${{ github.event.inputs.releaseVersion }}
experimental: true
name: Release version
- snapshot_version: ${{ github.event.inputs.snapshotVersion }}
experimental: true
name: Snapshot version
- server_id: ${{ github.event.inputs.serverId }}
experimental: true
name: Server id
- release_repo_id: ${{ github.event.inputs.releaseRepoId }}
experimental: true
name: The release repository to run on
- build_threads: ${{ github.event.inputs.buildThreads }}
experimental: true
name: The number of threads to build libnd4j with
runs-on: ${{ github.event.inputs.runsOn }}
steps:
- uses: AutoModality/action-clean@v1
- name: Cancel Previous Runs
uses: styfle/cancel-workflow-action@0.8.0
with:
access_token: ${{ github.token }}
- uses: actions/checkout@v4
- name: Free Disk Space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Configure swap space
shell: bash
run: |
echo "=== Initial system status ==="
free -h
df -h
echo "Available disk space on root:"
df -h /
# Check if swap is already enabled
if swapon --show | grep -q "/"; then
echo "Swap already configured:"
swapon --show
else
echo "Configuring swap space..."
# Use fixed 4GB swap size
SWAP_SIZE="12G"
echo "Creating ${SWAP_SIZE} swap file..."
# Create swap file
sudo fallocate -l ${SWAP_SIZE} /swapfile || {
echo "fallocate failed, trying dd..."
sudo dd if=/dev/zero of=/swapfile bs=1G count=12 status=progress
}
# Set up swap
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
# Verify swap is active
echo "=== Swap configuration completed ==="
free -h
swapon --show
fi
# Tune swappiness for build workloads
# Lower values (10-20) prefer RAM, higher values (60+) use swap more aggressively
echo "Current swappiness: $(cat /proc/sys/vm/swappiness)"
sudo sysctl vm.swappiness=80
echo "Adjusted swappiness: $(cat /proc/sys/vm/swappiness)"
sudo sysctl vm.overcommit_memory=2 # Allow overcommit
sudo sysctl vm.overcommit_ratio=80
# Show final memory status
echo "=== Final memory status ==="
free -h
- uses: ./.github/actions/set-linux-distro-version
- uses: ./.github/actions/update-deps-linux
- name: Cache cmake install
uses: actions/cache@v4
id: cache-cmake-install
with:
path: /opt/cmake
key: ${{ runner.os }}-cmake-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-cmake-
- name: Cache protobuf install
uses: actions/cache@v4
id: cache-protobuf
with:
path: /opt/protobuf
key: ${{ runner.os }}-protobuf-${{ github.run_id }}
restore-keys: |
${{ runner.os }}-protobuf-
- uses: ./.github/actions/install-protobuf-linux
if: steps.cache-protobuf.outputs.cache-hit != 'true'
- uses: ./.github/actions/install-cmake-linux
if: steps.cache-cmake-install.outputs.cache-hit != 'true'
- name: Set up Java for publishing to OSSRH
uses: actions/setup-java@v4
with:
java-version: 11
distribution: 'temurin'
server-id: ${{ github.event.inputs.serverId }}
server-username: MAVEN_USERNAME
server-password: MAVEN_PASSWORD
gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
cache: 'maven'
- uses: nttld/setup-ndk@v1
id: setup-ndk
with:
ndk-version: r26d
- name: Set mvn build command and environment variables
shell: bash
run: |
# Set constants for Android X86_64
LIBND4J_CLASSIFIER="android-x86_64"
# Define the specific OpenBLAS JAR for android-x86_64
OPENBLAS_VERSION="0.3.28-1.5.11"
OPENBLAS_JAR="openblas-${OPENBLAS_VERSION}-${LIBND4J_CLASSIFIER}.jar"
OPENBLAS_INSTALL_DIR="${GITHUB_WORKSPACE}/openblas_home"
NDK_VERSION_ENV="r26d"
echo "LIBND4J_CLASSIFIER=${LIBND4J_CLASSIFIER}" >> $GITHUB_ENV
echo "OPENBLAS_VERSION=${OPENBLAS_VERSION}" >> $GITHUB_ENV
echo "OPENBLAS_JAR=${OPENBLAS_JAR}" >> $GITHUB_ENV
echo "OPENBLAS_INSTALL_DIR=${OPENBLAS_INSTALL_DIR}" >> $GITHUB_ENV
echo "CURRENT_TARGET=android-x86_64" >> $GITHUB_ENV
echo "NDK_VERSION=${NDK_VERSION_ENV}" >> $GITHUB_ENV
echo "ANDROID_VERSION=21" >> $GITHUB_ENV
if [ "${{ github.event.inputs.libnd4jUrl }}" != '' ]; then
modules=':nd4j-native,:nd4j-native-preset'
else
echo "Building libnd4j from source"
modules=':nd4j-native,:nd4j-native-preset,:libnd4j'
fi
# Base Maven command
command="mvn ${{ matrix.mvn_ext }} -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=${LIBND4J_CLASSIFIER} -Pcpu --also-make --batch-mode deploy -DskipTests"
# Updated conditional flags for the helper library
if [ "${{ matrix.helper }}" != '' ]; then
if [ "${{ matrix.helper }}" == "onednn" ]; then
mvn_ext=" -Dlibnd4j.helper=onednn -Djavacpp.platform.extension=-onednn -Dlibnd4j.classifier=${LIBND4J_CLASSIFIER}-onednn"
else
echo "Warning: Unsupported helper '${{ matrix.helper }}' defined in matrix. Only 'onednn' or empty is handled for Maven flags."
mvn_ext=""
fi
else
mvn_ext=""
fi
command="${command} ${mvn_ext}"
echo "Setting command for helper ${{ matrix.helper }} to ${command}"
echo "COMMAND=${command}" >> $GITHUB_ENV
- name: Setup OpenBLAS Library
shell: bash
env:
OPENBLAS_INSTALL_DIR: ${{ env.OPENBLAS_INSTALL_DIR }}
OPENBLAS_JAR: ${{ env.OPENBLAS_JAR }}
OPENBLAS_VERSION: ${{ env.OPENBLAS_VERSION }}
LIBND4J_CLASSIFIER: ${{ env.LIBND4J_CLASSIFIER }}
run: |
echo "=== Setting up OpenBLAS ${OPENBLAS_VERSION} for ${LIBND4J_CLASSIFIER} ==="
# Create and navigate to install directory
mkdir -p "${OPENBLAS_INSTALL_DIR}"
cd "${OPENBLAS_INSTALL_DIR}"
# Download OpenBLAS JAR
OPENBLAS_URL="https://repo1.maven.org/maven2/org/bytedeco/openblas/${OPENBLAS_VERSION}/${OPENBLAS_JAR}"
echo "Downloading ${OPENBLAS_JAR} from ${OPENBLAS_URL}..."
wget --quiet ${OPENBLAS_URL} || {
echo "❌ Failed to download OpenBLAS JAR"
exit 1
}
# Extract JAR
echo "Extracting ${OPENBLAS_JAR}..."
unzip -q ${OPENBLAS_JAR} || {
echo "❌ Failed to extract OpenBLAS JAR"
exit 1
}
echo "=== OpenBLAS JAR extraction completed ==="
# The JAR contains: lib/x86_64/include/ and lib/x86_64/
OPENBLAS_PLATFORM_PATH="lib/x86_64"
if [[ -d "$OPENBLAS_PLATFORM_PATH" ]]; then
echo "✅ Found OpenBLAS platform directory: $OPENBLAS_PLATFORM_PATH"
# Verify the expected structure
if [[ -d "$OPENBLAS_PLATFORM_PATH/include" && -f "$OPENBLAS_PLATFORM_PATH/include/cblas.h" ]]; then
echo "✅ Found headers at: $OPENBLAS_PLATFORM_PATH/include/"
else
echo "❌ Headers not found at expected location: $OPENBLAS_PLATFORM_PATH/include/"
exit 1
fi
if [[ -d "$OPENBLAS_PLATFORM_PATH/lib" ]] || [[ -f "$OPENBLAS_PLATFORM_PATH/libopenblas.so" ]]; then
echo "✅ Found libraries in: $OPENBLAS_PLATFORM_PATH/"
else
echo "❌ Libraries not found in: $OPENBLAS_PLATFORM_PATH/"
exit 1
fi
# Set the correct path - this should be the full path to lib/x86_64
OPENBLAS_FINAL_PATH="$(realpath "$OPENBLAS_PLATFORM_PATH")"
else
echo "❌ Could not find OpenBLAS platform directory: $OPENBLAS_PLATFORM_PATH"
echo "Available directories:"
find . -type d -maxdepth 3 | head -20
exit 1
fi
# Set final environment variables - this is the key fix
echo "=== Setting final environment variables ==="
echo "OPENBLAS_PATH=${OPENBLAS_FINAL_PATH}" >> $GITHUB_ENV
echo "OPENBLAS_INCLUDE_PATH=${OPENBLAS_FINAL_PATH}/include" >> $GITHUB_ENV
echo "OPENBLAS_LIB_PATH=${OPENBLAS_FINAL_PATH}/" >> $GITHUB_ENV
echo "=== OpenBLAS setup completed ==="
echo "Final OpenBLAS path: ${OPENBLAS_FINAL_PATH}"
echo "Include path: ${OPENBLAS_FINAL_PATH}/include"
echo "Library path: ${OPENBLAS_FINAL_PATH}/"
# Verify the final setup
echo "=== Final verification ==="
ls -la "${OPENBLAS_FINAL_PATH}/include/" | head -10
ls -la "${OPENBLAS_FINAL_PATH}/" | grep -E "\.(so|a)$" | head -5
- name: OpenBLAS Configuration Verification
shell: bash
run: |
echo "=== OpenBLAS Configuration Verification ==="
echo "OPENBLAS_PATH: ${OPENBLAS_PATH}"
echo "OPENBLAS_INCLUDE_PATH: ${OPENBLAS_INCLUDE_PATH}"
echo "OPENBLAS_LIB_PATH: ${OPENBLAS_LIB_PATH}"
echo
# Test that paths are accessible
if [[ -d "$OPENBLAS_PATH" ]]; then
echo "✅ OPENBLAS_PATH is accessible"
else
echo "❌ OPENBLAS_PATH is not accessible: $OPENBLAS_PATH"
exit 1
fi
if [[ -d "$OPENBLAS_INCLUDE_PATH" ]]; then
echo "✅ OPENBLAS_INCLUDE_PATH is accessible"
else
echo "❌ OPENBLAS_INCLUDE_PATH is not accessible: $OPENBLAS_INCLUDE_PATH"
exit 1
fi
# Test header file accessibility
if [[ -f "$OPENBLAS_INCLUDE_PATH/cblas.h" ]]; then
echo "✅ cblas.h is accessible"
echo "First few lines of cblas.h:"
head -5 "$OPENBLAS_INCLUDE_PATH/cblas.h"
else
echo "❌ cblas.h is not accessible"
echo "Contents of include directory:"
ls -la "$OPENBLAS_INCLUDE_PATH"
exit 1
fi
if [[ -f "$OPENBLAS_INCLUDE_PATH/openblas_config.h" ]]; then
echo "✅ openblas_config.h is accessible"
else
echo "❌ openblas_config.h is not accessible"
exit 1
fi
echo "=== Verification completed successfully ==="
- name: Debug Info
shell: bash
run: |
echo "--- Workflow Inputs ---"
echo "buildThreads: ${{ github.event.inputs.buildThreads }}"
echo "deployToReleaseStaging: ${{ github.event.inputs.deployToReleaseStaging }}"
echo "releaseVersion: ${{ github.event.inputs.releaseVersion }}"
echo "snapshotVersion: ${{ github.event.inputs.snapshotVersion }}"
echo "releaseRepoId: ${{ github.event.inputs.releaseRepoId }}"
echo "serverId: ${{ github.event.inputs.serverId }}"
echo "mvnFlags: ${{ github.event.inputs.mvnFlags }}"
echo "libnd4jUrl: ${{ github.event.inputs.libnd4jUrl }}"
echo "runsOn: ${{ github.event.inputs.runsOn }}"
echo "debug_enabled: ${{ github.event.inputs.debug_enabled }}"
echo "--- Matrix Values ---"
echo "Helper: ${{ matrix.helper }}"
echo "--- Environment Variables ---"
echo "LIBND4J_CLASSIFIER: $LIBND4J_CLASSIFIER"
echo "OPENBLAS_VERSION: $OPENBLAS_VERSION"
echo "OPENBLAS_JAR: $OPENBLAS_JAR"
echo "OPENBLAS_INSTALL_DIR: $OPENBLAS_INSTALL_DIR"
echo "OPENBLAS_PATH: $OPENBLAS_PATH"
echo "OPENBLAS_INCLUDE_PATH: $OPENBLAS_INCLUDE_PATH"
echo "OPENBLAS_LIB_PATH: $OPENBLAS_LIB_PATH"
echo "CURRENT_TARGET: $CURRENT_TARGET"
echo "NDK_VERSION: $NDK_VERSION"
echo "ANDROID_VERSION: $ANDROID_VERSION"
echo "NDK Path (from step): ${{ steps.setup-ndk.outputs.ndk-path }}"
echo "COMMAND: $COMMAND"
echo "--- System Info ---"
export PATH=/opt/protobuf/bin:/opt/cmake/bin:$PATH
mvn --version
cmake --version
protoc --version
- name: Monitor memory during build
shell: bash
run: |
# Start background memory monitoring
(while true; do
echo "$(date '+%H:%M:%S'): $(free -h | head -n 2 | tail -n 1)"
if swapon --show | grep -q "/"; then
echo "$(date '+%H:%M:%S'): Swap: $(swapon --show --noheadings)"
fi
sleep 120 # Check every 2 minutes
done) > memory_monitor.log 2>&1 &
MONITOR_PID=$!
echo "MONITOR_PID=${MONITOR_PID}" >> $GITHUB_ENV
echo "Started memory monitoring (PID: ${MONITOR_PID})"
- name: Build with Maven
shell: bash
env:
MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }}
MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }}
PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }}
RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }}
SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }}
RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }}
HELPER: ${{ matrix.helper }}
LIBND4J_BUILD_THREADS: ${{ github.event.inputs.buildThreads }}
ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }}
OPENBLAS_PATH: ${{ env.OPENBLAS_PATH }}
ANDROID_VERSION: ${{ env.ANDROID_VERSION }}
NDK_VERSION: ${{ env.NDK_VERSION }}
PROTO_EXEC: /opt/protobuf/bin/protoc
LD_LIBRARY_PATH: $LD_LIBRARY_PATH:${{ env.OPENBLAS_LIB_PATH }}
DEBIAN_FRONTEND: noninteractive
DEPLOY: 1
BUILD_USING_MAVEN: 1
TARGET_OS: android
PUBLISH_TO: central
DEPLOY_TO: central
MAVEN_OPTS: -Xmx2g
RLIMIT_AS: 6442450944 # 6GB virtual memory limit
RLIMIT_DATA: 4294967296 # 4GB data segment limit
run: |
echo "=== Build Environment Summary ==="
echo "NDK Path: ${ANDROID_NDK}"
echo "OpenBLAS Path: ${OPENBLAS_PATH}"
echo "OpenBLAS Include Path: ${OPENBLAS_INCLUDE_PATH}"
echo "OpenBLAS Lib Path: ${OPENBLAS_LIB_PATH}"
echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
echo "Build Threads: ${LIBND4J_BUILD_THREADS}"
echo "Helper: ${HELPER}"
echo "================================="
# Show memory status before build
echo "=== Memory status before build ==="
free -h
# Pre-build verification
echo "=== Pre-build OpenBLAS verification ==="
if [[ -f "$OPENBLAS_PATH/include/cblas.h" ]]; then
echo "✅ cblas.h found and accessible"
else
echo "❌ cblas.h not found at $OPENBLAS_PATH/include/cblas.h"
echo "Attempting to locate cblas.h..."
find "$OPENBLAS_PATH" -name "cblas.h" 2>/dev/null | head -5
exit 1
fi
# Execute build
if [ "$PERFORM_RELEASE" == 1 ] || [ "$PERFORM_RELEASE" == "true" ]; then
echo "=== Executing release build ==="
bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}"
else
echo "=== Executing snapshot build ==="
echo "Command: ${COMMAND}"
eval "${COMMAND}"
fi
- name: Show memory usage summary
if: always()
shell: bash
run: |
# Stop memory monitoring
if [ -n "$MONITOR_PID" ]; then
kill $MONITOR_PID 2>/dev/null || true
fi
echo "=== Final memory status ==="
free -h
if swapon --show | grep -q "/"; then
echo "=== Swap usage ==="
swapon --show
fi
echo "=== Memory usage during build ==="
if [ -f memory_monitor.log ]; then
cat memory_monitor.log
else
echo "No memory monitoring log found"
fi
- name: Cleanup swap
if: always()
shell: bash
run: |
echo "Cleaning up swap configuration..."
sudo swapoff /swapfile 2>/dev/null || true
sudo rm -f /swapfile 2>/dev/null || true
echo "Swap cleanup completed"
- name: Upload build artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: build-logs-${{ matrix.helper }}-${{ github.run_id }}
path: |
memory_monitor.log
**/cmake_install.log
**/build.log
**/CMakeFiles/CMakeError.log
**/CMakeFiles/CMakeOutput.log
retention-days: 3