# Enhanced CUDA 12.6 Windows Build Workflow with Clean Toolchain Management name: CUDA 12.6 Windows Build on: workflow_dispatch: inputs: buildThreads: description: 'Build threads for libnd4j (controls memory usage)' required: true default: '4' deployToReleaseStaging: description: 'Deploy to release staging' required: false default: '0' releaseVersion: description: 'Release version target' required: false default: '1.0.0-M3' snapshotVersion: description: 'Snapshot version target' required: false default: '1.0.0-SNAPSHOT' releaseRepoId: description: 'Release repository id' required: false default: '' serverId: description: 'Server id to publish to' required: false default: 'central' mvnFlags: description: 'Extra maven flags' required: false default: '' libnd4jUrl: description: 'Download libnd4j from URL' required: false default: '' runsOn: description: 'System to run on' required: false default: 'windows-2022' debug_enabled: description: 'Enable tmate debugging' required: false default: 'false' jobs: windows-x86_64-cuda-12-6: strategy: fail-fast: false matrix: helper: [ 'cudnn', '' ] include: - runs_on: ${{ github.event.inputs.runsOn }} - build_threads: ${{ github.event.inputs.buildThreads }} runs-on: ${{ matrix.runs_on }} steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} - name: Checkout Repository uses: actions/checkout@v4 - name: Free Disk Space shell: powershell run: | Write-Host "=== Freeing Disk Space ===" Write-Host "Initial disk space:" Get-PSDrive C | Select-Object Used,Free # Clean Windows temp and cache directories $cleanupPaths = @( "$env:TEMP\*", "$env:SystemRoot\Temp\*", "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*", "$env:SystemRoot\SoftwareDistribution\Download\*" ) foreach ($path in $cleanupPaths) { try { Remove-Item -Path $path -Recurse -Force -ErrorAction SilentlyContinue Write-Host "Cleaned: $path" } catch { Write-Host "Could not clean: $path" } } # Clean package caches if (Get-Command choco -ErrorAction SilentlyContinue) { choco cache remove -y -ErrorAction SilentlyContinue } # Windows component cleanup try { dism.exe /online /Cleanup-Image /StartComponentCleanup /ResetBase } catch { Write-Host "DISM cleanup skipped" } Write-Host "Final disk space:" Get-PSDrive C | Select-Object Used,Free - name: Clean Install Visual Studio Build Tools shell: powershell run: | Write-Host "=== Visual Studio Build Tools Setup ===" # Function to completely remove VS installations function Remove-VSInstallation { param([string]$Path) if (Test-Path $Path) { Write-Host "Removing VS installation: $Path" Remove-Item -Path $Path -Recurse -Force -ErrorAction SilentlyContinue Start-Sleep -Seconds 2 } } # Stop VS processes Get-Process -Name "*vs_*", "vs_installer", "VSIXInstaller", "*devenv*", "*MSBuild*" -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue Start-Sleep -Seconds 5 # Remove all VS installations $vsPaths = @( "${env:ProgramFiles}\Microsoft Visual Studio", "${env:ProgramFiles(x86)}\Microsoft Visual Studio" ) foreach ($vsPath in $vsPaths) { if (Test-Path $vsPath) { Remove-VSInstallation -Path $vsPath } } Write-Host "=== Installing Fresh VS Build Tools 2022 (17.8 LTSC) ===" # Download VS Build Tools installer $installerUrl = "https://aka.ms/vs/17/release.ltsc.17.8/vs_buildtools.exe" $installerPath = "$env:TEMP\vs_buildtools.exe" try { Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing -TimeoutSec 600 Write-Host "Downloaded VS installer: $installerPath" } catch { Write-Error "Failed to download VS installer: $($_.Exception.Message)" exit 1 } # Install with Windows SDK 10.0.22621 (compatible with CUDA 12.6) $installArgs = @( "--quiet", "--wait", "--norestart", "--nocache", "--installPath", "`"C:\BuildTools`"", "--add", "Microsoft.VisualStudio.Workload.VCTools", "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "--add", "Microsoft.VisualStudio.Component.VC.CMake.Project", "--add", "Microsoft.VisualStudio.Component.Windows10SDK.22621", "--add", "Microsoft.VisualStudio.Component.VC.ATL", "--includeRecommended" ) Write-Host "Installing VS Build Tools..." $process = Start-Process -FilePath $installerPath -ArgumentList $installArgs -Wait -PassThru -NoNewWindow if ($process.ExitCode -ne 0) { Write-Error "VS Build Tools installation failed with exit code: $($process.ExitCode)" exit 1 } # Verify installation $vsPath = "C:\BuildTools" $vcvarsPath = "$vsPath\VC\Auxiliary\Build\vcvars64.bat" $msvcPath = "$vsPath\VC\Tools\MSVC" if (-not (Test-Path $vcvarsPath)) { Write-Error "VS Build Tools installation verification failed - vcvars64.bat not found" exit 1 } if (-not (Test-Path $msvcPath)) { Write-Error "VS Build Tools installation verification failed - MSVC tools not found" exit 1 } # Get MSVC version $msvcVersions = Get-ChildItem $msvcPath -Directory | Sort-Object Name -Descending if ($msvcVersions.Count -eq 0) { Write-Error "No MSVC versions found" exit 1 } $latestMSVC = $msvcVersions[0] Write-Host "SUCCESS: VS Build Tools installed successfully" Write-Host " Installation Path: $vsPath" Write-Host " MSVC Version: $($latestMSVC.Name)" # Export for later steps echo "VS_INSTALLATION_PATH=$vsPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append echo "MSVC_VERSION=$($latestMSVC.Name)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append echo "MSVC_ROOT=$($latestMSVC.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Install MSYS2 uses: msys2/setup-msys2@v2 with: msystem: MINGW64 update: true install: >- base-devel git tar pkg-config unzip p7zip zip mingw-w64-x86_64-cmake mingw-w64-x86_64-make mingw-w64-x86_64-toolchain mingw-w64-x86_64-gcc - name: Cache CUDA Installation uses: actions/cache@v4 id: cache-cuda with: path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6 key: cuda-12.6-update1-${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | cuda-12.6-update1-${{ runner.os }}- - name: Install CUDA 12.6 Update 1 if: steps.cache-cuda.outputs.cache-hit != 'true' shell: powershell run: | Write-Host "=== Installing CUDA 12.6 Update 1 ===" # Use network installer but with fixes for the -522190823 error $cudaUrl = "https://developer.download.nvidia.com/compute/cuda/12.6.3/network_installers/cuda_12.6.3_windows_network.exe" $cudaInstaller = "$env:TEMP\cuda_network_installer.exe" Write-Host "Downloading CUDA network installer..." try { $webClient = New-Object System.Net.WebClient $webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") $webClient.DownloadFile($cudaUrl, $cudaInstaller) $webClient.Dispose() if (-not (Test-Path $cudaInstaller) -or (Get-Item $cudaInstaller).Length -lt 10MB) { throw "Downloaded file is missing or too small" } Write-Host "Downloaded CUDA installer: $cudaInstaller ($(([math]::Round((Get-Item $cudaInstaller).Length / 1MB, 2))) MB)" } catch { Write-Error "Failed to download CUDA installer: $($_.Exception.Message)" exit 1 } # Create a temporary directory for CUDA installer logs and temp files $cudaTempDir = "$env:TEMP\cuda_install_temp" if (Test-Path $cudaTempDir) { Remove-Item -Path $cudaTempDir -Recurse -Force -ErrorAction SilentlyContinue } New-Item -Path $cudaTempDir -ItemType Directory -Force | Out-Null # Set environment variables to control installer behavior $env:CUDA_SETUP_ARGS = "--silent" $env:NVTOOLSEXT_INSTALL_DIR = "$cudaTempDir" # Essential CUDA components only for faster installation $cudaComponents = @( "nvcc_12.6", "cudart_12.6", "cublas_12.6", "cublas_dev_12.6", "cufft_12.6", "cufft_dev_12.6", "curand_12.6", "curand_dev_12.6", "cusolver_12.6", "cusolver_dev_12.6", "cusparse_12.6", "cusparse_dev_12.6", "thrust_12.6", "nvrtc_12.6", "nvrtc_dev_12.6" ) # Build installation arguments $installArgs = @("-s") + $cudaComponents Write-Host "Installing CUDA components: $($cudaComponents -join ', ')" Write-Host "Installation command: $cudaInstaller $($installArgs -join ' ')" try { # Set working directory to temp folder to avoid permission issues Set-Location $cudaTempDir # Run installation with timeout and proper error handling $installProcess = Start-Process -FilePath $cudaInstaller -ArgumentList $installArgs -Wait -PassThru -NoNewWindow -WorkingDirectory $cudaTempDir Write-Host "CUDA installer completed with exit code: $($installProcess.ExitCode)" # CUDA installer exit codes: # 0 = success # 1 = success with reboot required # 2 = already installed (newer version) # Other negative values = various errors if ($installProcess.ExitCode -notin @(0, 1, 2)) { throw "Installation failed with exit code: $($installProcess.ExitCode)" } } catch { Write-Error "CUDA installation failed: $($_.Exception.Message)" # Check for installation logs $logFiles = Get-ChildItem -Path $cudaTempDir -Filter "*.log" -ErrorAction SilentlyContinue if ($logFiles) { Write-Host "Installation logs found:" foreach ($logFile in $logFiles) { Write-Host "=== $($logFile.Name) ===" Get-Content $logFile.FullName | Select-Object -Last 20 } } exit 1 } Write-Host "SUCCESS: CUDA installation completed" # Verify installation $cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6" $nvccPath = "$cudaPath\bin\nvcc.exe" # Wait for filesystem to settle Start-Sleep -Seconds 10 if (-not (Test-Path $nvccPath)) { Write-Host "Primary CUDA path not found, checking for alternative locations..." # Check for any CUDA installation $cudaRoot = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA" if (Test-Path $cudaRoot) { Write-Host "Available CUDA installations:" $cudaVersions = Get-ChildItem $cudaRoot -Directory foreach ($version in $cudaVersions) { Write-Host " Found: $($version.Name)" $altNvccPath = "$($version.FullName)\bin\nvcc.exe" if (Test-Path $altNvccPath) { Write-Host " SUCCESS: nvcc.exe found in $($version.Name)" # Update our expected paths to the found version $cudaPath = $version.FullName $nvccPath = $altNvccPath break } } } # Final check if (-not (Test-Path $nvccPath)) { Write-Error "CUDA installation verification failed - nvcc.exe not found anywhere" exit 1 } } # Test nvcc functionality try { $nvccResult = & $nvccPath --version 2>&1 Write-Host "SUCCESS: CUDA installation verified - nvcc version:" Write-Host $nvccResult # Update environment variable if we found CUDA in a different location if ($cudaPath -ne "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6") { Write-Host "Updating CUDA_PATH to actual installation location: $cudaPath" echo "CUDA_ACTUAL_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append } } catch { Write-Error "nvcc found but functionality test failed: $($_.Exception.Message)" exit 1 } # Clean up installation files try { Set-Location $env:TEMP Remove-Item -Path $cudaTempDir -Recurse -Force -ErrorAction SilentlyContinue Remove-Item -Path $cudaInstaller -Force -ErrorAction SilentlyContinue Write-Host "Cleaned up installation files" } catch { Write-Host "Could not clean up some installation files (not critical)" } Write-Host "SUCCESS: CUDA 12.6 Update 1 network installation complete" - name: Setup Java uses: actions/setup-java@v4 with: java-version: 11 distribution: 'temurin' server-id: ${{ github.event.inputs.serverId }} server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE cache: 'maven' - name: Configure Build Environment shell: powershell run: | Write-Host "=== Configuring Build Environment ===" # Setup VS environment using clean vcvars64.bat $vsPath = $env:VS_INSTALLATION_PATH $vcvarsPath = "$vsPath\VC\Auxiliary\Build\vcvars64.bat" Write-Host "Initializing MSVC environment from: $vcvarsPath" # Create temporary batch file to capture environment $tempBatch = [System.IO.Path]::GetTempFileName() + ".bat" $tempEnv = [System.IO.Path]::GetTempFileName() + ".txt" @" @echo off call "$vcvarsPath" >nul 2>&1 if %ERRORLEVEL% NEQ 0 exit /b 1 set > "$tempEnv" "@ | Out-File -FilePath $tempBatch -Encoding ASCII $result = Start-Process -FilePath "cmd.exe" -ArgumentList "/c", "`"$tempBatch`"" -Wait -PassThru -NoNewWindow if ($result.ExitCode -ne 0 -or -not (Test-Path $tempEnv)) { Write-Error "Failed to initialize MSVC environment" exit 1 } # Parse and set environment variables $envCount = 0 Get-Content $tempEnv | ForEach-Object { if ($_ -match '^([^=]+)=(.*)$') { $name = $matches[1] $value = $matches[2] # Skip problematic variables if ($name -notmatch '^(TEMP|TMP|RANDOM|PROMPT|PATHEXT)$') { try { [Environment]::SetEnvironmentVariable($name, $value, 'Process') echo "$name=$value" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append $envCount++ } catch { # Silently continue on errors } } } } Write-Host "Set $envCount environment variables from vcvars64.bat" # Clean up Remove-Item $tempBatch, $tempEnv -ErrorAction SilentlyContinue # Setup CUDA environment $cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6" # Check if CUDA was installed in a different location if ($env:CUDA_ACTUAL_PATH) { $cudaPath = $env:CUDA_ACTUAL_PATH Write-Host "Using actual CUDA installation path: $cudaPath" } echo "CUDA_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append echo "CUDA_HOME=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append # Setup cuDNN only if needed if ("${{ matrix.helper }}" -eq "cudnn") { echo "CUDNN_ROOT_DIR=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append Write-Host "SUCCESS: cuDNN environment configured (CUDNN_ROOT_DIR=$cudaPath)" } else { Write-Host "INFO: Skipping cuDNN configuration (helper: ${{ matrix.helper }})" } # Setup clean PATH $cleanPath = @( "C:\msys64\mingw64\bin", "C:\msys64\usr\bin", "$cudaPath\bin", "$cudaPath\libnvvp", "$env:MSVC_ROOT\bin\Hostx64\x64", "$env:SystemRoot\system32", "$env:SystemRoot" ) -join ";" echo $cleanPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append Write-Host "SUCCESS: Build environment configured" Write-Host " VS Path: $vsPath" Write-Host " MSVC Version: $env:MSVC_VERSION" Write-Host " CUDA Path: $cudaPath" Write-Host " Helper: ${{ matrix.helper }}" - name: Verify Build Tools shell: cmd run: | echo "=== Verifying Build Tools ===" echo "Checking cl.exe..." where cl.exe if %ERRORLEVEL% NEQ 0 ( echo "ERROR: cl.exe not found in PATH" exit /b 1 ) cl.exe 2>&1 | findstr "Microsoft" echo "Checking nvcc.exe..." where nvcc.exe if %ERRORLEVEL% NEQ 0 ( echo "ERROR: nvcc.exe not found in PATH" exit /b 1 ) nvcc.exe --version | findstr "release" echo "Environment Check:" echo "CUDA_PATH: %CUDA_PATH%" echo "MSVC_ROOT: %MSVC_ROOT%" echo "Helper: ${{ matrix.helper }}" if "${{ matrix.helper }}"=="cudnn" ( echo "CUDNN_ROOT_DIR: %CUDNN_ROOT_DIR%" if not defined CUDNN_ROOT_DIR ( echo "WARNING: CUDNN_ROOT_DIR not set but cuDNN helper enabled" ) ) echo "SUCCESS: All build tools verified" - name: Configure Maven Command shell: powershell run: | Write-Host "=== Configuring Maven Command ===" # Base modules if ("${{ github.event.inputs.libnd4jUrl }}" -ne "") { $modules = ":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6" } elseif ("${{ matrix.helper }}" -ne "") { $modules = ":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6,libnd4j" } else { $modules = ":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6,libnd4j,:nd4j-cuda-12.6-platform" } # Base command $baseCommand = "mvn ${{ github.event.inputs.mvnFlags }} -Pcuda -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda --also-make -pl $modules" $baseCommand += " -Dlibnd4j.compute=`"8.6 9.0`" -Dlibnd4j.cpu.compile.skip=true" $baseCommand += " -Dlibnd4j.buildthreads=${{ matrix.build_threads }}" $baseCommand += " -Djavacpp.platform=windows-x86_64 -Dlibnd4j.platform=windows-x86_64" $baseCommand += " -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3" $baseCommand += " -Possrh deploy -DskipTests" # Helper-specific extensions if ("${{ matrix.helper }}" -ne "") { $helperExt = " -Dlibnd4j.classifier=windows-x86_64-cuda-12.6-${{ matrix.helper }}" $helperExt += " -Dlibnd4j.extension=${{ matrix.helper }}" $helperExt += " -Djavacpp.platform.extension=-${{ matrix.helper }}" $helperExt += " -Dlibnd4j.helper=${{ matrix.helper }}" $libnd4jFileName = "windows-cuda-12.6-${{ matrix.helper }}" } else { $helperExt = " -Dlibnd4j.classifier=windows-x86_64-cuda-12.6" $libnd4jFileName = "windows-cuda-12.6" } $fullCommand = $baseCommand + $helperExt # Save command echo $fullCommand | Out-File -FilePath "$env:GITHUB_WORKSPACE\mvn-command.bat" -Encoding utf8 echo "MAVEN_COMMAND=$fullCommand" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append if ("${{ github.event.inputs.libnd4jUrl }}" -ne "") { echo "LIBND4J_FILE_NAME=${{ github.event.inputs.libnd4jUrl }}/$libnd4jFileName" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append } Write-Host "SUCCESS: Maven command configured for helper: ${{ matrix.helper }}" Write-Host "Command: $fullCommand" - name: Run CUDA Build shell: cmd env: MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }} RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }} SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }} RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }} MAVEN_OPTS: "-Xmx2g" run: | echo "=== Starting CUDA Build ===" echo "Build threads: ${{ matrix.build_threads }}" echo "Helper: ${{ matrix.helper }}" echo "Deploy to release: %PERFORM_RELEASE%" echo "Release version: %RELEASE_VERSION%" echo "Snapshot version: %SNAPSHOT_VERSION%" rem Verify environment one more time echo "CUDA_PATH: %CUDA_PATH%" echo "MSVC_ROOT: %MSVC_ROOT%" if "${{ matrix.helper }}"=="cudnn" ( echo "CUDNN_ROOT_DIR: %CUDNN_ROOT_DIR%" ) rem Change CUDA versions bash ./change-cuda-versions.sh 12.6 if "%PERFORM_RELEASE%"=="1" ( echo "Running release build..." bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.6 "${{ matrix.helper }}" "" bash "./release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%" ) else ( echo "Running snapshot build..." if "%LIBND4J_FILE_NAME%" NEQ "" ( bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.6 "${{ matrix.helper }}" "" ) call "%GITHUB_WORKSPACE%\mvn-command.bat" ) echo "SUCCESS: Build completed" - name: Upload Build Artifacts if: always() uses: actions/upload-artifact@v4 with: name: build-logs-${{ matrix.helper || 'no-helper' }} path: | target/ libnd4j/blasbuild/ **/*.log retention-days: 7 - name: Setup tmate session for debugging if: ${{ github.event.inputs.debug_enabled == 'true' && failure() }} uses: mxschmitt/action-tmate@v3 timeout-minutes: 30