wehub-resource-sync
535 lines
19 KiB
YAML
535 lines
19 KiB
YAML
name: Python-package
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- master
|
|
|
|
# automatically cancel in-progress builds if another commit is pushed
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
# default to 0 permissions
|
|
# (job-level overrides add the minimal permissions needed)
|
|
permissions:
|
|
contents: none
|
|
|
|
env:
|
|
# tell scripts where to put artifacts
|
|
# (this variable name is left over from when jobs ran on Azure DevOps)
|
|
BUILD_ARTIFACTSTAGINGDIRECTORY: '${{ github.workspace }}/artifacts'
|
|
# where repo sources are cloned to
|
|
BUILD_SOURCESDIRECTORY: '${{ github.workspace }}'
|
|
CMAKE_BUILD_PARALLEL_LEVEL: 4
|
|
# avoid interactive prompts in Debian-based distributions
|
|
DEBIAN_FRONTEND: noninteractive
|
|
# On runners using nvidia-container-runtime with Docker, ensure GPUs are available to running processes.
|
|
#
|
|
# ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html
|
|
NVIDIA_VISIBLE_DEVICES: 'all'
|
|
SKBUILD_STRICT_CONFIG: true
|
|
|
|
jobs:
|
|
test-linux-aarch64:
|
|
name: bdist wheel (manylinux2014-arm, gcc, Python 3.14)
|
|
runs-on: ubuntu-24.04-arm
|
|
timeout-minutes: 60
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
fetch-depth: 5
|
|
persist-credentials: false
|
|
submodules: true
|
|
- name: Setup and run tests
|
|
shell: bash
|
|
# this uses 'docker run' instead of just setting 'container:'
|
|
# because actions/checkout requires GLIBC 2.28 and that is too
|
|
# new for manylinux2014
|
|
env:
|
|
BUILD_DIRECTORY: /LightGBM
|
|
run: |
|
|
cat > ./docker-script.sh <<EOF
|
|
mkdir -p \$BUILD_ARTIFACTSTAGINGDIRECTORY
|
|
export CONDA=\$HOME/miniforge
|
|
export PATH=\$CONDA/bin:\$PATH
|
|
${{ env.BUILD_DIRECTORY }}/.ci/setup.sh || exit 1
|
|
${{ env.BUILD_DIRECTORY }}/.ci/test.sh || exit 1
|
|
EOF
|
|
IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
|
|
docker pull "${IMAGE_URI}" || exit 1
|
|
docker run \
|
|
--platform "${PLATFORM}" \
|
|
--rm \
|
|
--env BUILD_DIRECTORY=${{ env.BUILD_DIRECTORY }} \
|
|
--env BUILD_ARTIFACTSTAGINGDIRECTORY=${{ env.BUILD_DIRECTORY }}/artifacts/ \
|
|
--env COMPILER=gcc \
|
|
--env METHOD=wheel \
|
|
--env OS_NAME=linux \
|
|
--env PRODUCES_ARTIFACTS=true \
|
|
--env PYTHON_VERSION="3.14" \
|
|
--env TASK=bdist \
|
|
-v "${PWD}":"${{ env.BUILD_DIRECTORY }}" \
|
|
-w ${{ env.BUILD_DIRECTORY }} \
|
|
"${IMAGE_URI}" \
|
|
/bin/bash ./docker-script.sh
|
|
- name: Upload artifacts
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: linux-aarch64-wheel
|
|
path: artifacts/*.whl
|
|
if-no-files-found: error
|
|
test:
|
|
# yamllint disable-line rule:line-length
|
|
name: ${{ matrix.task }} ${{ matrix.method }} (${{ matrix.os-display-name || matrix.os }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }})
|
|
runs-on: ${{ matrix.os }}
|
|
container: ${{ matrix.container }}
|
|
timeout-minutes: 60
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: macos-15-intel
|
|
task: sdist
|
|
compiler: clang
|
|
python_version: '3.10'
|
|
pixi-env: 'py310'
|
|
- os: windows-2022
|
|
task: sdist
|
|
compiler: MSVC
|
|
python_version: '3.10'
|
|
pixi-env: 'py310'
|
|
- os: ubuntu-latest
|
|
container: &ubuntu_latest_image ubuntu:22.04
|
|
os-display-name: &ubuntu_latest_display_name 'ubuntu22.04'
|
|
task: regular
|
|
compiler: clang-17
|
|
python_version: '3.13'
|
|
in-ubuntu-base-container: 'true'
|
|
- os: ubuntu-latest
|
|
container: *ubuntu_latest_image
|
|
os-display-name: *ubuntu_latest_display_name
|
|
task: sdist
|
|
compiler: clang-17
|
|
python_version: '3.14'
|
|
in-ubuntu-base-container: 'true'
|
|
#########################
|
|
# OpenCL-based GPU jobs #
|
|
#########################
|
|
# clang-17 jobs are run on machines with actual GPUs because
|
|
# the package built with that compiler toolchain can't target the
|
|
# CPUs on GitHub's runners.
|
|
#
|
|
# ref: https://github.com/lightgbm-org/LightGBM/pull/7096/files#r2590879590
|
|
# TODO(jameslamb): restore these GPU jobs when we've clarified handling of GPU CI
|
|
# ref: https://github.com/lightgbm-org/LightGBM/issues/7187
|
|
# - os: github-hosted-linux-gpu-amd64
|
|
# container: nvcr.io/nvidia/cuda:12.9.1-devel-ubuntu22.04
|
|
# os-display-name: ubuntu22.04-with-gpu
|
|
# task: bdist
|
|
# compiler: clang-17
|
|
# method: wheel
|
|
# python_version: '3.11'
|
|
# in-ubuntu-base-container: 'true'
|
|
# - os: github-hosted-linux-gpu-amd64
|
|
# container: nvcr.io/nvidia/cuda:12.9.1-devel-ubuntu22.04
|
|
# os-display-name: ubuntu22.04-with-gpu
|
|
# task: gpu
|
|
# compiler: clang-17
|
|
# method: source
|
|
# python_version: '3.12'
|
|
# in-ubuntu-base-container: 'true'
|
|
- os: ubuntu-latest
|
|
container: &manylinux_amd64_image ghcr.io/lightgbm-org/lightgbm-vsts-agent:manylinux_2_28_x86_64
|
|
os-display-name: &manylinux_amd64_display_name 'manylinux_2_28'
|
|
task: gpu
|
|
method: source
|
|
compiler: gcc
|
|
python_version: '3.14'
|
|
in-ubuntu-base-container: 'false'
|
|
setup-conda: 'false'
|
|
############
|
|
# MPI jobs #
|
|
############
|
|
- os: macos-15-intel
|
|
task: mpi
|
|
compiler: gcc
|
|
method: source
|
|
python_version: '3.12'
|
|
- os: ubuntu-latest
|
|
container: *manylinux_amd64_image
|
|
os-display-name: *manylinux_amd64_display_name
|
|
task: mpi
|
|
compiler: gcc
|
|
method: source
|
|
python_version: '3.11'
|
|
in-ubuntu-base-container: 'false'
|
|
setup-conda: 'false'
|
|
- os: ubuntu-latest
|
|
container: *ubuntu_latest_image
|
|
os-display-name: *ubuntu_latest_display_name
|
|
task: mpi
|
|
compiler: clang-17
|
|
method: source
|
|
python_version: '3.13'
|
|
in-ubuntu-base-container: 'true'
|
|
- os: ubuntu-latest
|
|
container: *ubuntu_latest_image
|
|
os-display-name: *ubuntu_latest_display_name
|
|
task: mpi
|
|
compiler: clang-17
|
|
method: pip
|
|
python_version: '3.12'
|
|
in-ubuntu-base-container: 'true'
|
|
- os: ubuntu-latest
|
|
container: *ubuntu_latest_image
|
|
os-display-name: *ubuntu_latest_display_name
|
|
task: mpi
|
|
compiler: clang-17
|
|
method: wheel
|
|
python_version: '3.10'
|
|
pixi-env: 'py310'
|
|
in-ubuntu-base-container: 'true'
|
|
#####################
|
|
# jobs that create #
|
|
# release artifacts #
|
|
#####################
|
|
- os: macos-15-intel
|
|
task: bdist
|
|
compiler: clang
|
|
method: wheel
|
|
produces-artifacts: 'true'
|
|
artifact-name: 'macosx-amd64-wheel'
|
|
python_version: '3.13'
|
|
- os: macos-14
|
|
task: bdist
|
|
compiler: clang
|
|
method: wheel
|
|
produces-artifacts: 'true'
|
|
artifact-name: 'macosx-arm64-wheel'
|
|
python_version: '3.11'
|
|
- os: macos-15-intel
|
|
task: regular
|
|
compiler: clang
|
|
produces-artifacts: 'true'
|
|
artifact-name: 'macosx-amd64-liblightgbm'
|
|
python_version: '3.11'
|
|
- os: ubuntu-latest
|
|
container: *manylinux_amd64_image
|
|
os-display-name: *manylinux_amd64_display_name
|
|
task: regular
|
|
compiler: gcc
|
|
python_version: '3.11'
|
|
in-ubuntu-base-container: 'false'
|
|
setup-conda: 'false'
|
|
produces-artifacts: 'true'
|
|
artifact-name: 'linux-amd64-liblightgbm'
|
|
- os: ubuntu-latest
|
|
container: *manylinux_amd64_image
|
|
os-display-name: *manylinux_amd64_display_name
|
|
task: bdist
|
|
compiler: gcc
|
|
method: wheel
|
|
python_version: '3.10'
|
|
pixi-env: 'py310'
|
|
in-ubuntu-base-container: 'false'
|
|
setup-conda: 'false'
|
|
produces-artifacts: 'true'
|
|
artifact-name: 'linux-amd64-wheel'
|
|
- os: ubuntu-latest
|
|
container: *manylinux_amd64_image
|
|
os-display-name: *manylinux_amd64_display_name
|
|
task: sdist
|
|
compiler: gcc
|
|
python_version: '3.10'
|
|
pixi-env: 'py310'
|
|
in-ubuntu-base-container: 'false'
|
|
setup-conda: 'false'
|
|
produces-artifacts: 'true'
|
|
artifact-name: 'sdist'
|
|
- os: windows-2022
|
|
task: bdist
|
|
compiler: MSVC
|
|
method: wheel
|
|
produces-artifacts: 'true'
|
|
artifact-name: 'windows-amd64-wheel'
|
|
python_version: '3.13'
|
|
- os: windows-2022
|
|
task: regular
|
|
compiler: MSVC
|
|
produces-artifacts: 'true'
|
|
artifact-name: 'windows-amd64-cli-and-liblightgbm'
|
|
python_version: '3.11'
|
|
steps:
|
|
- name: Install packages used by third-party actions
|
|
if: ${{ matrix.in-ubuntu-base-container == 'true' }}
|
|
shell: bash
|
|
run: |
|
|
apt-get update -y
|
|
apt-get install --no-install-recommends -y \
|
|
ca-certificates \
|
|
dirmngr \
|
|
gpg \
|
|
gpg-agent \
|
|
software-properties-common \
|
|
sudo
|
|
# install newest version of git
|
|
# ref:
|
|
# - https://unix.stackexchange.com/a/170831/550004
|
|
# - https://git-scm.com/download/linux
|
|
add-apt-repository ppa:git-core/ppa -y
|
|
apt-get update -y
|
|
apt-get install --no-install-recommends -y \
|
|
git
|
|
- name: Trust git cloning LightGBM
|
|
if: startsWith(matrix.os, 'ubuntu')
|
|
run: |
|
|
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
|
|
- name: Checkout repository
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
fetch-depth: 5
|
|
persist-credentials: false
|
|
submodules: true
|
|
- name: Set up 'pixi' for jobs that need it
|
|
if: matrix.pixi-env != ''
|
|
uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
|
|
with:
|
|
environments: ${{ matrix.pixi-env }}
|
|
- name: Setup and run tests on Linux and macOS
|
|
if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu') || endsWith(matrix.os, 'gpu-amd64')
|
|
shell: bash
|
|
run: |
|
|
export COMPILER="${{ matrix.compiler }}"
|
|
export IN_UBUNTU_BASE_CONTAINER="${{ matrix.in-ubuntu-base-container }}"
|
|
export SETUP_CONDA="${{ matrix.setup-conda }}"
|
|
export TASK="${{ matrix.task }}"
|
|
export METHOD="${{ matrix.method }}"
|
|
if [[ "${{ matrix.os }}" =~ ^macos ]]; then
|
|
export OS_NAME="macos"
|
|
elif \
|
|
[[ "${{ matrix.os }}" == "ubuntu-latest" ]] \
|
|
|| [[ "${{ matrix.os }}" == "github-hosted-linux-gpu-amd64" ]];
|
|
then
|
|
export OS_NAME="linux"
|
|
export PATH="/usr/lib64/openmpi/bin:${PATH}"
|
|
fi
|
|
export PYTHON_VERSION="${{ matrix.python_version }}"
|
|
export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
|
|
# some pre-built images already have 'CONDA' set
|
|
if [[ -z "${CONDA:-}" ]]; then
|
|
export CONDA=${HOME}/miniforge
|
|
fi
|
|
export PATH=${CONDA}/bin:${PATH}
|
|
export PRODUCES_ARTIFACTS="${{ matrix.produces-artifacts }}"
|
|
$GITHUB_WORKSPACE/.ci/setup.sh || exit 1
|
|
$GITHUB_WORKSPACE/.ci/test.sh || exit 1
|
|
- name: Install OpenCL on Windows
|
|
if: startsWith(matrix.os, 'windows') && (matrix.task == 'bdist')
|
|
shell: pwsh -command ". {0}"
|
|
run: |
|
|
& "$env:GITHUB_WORKSPACE/.ci/install-opencl.ps1"
|
|
# 'conda init powershell' needs to be run in a separate process
|
|
# ref: https://docs.conda.io/projects/conda/en/stable/dev-guide/deep-dives/activation.html
|
|
- name: Initialize conda on Windows
|
|
if: startsWith(matrix.os, 'windows')
|
|
shell: pwsh -command ". {0}"
|
|
run: |
|
|
$env:PATH = "$env:CONDA/Scripts;$env:PATH"
|
|
conda init powershell
|
|
- name: Setup and run tests on Windows
|
|
if: startsWith(matrix.os, 'windows')
|
|
shell: pwsh -command ". {0}"
|
|
run: |
|
|
$env:COMPILER = "${{ matrix.compiler }}"
|
|
$env:METHOD = "${{ matrix.method }}"
|
|
$env:PRODUCES_ARTIFACTS = "${{ matrix.produces-artifacts }}"
|
|
$env:PYTHON_VERSION = "${{ matrix.python_version }}"
|
|
$env:TASK = "${{ matrix.task }}"
|
|
& "$env:GITHUB_WORKSPACE/.ci/test-windows.ps1"
|
|
- name: Upload artifacts
|
|
if: ${{ matrix.produces-artifacts == 'true' }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: ${{ matrix.artifact-name }}
|
|
path: |
|
|
artifacts/*.dll
|
|
artifacts/*.dylib
|
|
artifacts/*.exe
|
|
artifacts/*.so
|
|
artifacts/*.tar.gz
|
|
artifacts/*.whl
|
|
if-no-files-found: error
|
|
create-nuget:
|
|
name: NuGet package
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
needs:
|
|
- test
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
fetch-depth: 5
|
|
persist-credentials: false
|
|
submodules: false
|
|
- name: Download lib_lightgbm (all operating systems) and Windows CLI
|
|
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
with:
|
|
merge-multiple: true
|
|
path: ${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}
|
|
pattern: '*amd64*-liblightgbm'
|
|
run-id: ${{ github.run_id }}
|
|
- name: Setup NuGet CLI
|
|
uses: NuGet/setup-nuget@fd55a6f3b34392fa83fde1454582407d8c714123 # v4.0
|
|
# ref: https://github.com/NuGet/setup-nuget/issues/168
|
|
- name: Setup mono
|
|
run: |
|
|
sudo apt-get update -y
|
|
sudo apt-get install --no-install-recommends -y \
|
|
mono-devel
|
|
- name: Create NuGet package
|
|
run: |
|
|
python .ci/create-nuget.py "${BUILD_ARTIFACTSTAGINGDIRECTORY}"
|
|
nuget pack \
|
|
$(pwd)/.ci/nuget/LightGBM.nuspec \
|
|
-NonInteractive \
|
|
-Verbosity detailed \
|
|
-OutputDirectory "${BUILD_ARTIFACTSTAGINGDIRECTORY}"
|
|
- name: Upload artifacts
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: nuget-package
|
|
path: artifacts/*.nupkg
|
|
if-no-files-found: error
|
|
|
|
test-latest-versions:
|
|
name: Python - latest versions (manylinux_2_28)
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
fetch-depth: 5
|
|
persist-credentials: false
|
|
submodules: true
|
|
- name: Create wheel
|
|
run: |
|
|
docker run \
|
|
--rm \
|
|
--env CMAKE_BUILD_PARALLEL_LEVEL=${{ env.CMAKE_BUILD_PARALLEL_LEVEL }} \
|
|
-v $(pwd):/opt/lgb-build \
|
|
-w /opt/lgb-build \
|
|
lightgbm/vsts-agent:manylinux_2_28_x86_64 \
|
|
/bin/bash -c 'PATH=/opt/miniforge/bin:$PATH sh ./build-python.sh bdist_wheel --nomp'
|
|
- name: Test compatibility
|
|
run: |
|
|
docker run \
|
|
--rm \
|
|
-v $(pwd):/opt/lgb-build \
|
|
-w /opt/lgb-build \
|
|
python:3.14 \
|
|
/bin/bash ./.ci/test-python-latest.sh
|
|
|
|
test-old-versions:
|
|
name: Python - oldest supported versions (manylinux_2_28, Python ${{ matrix.python_version }})
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# This should always include at least the oldest
|
|
# not-yet-end-of-life Python version
|
|
python_version:
|
|
- '3.10'
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
with:
|
|
fetch-depth: 5
|
|
persist-credentials: false
|
|
submodules: true
|
|
- name: Create wheel
|
|
run: |
|
|
docker run \
|
|
--rm \
|
|
--env CMAKE_BUILD_PARALLEL_LEVEL=${{ env.CMAKE_BUILD_PARALLEL_LEVEL }} \
|
|
-v $(pwd):/opt/lgb-build \
|
|
-w /opt/lgb-build \
|
|
lightgbm/vsts-agent:manylinux_2_28_x86_64 \
|
|
/bin/bash -c 'PATH=/opt/miniforge/bin:$PATH sh ./build-python.sh bdist_wheel --nomp'
|
|
- name: Test compatibility
|
|
run: |
|
|
docker run \
|
|
--rm \
|
|
-v $(pwd):/opt/lgb-build \
|
|
-w /opt/lgb-build \
|
|
python:${{ matrix.python_version }} \
|
|
/bin/bash ./.ci/test-python-oldest.sh
|
|
|
|
upload-nightlies:
|
|
needs:
|
|
- test
|
|
- test-linux-aarch64
|
|
runs-on: ubuntu-latest
|
|
# publish whenever a new commit is pushed to 'main'
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
environment:
|
|
name: nightly-python-uploads
|
|
url: https://anaconda.org/lightgbm-packages/lightgbm
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
steps:
|
|
- name: setup
|
|
run: |
|
|
mkdir -p "${BUILD_ARTIFACTSTAGINGDIRECTORY}"
|
|
pip install --prefer-binary \
|
|
anaconda-auth \
|
|
anaconda-client
|
|
- name: download wheels
|
|
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
with:
|
|
merge-multiple: true
|
|
path: ${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}
|
|
pattern: '*-wheel'
|
|
run-id: ${{ github.run_id }}
|
|
- name: download sdists
|
|
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
with:
|
|
name: sdist
|
|
path: ${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}
|
|
run-id: ${{ github.run_id }}
|
|
- name: upload to anaconda.org
|
|
env:
|
|
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN_LIGHTGBM_PACKAGES }}
|
|
run: |
|
|
anaconda upload \
|
|
--package lightgbm \
|
|
--force \
|
|
-t pypi \
|
|
${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbm-*.tar.gz \
|
|
${BUILD_ARTIFACTSTAGINGDIRECTORY}/*.whl
|
|
|
|
all-python-package-jobs-successful:
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
needs:
|
|
- test-latest-versions
|
|
- test
|
|
- test-linux-aarch64
|
|
- test-old-versions
|
|
permissions:
|
|
statuses: read
|
|
steps:
|
|
- name: Note that all tests succeeded
|
|
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
|
|
with:
|
|
jobs: ${{ toJSON(needs) }}
|