chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,40 @@
|
||||
version: 4.6.0.99.{build}
|
||||
|
||||
image: Visual Studio 2017
|
||||
platform: x64
|
||||
configuration:
|
||||
- '3.10'
|
||||
|
||||
# only build on 'master' / 'main' and pull requests targeting it
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- main
|
||||
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: MSVC
|
||||
TASK: python
|
||||
- COMPILER: MINGW
|
||||
TASK: python
|
||||
|
||||
clone_depth: 5
|
||||
|
||||
install:
|
||||
- git submodule update --init --recursive # get `external_libs` folder
|
||||
- set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
|
||||
- set PYTHON_VERSION=%CONFIGURATION%
|
||||
- powershell.exe -ExecutionPolicy ByPass -c "Invoke-RestMethod https://pixi.sh/install.ps1 | Invoke-Expression"
|
||||
- set PATH=C:\Users\appveyor\.pixi\bin;%PATH%
|
||||
- ps: |
|
||||
$env:APPVEYOR = "true"
|
||||
$env:CMAKE_BUILD_PARALLEL_LEVEL = 4
|
||||
$env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER"
|
||||
# tell scripts where to put artifacts
|
||||
# (this variable name is left over from when jobs ran on Azure DevOps)
|
||||
$env:BUILD_ARTIFACTSTAGINGDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER/artifacts"
|
||||
|
||||
build: false
|
||||
|
||||
test_script:
|
||||
- powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test-windows.ps1
|
||||
@@ -0,0 +1,6 @@
|
||||
Helper Scripts for CI
|
||||
=====================
|
||||
|
||||
This folder contains scripts which are run on CI services.
|
||||
|
||||
Dockerfile used on CI service is maintained in a separate [GitHub repository](https://github.com/guolinke/lightgbm-ci-docker) and can be pulled from [Docker Hub](https://hub.docker.com/r/lightgbm/vsts-agent).
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# [description]
|
||||
# Post a comment to a pull request.
|
||||
#
|
||||
# [usage]
|
||||
# append-comment.sh <PULL_REQUEST_ID> <BODY>
|
||||
#
|
||||
# PULL_REQUEST_ID: ID of PR to post the comment on.
|
||||
#
|
||||
# BODY: Text of the comment to be posted.
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
if [ -z "$GITHUB_ACTIONS" ]; then
|
||||
echo "Must be run inside GitHub Actions CI"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "Usage: $0 <PULL_REQUEST_ID> <BODY>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pr_id=$1
|
||||
body=$2
|
||||
|
||||
body=${body/failure/failure ❌}
|
||||
body=${body/error/failure ❌}
|
||||
body=${body/cancelled/failure ❌}
|
||||
body=${body/timed_out/failure ❌}
|
||||
body=${body/success/success ✔️}
|
||||
data=$(
|
||||
jq -n \
|
||||
--argjson body "\"$body\"" \
|
||||
'{"body": $body}'
|
||||
)
|
||||
curl -sL \
|
||||
--fail \
|
||||
-X POST \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||
-d "$data" \
|
||||
"${GITHUB_API_URL}/repos/lightgbm-org/LightGBM/issues/${pr_id}/comments"
|
||||
@@ -0,0 +1,19 @@
|
||||
loadNamespace("pkgdown")
|
||||
loadNamespace("roxygen2")
|
||||
|
||||
roxygen2::roxygenize(
|
||||
"R-package/"
|
||||
, load = "installed"
|
||||
)
|
||||
|
||||
pkgdown::build_site(
|
||||
"R-package/"
|
||||
, lazy = FALSE
|
||||
, install = FALSE
|
||||
, devel = FALSE
|
||||
, examples = TRUE
|
||||
, run_dont_run = TRUE
|
||||
, seed = 42L
|
||||
, preview = FALSE
|
||||
, new_process = TRUE
|
||||
)
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
conda env create \
|
||||
--name test-env \
|
||||
--file ./docs/env.yml \
|
||||
|| exit 1
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source activate test-env
|
||||
|
||||
make -C docs html || exit 1
|
||||
Executable
+129
@@ -0,0 +1,129 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# [description]
|
||||
# Helper script for checking versions in the dynamic symbol table.
|
||||
# This script checks that LightGBM library is linked to the appropriate symbol versions.
|
||||
# Linking to newer symbol versions at compile time is problematic because it could result
|
||||
# in built artifacts being unusable on older platforms.
|
||||
#
|
||||
# Version history for these symbols can be found at the following:
|
||||
# * GLIBC: https://sourceware.org/glibc/wiki/Glibc%20Timeline
|
||||
# * GLIBCXX: https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html
|
||||
# * OMP/GOMP: https://github.com/gcc-mirror/gcc/blob/master/libgomp/libgomp.map
|
||||
#
|
||||
# [usage]
|
||||
# check-dynamic-dependencies.sh <PATH>
|
||||
#
|
||||
# PATH: Path to the file.
|
||||
# Path to the file with the dynamic symbol table entries of the file
|
||||
# (result of `objdump -T` command).
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "Usage: $0 <PATH>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INPUT_FILE="$1"
|
||||
|
||||
if [ ! -f "$INPUT_FILE" ]; then
|
||||
echo "Error: File '$INPUT_FILE' not found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
awk '
|
||||
BEGIN {
|
||||
glibc_count = 0
|
||||
glibcxx_count = 0
|
||||
gomp_count = 0
|
||||
|
||||
has_error = 0
|
||||
}
|
||||
|
||||
# --- Check GLIBC ---
|
||||
/0{16}[ \t\(]+GLIBC_[0-9]+\.[0-9]+/ {
|
||||
match($0, /GLIBC_([0-9]+)\.([0-9]+)/, parts)
|
||||
if (RSTART > 0) {
|
||||
match($0, /GLIBC_[0-9]+\.[0-9]+/)
|
||||
ver_str = substr($0, RSTART+6, RLENGTH-6) # skip "GLIBC_"
|
||||
split(ver_str, v, ".")
|
||||
major = v[1] + 0
|
||||
minor = v[2] + 0
|
||||
|
||||
glibc_count++
|
||||
|
||||
if (major > 2 || (major == 2 && minor > 28)) {
|
||||
print "Error: found unexpected GLIBC version: \x27" major "." minor "\x27"
|
||||
has_error = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Check GLIBCXX ---
|
||||
/0{16}[ \t\(]+GLIBCXX_[0-9]+\.[0-9]+/ {
|
||||
match($0, /GLIBCXX_[0-9]+\.[0-9]+(\.[0-9]+)?/)
|
||||
if (RSTART > 0) {
|
||||
ver_str = substr($0, RSTART+8, RLENGTH-8) # skip "GLIBCXX_"
|
||||
n = split(ver_str, v, ".")
|
||||
major = v[1] + 0
|
||||
minor = v[2] + 0
|
||||
patch = (n >= 3) ? v[3] + 0 : 0
|
||||
patch_str = (n >= 3) ? v[3] : ""
|
||||
|
||||
glibcxx_count++
|
||||
|
||||
msg_ver = major "." minor
|
||||
if (n >= 3) msg_ver = msg_ver "." patch
|
||||
|
||||
if (major != 3 || minor != 4) {
|
||||
print "Error: found unexpected GLIBCXX version: \x27" msg_ver "\x27"
|
||||
has_error = 1
|
||||
}
|
||||
if (n >= 3 && patch > 22) {
|
||||
print "Error: found unexpected GLIBCXX version: \x27" msg_ver "\x27"
|
||||
has_error = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Check OMP/GOMP ---
|
||||
/0{16}[ \t\(]+G?OMP_[0-9]+\.[0-9]+/ {
|
||||
match($0, /G?OMP_[0-9]+\.[0-9]+/)
|
||||
if (RSTART > 0) {
|
||||
full_match = substr($0, RSTART, RLENGTH)
|
||||
us_idx = index(full_match, "_")
|
||||
ver_str = substr(full_match, us_idx + 1)
|
||||
|
||||
split(ver_str, v, ".")
|
||||
major = v[1] + 0
|
||||
minor = v[2] + 0
|
||||
|
||||
gomp_count++
|
||||
|
||||
if (major > 4 || (major == 4 && minor > 5)) {
|
||||
print "Error: found unexpected OMP/GOMP version: \x27" major "." minor "\x27"
|
||||
has_error = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
END {
|
||||
if (glibc_count <= 1) {
|
||||
print "Error: Not enough GLIBC symbols found (found " glibc_count ", expected > 1)"
|
||||
has_error = 1
|
||||
}
|
||||
if (glibcxx_count <= 1) {
|
||||
print "Error: Not enough GLIBCXX symbols found (found " glibcxx_count ", expected > 1)"
|
||||
has_error = 1
|
||||
}
|
||||
if (gomp_count <= 1) {
|
||||
print "Error: Not enough OMP/GOMP symbols found (found " gomp_count ", expected > 1)"
|
||||
has_error = 1
|
||||
}
|
||||
|
||||
if (has_error == 1) {
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
' "$INPUT_FILE"
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -u
|
||||
|
||||
echo "checking that all OpenMP pragmas specify num_threads()"
|
||||
get_omp_pragmas_without_num_threads() {
|
||||
grep \
|
||||
-n \
|
||||
-R \
|
||||
--include='*.c' \
|
||||
--include='*.cc' \
|
||||
--include='*.cpp' \
|
||||
--include='*.h' \
|
||||
--include='*.hpp' \
|
||||
'pragma omp parallel' \
|
||||
| grep -v ' num_threads'
|
||||
}
|
||||
|
||||
# 'grep' returns a non-0 exit code if 0 lines were found.
|
||||
# Turning off '-e -o pipefail' options here so that bash doesn't
|
||||
# consider this a failure and stop execution of the script.
|
||||
#
|
||||
# ref: https://www.gnu.org/software/grep/manual/html_node/Exit-Status.html
|
||||
set +e
|
||||
PROBLEMATIC_LINES=$(
|
||||
get_omp_pragmas_without_num_threads
|
||||
)
|
||||
set -e
|
||||
if test "${PROBLEMATIC_LINES}" != ""; then
|
||||
get_omp_pragmas_without_num_threads
|
||||
echo "Found '#pragma omp parallel' not using explicit num_threads() configuration. Fix those."
|
||||
echo "For details, see https://www.openmp.org/spec-html/5.0/openmpse14.html#x54-800002.6"
|
||||
exit 1
|
||||
fi
|
||||
echo "done checking OpenMP pragmas"
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e -u
|
||||
|
||||
DIST_DIR=${1}
|
||||
|
||||
# defaults
|
||||
METHOD=${METHOD:-""}
|
||||
TASK=${TASK:-""}
|
||||
|
||||
echo "checking Python-package distributions in '${DIST_DIR}'"
|
||||
|
||||
pip install \
|
||||
-qq \
|
||||
check-wheel-contents \
|
||||
twine || exit 1
|
||||
|
||||
echo "twine check..."
|
||||
twine check --strict "$(echo "${DIST_DIR}"/*)" || exit 1
|
||||
|
||||
if { test "${TASK}" = "bdist" || test "${METHOD}" = "wheel"; }; then
|
||||
echo "check-wheel-contents..."
|
||||
check-wheel-contents "$(echo "${DIST_DIR}"/*.whl)" || exit 1
|
||||
fi
|
||||
|
||||
PY_MINOR_VER=$(python -c "import sys; print(sys.version_info.minor)")
|
||||
if [ "$PY_MINOR_VER" -gt 7 ]; then
|
||||
echo "pydistcheck..."
|
||||
pip install 'pydistcheck>=0.9.1'
|
||||
if { test "${TASK}" = "cuda" || test "${METHOD}" = "wheel"; }; then
|
||||
pydistcheck \
|
||||
--inspect \
|
||||
--ignore 'compiled-objects-have-debug-symbols'\
|
||||
--ignore 'distro-too-large-compressed' \
|
||||
--max-allowed-size-uncompressed '550M' \
|
||||
--max-allowed-files 806 \
|
||||
"$(echo "${DIST_DIR}"/*)" || exit 1
|
||||
elif { test "$(uname -m)" = "aarch64"; }; then
|
||||
pydistcheck \
|
||||
--inspect \
|
||||
--ignore 'compiled-objects-have-debug-symbols' \
|
||||
--max-allowed-size-compressed '5M' \
|
||||
--max-allowed-size-uncompressed '15M' \
|
||||
--max-allowed-files 806 \
|
||||
"$(echo "${DIST_DIR}"/*)" || exit 1
|
||||
else
|
||||
pydistcheck \
|
||||
--inspect \
|
||||
--max-allowed-size-compressed '5M' \
|
||||
--max-allowed-size-uncompressed '15M' \
|
||||
--max-allowed-files 806 \
|
||||
"$(echo "${DIST_DIR}"/*)" || exit 1
|
||||
fi
|
||||
else
|
||||
echo "skipping pydistcheck (does not support Python 3.${PY_MINOR_VER})"
|
||||
fi
|
||||
|
||||
echo "done checking Python-package distributions"
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
|
||||
# [description]
|
||||
#
|
||||
# Look for the last run of a given GitHub Actions workflow on a given branch.
|
||||
# If there's never been one (as might be the case with optional workflows like valgrind),
|
||||
# exit with 0.
|
||||
#
|
||||
# Otherwise, check the status of that latest run.
|
||||
# If it wasn't successful, exit with a non-0 exit code.
|
||||
#
|
||||
# [usage]
|
||||
#
|
||||
# check-workflow-status.sh <BRANCH> <WORKFLOW_FILE>
|
||||
#
|
||||
# BRANCH: name of a branch involved in a pull request.
|
||||
#
|
||||
# WORKFLOW_FILE: filename (e.g. 'r_valgrind.yml') defining the GitHub Actions workflow.
|
||||
#
|
||||
|
||||
set -e -u -o pipefail
|
||||
|
||||
BRANCH="${1}"
|
||||
WORKFLOW_FILE="${2}"
|
||||
PR_NUMBER="${3}"
|
||||
|
||||
# Limit how much data is pulled from the API and needs to be parsed locally.
|
||||
OLDEST_ALLOWED_RUN_DATE=$(date --date='7 days ago' '+%F')
|
||||
|
||||
echo "Searching for latest run of '${WORKFLOW_FILE}' on branch '${BRANCH}' "
|
||||
|
||||
LATEST_RUN_ID=$(
|
||||
gh run list \
|
||||
--repo 'lightgbm-org/LightGBM' \
|
||||
--event 'workflow_dispatch' \
|
||||
--created ">= ${OLDEST_ALLOWED_RUN_DATE}" \
|
||||
--workflow "${WORKFLOW_FILE}" \
|
||||
--json 'createdAt,databaseId,name' \
|
||||
--jq "sort_by(.createdAt) | reverse | map(select(.name | contains (\"pr=${PR_NUMBER}\"))) | .[0] | .databaseId"
|
||||
)
|
||||
|
||||
if [[ "${LATEST_RUN_ID}" == "" ]]; then
|
||||
echo "No runs of '${WORKFLOW_FILE}' found on branch from pull request ${PR_NUMBER} (on or after ${OLDEST_ALLOWED_RUN_DATE})."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Checking status of workflow run '${LATEST_RUN_ID}'"
|
||||
gh run view \
|
||||
--repo "lightgbm-org/LightGBM" \
|
||||
--exit-status \
|
||||
"${LATEST_RUN_ID}"
|
||||
@@ -0,0 +1,11 @@
|
||||
# conda-envs
|
||||
|
||||
This directory contains files used to create `conda` environments for development
|
||||
and testing of LightGBM.
|
||||
|
||||
The `.txt` files here are intended to be used with `conda create --file`.
|
||||
|
||||
For details on that, see the `conda` docs:
|
||||
|
||||
* `conda create` docs ([link](https://conda.io/projects/conda/en/latest/commands/create.html))
|
||||
* "Managing Environments" ([link](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html))
|
||||
@@ -0,0 +1,45 @@
|
||||
# [description]
|
||||
#
|
||||
# Core dependencies used across most LightGBM continuous integration (CI) jobs.
|
||||
#
|
||||
# 'python' constraint is intentionally omitted, so this file can be reused across
|
||||
# Python versions.
|
||||
#
|
||||
# These floors are not the oldest versions LightGBM supports... they're here just to make conda
|
||||
# solves faster, and should generally be the latest versions that work for all CI jobs using this.
|
||||
#
|
||||
# [usage]
|
||||
#
|
||||
# conda create \
|
||||
# --name test-env \
|
||||
# --file ./.ci/conda-envs/ci-core.txt \
|
||||
# python=3.10
|
||||
#
|
||||
|
||||
# direct imports
|
||||
cffi>=1.16
|
||||
dask>=2025.4.0
|
||||
joblib>=1.3.2
|
||||
matplotlib-base>=3.7.3
|
||||
narwhals>=1.15
|
||||
numpy>=1.24.4
|
||||
pandas>2.0
|
||||
polars>=1.0.0
|
||||
pyarrow-core>=16.0
|
||||
python-graphviz>=0.20.3
|
||||
scikit-learn>=1.3.2
|
||||
scipy>=1.7
|
||||
|
||||
# starting some time around 2026, 'pip' is no longer guaranteed to be installed
|
||||
pip>=24
|
||||
|
||||
# testing-only dependencies
|
||||
cloudpickle>=3.0.0
|
||||
psutil>=5.9.8
|
||||
pytest>=8.1.1
|
||||
|
||||
# other recursive dependencies, just
|
||||
# pinned here to help speed up solves
|
||||
pluggy>=1.4.0
|
||||
setuptools>=69.2
|
||||
wheel>=0.43
|
||||
@@ -0,0 +1,85 @@
|
||||
# coding: utf-8
|
||||
"""Script for generating files with NuGet package metadata."""
|
||||
|
||||
import datetime
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from shutil import copyfile
|
||||
|
||||
if __name__ == "__main__":
|
||||
source = Path(sys.argv[1])
|
||||
nuget_dir = Path(__file__).absolute().parent / "nuget"
|
||||
print(f"Creating nuget directory '{nuget_dir}'")
|
||||
linux_folder_path = nuget_dir / "runtimes" / "linux-x64" / "native"
|
||||
linux_folder_path.mkdir(parents=True, exist_ok=True)
|
||||
osx_folder_path = nuget_dir / "runtimes" / "osx-x64" / "native"
|
||||
osx_folder_path.mkdir(parents=True, exist_ok=True)
|
||||
windows_folder_path = nuget_dir / "runtimes" / "win-x64" / "native"
|
||||
windows_folder_path.mkdir(parents=True, exist_ok=True)
|
||||
build_folder_path = nuget_dir / "build"
|
||||
build_folder_path.mkdir(parents=True, exist_ok=True)
|
||||
print(f"Looking for libraries in '{source}'")
|
||||
copyfile(source / "lib_lightgbm.so", linux_folder_path / "lib_lightgbm.so")
|
||||
copyfile(source / "lib_lightgbm.dylib", osx_folder_path / "lib_lightgbm.dylib")
|
||||
copyfile(source / "lib_lightgbm.dll", windows_folder_path / "lib_lightgbm.dll")
|
||||
copyfile(source / "lightgbm.exe", windows_folder_path / "lightgbm.exe")
|
||||
version = (nuget_dir.parents[1] / "VERSION.txt").read_text(encoding="utf-8").strip().replace("rc", "-rc")
|
||||
print(f"Setting version to '{version}'")
|
||||
nuget_str = rf"""<?xml version="1.0"?>
|
||||
<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
|
||||
<metadata>
|
||||
<id>LightGBM</id>
|
||||
<version>{version}</version>
|
||||
<authors>Guolin Ke</authors>
|
||||
<owners>Guolin Ke</owners>
|
||||
<license type="expression">MIT</license>
|
||||
<projectUrl>https://github.com/lightgbm-org/LightGBM</projectUrl>
|
||||
<requireLicenseAcceptance>false</requireLicenseAcceptance>
|
||||
<description>A fast, distributed, high performance gradient boosting framework</description>
|
||||
<copyright>Copyright {datetime.datetime.now().year} @ Microsoft</copyright>
|
||||
<tags>machine-learning data-mining distributed native boosting gbdt</tags>
|
||||
<dependencies> </dependencies>
|
||||
</metadata>
|
||||
<files>
|
||||
<file src="build\**" target="build"/>
|
||||
<file src="runtimes\**" target="runtimes"/>
|
||||
</files>
|
||||
</package>
|
||||
"""
|
||||
prop_str = r"""
|
||||
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Condition="Exists('packages.config') OR
|
||||
Exists('$(MSBuildProjectName).packages.config') OR
|
||||
Exists('packages.$(MSBuildProjectName).config')">
|
||||
<Content Include="$(MSBuildThisFileDirectory)/../runtimes/win-x64/native/*.dll"
|
||||
Condition="'$(PlatformTarget)' == 'x64'">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<Visible>false</Visible>
|
||||
</Content>
|
||||
<Content Include="$(MSBuildThisFileDirectory)/../runtimes/win-x64/native/*.exe"
|
||||
Condition="'$(PlatformTarget)' == 'x64'">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<Visible>false</Visible>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
"""
|
||||
target_str = r"""
|
||||
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<EnableLightGBMUnsupportedPlatformTargetCheck Condition="'$(EnableLightGBMUnsupportedPlatformTargetCheck)' == ''">true</EnableLightGBMUnsupportedPlatformTargetCheck>
|
||||
</PropertyGroup>
|
||||
<Target Name="_LightGBMCheckForUnsupportedPlatformTarget"
|
||||
Condition="'$(EnableLightGBMUnsupportedPlatformTargetCheck)' == 'true'"
|
||||
AfterTargets="_CheckForInvalidConfigurationAndPlatform">
|
||||
<Error Condition="'$(PlatformTarget)' != 'x64' AND
|
||||
('$(OutputType)' == 'Exe' OR '$(OutputType)'=='WinExe') AND
|
||||
!('$(TargetFrameworkIdentifier)' == '.NETCoreApp' AND '$(PlatformTarget)' == '')"
|
||||
Text="LightGBM currently supports 'x64' processor architectures. Please ensure your application is targeting 'x64'." />
|
||||
</Target>
|
||||
</Project>
|
||||
"""
|
||||
(nuget_dir / "LightGBM.nuspec").write_text(nuget_str, encoding="utf-8")
|
||||
(nuget_dir / "build" / "LightGBM.props").write_text(prop_str, encoding="utf-8")
|
||||
(nuget_dir / "build" / "LightGBM.targets").write_text(target_str, encoding="utf-8")
|
||||
print("Done creating NuGet package")
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
|
||||
# [description]
|
||||
# Collect and download artifacts from all workflow runs for a commit.
|
||||
#
|
||||
# [usage]
|
||||
# ./download-artifacts.sh <COMMIT_ID>
|
||||
#
|
||||
|
||||
set -e -u -E -o pipefail
|
||||
|
||||
COMMIT_ID="${1}"
|
||||
OUTPUT_DIR="./release-artifacts"
|
||||
|
||||
get-latest-run-id() {
|
||||
gh run list \
|
||||
--repo "lightgbm-org/LightGBM" \
|
||||
--commit "${1}" \
|
||||
--workflow "${2}" \
|
||||
--json 'createdAt,databaseId' \
|
||||
--jq 'sort_by(.createdAt) | reverse | .[0] | .databaseId'
|
||||
}
|
||||
|
||||
# ensure directory for storing artifacts exists
|
||||
echo "preparing to download artifacts for commit '${COMMIT_ID}' to '${OUTPUT_DIR}'"
|
||||
mkdir -p "${OUTPUT_DIR}"
|
||||
|
||||
# get core artifacts
|
||||
echo "downloading core artifacts"
|
||||
gh run download \
|
||||
--repo "lightgbm-org/LightGBM" \
|
||||
--dir "${OUTPUT_DIR}" \
|
||||
"$(get-latest-run-id "${COMMIT_ID}" 'build.yml')"
|
||||
echo "done downloading core artifacts"
|
||||
|
||||
# get python-package artifacts
|
||||
echo "downloading python-package artifacts and NuGet package"
|
||||
gh run download \
|
||||
--repo "lightgbm-org/LightGBM" \
|
||||
--dir "${OUTPUT_DIR}" \
|
||||
"$(get-latest-run-id "${COMMIT_ID}" 'python_package.yml')"
|
||||
echo "done downloading python-package artifacts and NuGet package"
|
||||
|
||||
# get R-package artifacts
|
||||
echo "downloading R-package artifacts"
|
||||
gh run download \
|
||||
--repo "lightgbm-org/LightGBM" \
|
||||
--dir "${OUTPUT_DIR}" \
|
||||
"$(get-latest-run-id "${COMMIT_ID}" 'r_package.yml')"
|
||||
echo "done downloading R-package artifacts"
|
||||
|
||||
# get SWIG artifacts
|
||||
echo "downloading SWIG artifacts"
|
||||
gh run download \
|
||||
--repo "lightgbm-org/LightGBM" \
|
||||
--dir "${OUTPUT_DIR}" \
|
||||
"$(get-latest-run-id "${COMMIT_ID}" 'swig.yml')"
|
||||
echo "done downloading SWIG artifacts"
|
||||
|
||||
# 'gh run download' unpackages into nested directories like {artifact-name}/{file}.
|
||||
#
|
||||
# This moves all files to the top level and then deletes those {artifact-name}/ directories,
|
||||
# to make it easier to bulk upload all files to a release.
|
||||
echo "flattening directory structure"
|
||||
find "${OUTPUT_DIR}" -type f -mindepth 2 -exec mv -i '{}' "${OUTPUT_DIR}" \;
|
||||
find "${OUTPUT_DIR}" -type d -mindepth 1 -exec rm -r '{}' \+
|
||||
|
||||
echo "downloaded artifacts:"
|
||||
find "${OUTPUT_DIR}" -type f
|
||||
@@ -0,0 +1,40 @@
|
||||
Write-Output "Installing OpenCL CPU platform"
|
||||
|
||||
$installer = "AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe"
|
||||
|
||||
Write-Output "Downloading OpenCL platform installer"
|
||||
$ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows down download speed
|
||||
$params = @{
|
||||
OutFile = "$installer"
|
||||
Uri = "https://github.com/lightgbm-org/LightGBM/releases/download/v2.0.12/$installer"
|
||||
}
|
||||
Invoke-WebRequest @params
|
||||
|
||||
if (Test-Path "$installer") {
|
||||
Write-Output "Successfully downloaded OpenCL platform installer"
|
||||
} else {
|
||||
Write-Output "Unable to download OpenCL platform installer"
|
||||
Write-Output "Setting EXIT"
|
||||
$host.SetShouldExit(-1)
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Install OpenCL platform from installer executable
|
||||
Write-Output "Running OpenCL installer"
|
||||
Invoke-Command -ScriptBlock {
|
||||
Start-Process "$installer" -ArgumentList '/S /V"/quiet /norestart /passive /log opencl.log"' -Wait
|
||||
}
|
||||
|
||||
$property = Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors
|
||||
if ($null -eq $property) {
|
||||
Write-Output "Unable to install OpenCL CPU platform"
|
||||
Write-Output "OpenCL installation log:"
|
||||
Get-Content "opencl.log"
|
||||
Write-Output "Setting EXIT"
|
||||
$host.SetShouldExit(-1)
|
||||
exit 1
|
||||
} else {
|
||||
Write-Output "Successfully installed OpenCL CPU platform"
|
||||
Write-Output "Current OpenCL drivers:"
|
||||
Write-Output $property
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
# Install R dependencies, using only base R.
|
||||
#
|
||||
# Supported arguments:
|
||||
#
|
||||
# --all Install all the 'Depends', 'Imports', 'LinkingTo', and 'Suggests' dependencies
|
||||
# (automatically implies --build --test).
|
||||
#
|
||||
# --build Install the packages needed to build.
|
||||
#
|
||||
# --exclude=<pkg1,pkg2,...> Comma-delimited list of packages to NOT install.
|
||||
#
|
||||
# --include=<pkg1,pkg2,...> Comma-delimited list of additional packages to install.
|
||||
# These will always be installed, unless also used in "--exclude".
|
||||
#
|
||||
# --test Install packages needed to run tests.
|
||||
#
|
||||
|
||||
|
||||
# [description] Parse command line arguments into an R list.
|
||||
# Returns a list where keys are arguments and values
|
||||
# are either TRUE (for flags) or a vector of values passed via a
|
||||
# comma-delimited list.
|
||||
.parse_args <- function(args) {
|
||||
out <- list(
|
||||
"--all" = FALSE
|
||||
, "--build" = FALSE
|
||||
, "--exclude" = character(0L)
|
||||
, "--include" = character(0L)
|
||||
, "--test" = FALSE
|
||||
)
|
||||
for (arg in args) {
|
||||
parsed_arg <- unlist(strsplit(arg, "=", fixed = TRUE))
|
||||
arg_name <- parsed_arg[[1L]]
|
||||
if (!(arg_name %in% names(out))) {
|
||||
stop(sprintf("Unrecognized argument: '%s'", arg_name))
|
||||
}
|
||||
if (length(parsed_arg) == 2L) {
|
||||
# lists, like "--include=roxygen2,testthat"
|
||||
values <- unlist(strsplit(parsed_arg[[2L]], ",", fixed = TRUE))
|
||||
out[[arg_name]] <- values
|
||||
} else {
|
||||
# flags, like "--build"
|
||||
out[[arg]] <- TRUE
|
||||
}
|
||||
}
|
||||
return(out)
|
||||
}
|
||||
|
||||
args <- .parse_args(
|
||||
commandArgs(trailingOnly = TRUE)
|
||||
)
|
||||
|
||||
# which dependencies to install
|
||||
ALL_DEPS <- isTRUE(args[["--all"]])
|
||||
BUILD_DEPS <- ALL_DEPS || isTRUE(args[["--build"]])
|
||||
TEST_DEPS <- ALL_DEPS || isTRUE(args[["--test"]])
|
||||
|
||||
# force downloading of binary packages on macOS
|
||||
COMPILE_FROM_SOURCE <- "both"
|
||||
PACKAGE_TYPE <- getOption("pkgType")
|
||||
|
||||
# CRAN has precompiled binaries for macOS and Windows... prefer those,
|
||||
# for faster installation.
|
||||
if (Sys.info()[["sysname"]] == "Darwin" || .Platform$OS.type == "windows") {
|
||||
COMPILE_FROM_SOURCE <- "never"
|
||||
PACKAGE_TYPE <- "binary"
|
||||
}
|
||||
options(
|
||||
install.packages.check.source = "no"
|
||||
, install.packages.compile.from.source = COMPILE_FROM_SOURCE
|
||||
)
|
||||
|
||||
# always use the same CRAN mirror
|
||||
CRAN_MIRROR <- Sys.getenv("CRAN_MIRROR", unset = "https://cran.r-project.org")
|
||||
|
||||
# we always want these
|
||||
deps_to_install <- c(
|
||||
"data.table"
|
||||
, "jsonlite"
|
||||
, "Matrix"
|
||||
, "R6"
|
||||
)
|
||||
|
||||
if (isTRUE(BUILD_DEPS)) {
|
||||
deps_to_install <- c(
|
||||
deps_to_install
|
||||
, "knitr"
|
||||
, "markdown"
|
||||
)
|
||||
}
|
||||
|
||||
if (isTRUE(TEST_DEPS)) {
|
||||
deps_to_install <- c(
|
||||
deps_to_install
|
||||
, "RhpcBLASctl"
|
||||
, "testthat"
|
||||
)
|
||||
}
|
||||
|
||||
# add packages passed through '--include'
|
||||
deps_to_install <- unique(c(
|
||||
deps_to_install
|
||||
, args[["--include"]]
|
||||
))
|
||||
|
||||
# remove packages passed through '--exclude'
|
||||
deps_to_install <- setdiff(
|
||||
x = deps_to_install
|
||||
, args[["--exclude"]]
|
||||
)
|
||||
|
||||
msg <- sprintf(
|
||||
"[install-r-deps] installing R packages: %s\n"
|
||||
, toString(sort(deps_to_install))
|
||||
)
|
||||
cat(msg)
|
||||
|
||||
install.packages( # nolint: undesirable_function.
|
||||
pkgs = deps_to_install
|
||||
, dependencies = c("Depends", "Imports", "LinkingTo")
|
||||
, lib = Sys.getenv("R_LIB_PATH", unset = .libPaths()[[1L]])
|
||||
, repos = CRAN_MIRROR
|
||||
, type = PACKAGE_TYPE
|
||||
, Ncpus = parallel::detectCores()
|
||||
)
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
pwsh -command "Install-Module -Name PSScriptAnalyzer -Scope CurrentUser -SkipPublisherCheck"
|
||||
echo "Linting PowerShell code"
|
||||
pwsh -file ./.ci/lint-powershell.ps1 || exit 1
|
||||
|
||||
conda create -q -y -n test-env \
|
||||
"python=3.14[build=*_cp*]" \
|
||||
'pre-commit>=3.8.0' \
|
||||
'r-lintr>=3.3.0'
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source activate test-env
|
||||
|
||||
echo "Running pre-commit checks"
|
||||
pre-commit run --all-files || exit 1
|
||||
|
||||
echo "Linting R code"
|
||||
Rscript ./.ci/lint-r-code.R "$(pwd)" || exit 1
|
||||
@@ -0,0 +1,87 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$settings = @{
|
||||
Severity = @(
|
||||
'Information',
|
||||
'Warning',
|
||||
'Error'
|
||||
)
|
||||
IncludeDefaultRules = $true
|
||||
ExcludeRules = @(
|
||||
'PSAvoidUsingInvokeExpression'
|
||||
)
|
||||
# Additional rules that are disabled by default.
|
||||
#
|
||||
# Some of the skips could be replaced with inline comments if PSScriptAnalyzer
|
||||
# supports that in the future (https://github.com/PowerShell/PSScriptAnalyzer/issues/849).
|
||||
Rules = @{
|
||||
PSAvoidExclaimOperator = @{
|
||||
Enable = $true
|
||||
}
|
||||
PSAvoidLongLines = @{
|
||||
Enable = $true
|
||||
MaximumLineLength = 120
|
||||
}
|
||||
PSAvoidSemicolonsAsLineTerminators = @{
|
||||
Enable = $true
|
||||
}
|
||||
PSPlaceCloseBrace = @{
|
||||
Enable = $true
|
||||
NoEmptyLineBefore = $true
|
||||
IgnoreOneLineBlock = $true
|
||||
NewLineAfter = $false
|
||||
}
|
||||
PSPlaceOpenBrace = @{
|
||||
Enable = $true
|
||||
OnSameLine = $true
|
||||
NewLineAfter = $true
|
||||
IgnoreOneLineBlock = $true
|
||||
}
|
||||
PSUseConsistentIndentation = @{
|
||||
Enable = $true
|
||||
IndentationSize = 4
|
||||
PipelineIndentation = 'IncreaseIndentationAfterEveryPipeline'
|
||||
Kind = 'space'
|
||||
}
|
||||
PSUseConsistentWhitespace = @{
|
||||
Enable = $true
|
||||
CheckInnerBrace = $true
|
||||
CheckOpenBrace = $true
|
||||
CheckOpenParen = $true
|
||||
CheckOperator = $true
|
||||
CheckSeparator = $true
|
||||
CheckPipe = $true
|
||||
CheckPipeForRedundantWhitespace = $true
|
||||
CheckParameter = $true
|
||||
IgnoreAssignmentOperatorInsideHashTable = $false
|
||||
}
|
||||
PSUseCorrectCasing = @{
|
||||
Enable = $true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# this pre-listing of files can be removed whenever PSScriptAnalyzer adds support for exclusions.
|
||||
#
|
||||
# see:
|
||||
#
|
||||
# * https://github.com/PowerShell/PSScriptAnalyzer/issues/561
|
||||
# * https://github.com/PowerShell/vscode-powershell/issues/3048
|
||||
#
|
||||
# lint-powershell.ps1 itself is included here because linting this script itself
|
||||
# sometimes fails (non-deterministically!) with an error like "Object reference not set to an instance of an object"
|
||||
#
|
||||
$files = @(
|
||||
Get-ChildItem -Path ./ -Recurse -Force -Filter '*.ps1' |
|
||||
Where-Object { $_.FullName -notmatch '[/\\]bin[/\\]' } |
|
||||
Where-Object { $_.FullName -notmatch '[/\\]external_libs[/\\]' } |
|
||||
Where-Object { $_.FullName -notmatch '[/\\]\.pixi[/\\]' } |
|
||||
Where-Object { $_.FullName -notmatch '[/\\]venv[/\\]' } |
|
||||
Where-Object { $_.Name -ne 'lint-powershell.ps1' } |
|
||||
ForEach-Object { $_.FullName }
|
||||
)
|
||||
|
||||
foreach ($file in $files) {
|
||||
Write-Output "linting '$file'"
|
||||
Invoke-ScriptAnalyzer -Path $file -EnableExit -Settings $settings
|
||||
}
|
||||
Executable
+154
@@ -0,0 +1,154 @@
|
||||
loadNamespace("lintr")
|
||||
|
||||
args <- commandArgs(
|
||||
trailingOnly = TRUE
|
||||
)
|
||||
SOURCE_DIR <- args[[1L]]
|
||||
|
||||
FILES_TO_LINT <- list.files(
|
||||
path = SOURCE_DIR
|
||||
, pattern = "\\.r$|\\.rmd$"
|
||||
, all.files = TRUE
|
||||
, ignore.case = TRUE
|
||||
, full.names = TRUE
|
||||
, recursive = TRUE
|
||||
, include.dirs = FALSE
|
||||
)
|
||||
|
||||
# text to use for pipe operators from packages like 'magrittr'
|
||||
pipe_text <- paste0(
|
||||
"For consistency and the sake of being explicit, this project's code "
|
||||
, "does not use the pipe operator."
|
||||
)
|
||||
|
||||
# text to use for functions that should only be called interactively
|
||||
interactive_text <- paste0(
|
||||
"Functions like '?', 'help', and 'install.packages()' should only be used "
|
||||
, "interactively, not in package code."
|
||||
)
|
||||
|
||||
LINTERS_TO_USE <- list(
|
||||
"absolute_path" = lintr::absolute_path_linter()
|
||||
, "any_duplicated" = lintr::any_duplicated_linter()
|
||||
, "any_is_na" = lintr::any_is_na_linter()
|
||||
, "assignment" = lintr::assignment_linter()
|
||||
, "backport" = lintr::backport_linter()
|
||||
, "boolean_arithmetic" = lintr::boolean_arithmetic_linter()
|
||||
, "braces" = lintr::brace_linter()
|
||||
, "class_equals" = lintr::class_equals_linter()
|
||||
, "commas" = lintr::commas_linter()
|
||||
, "conjunct_test" = lintr::conjunct_test_linter()
|
||||
, "duplicate_argument" = lintr::duplicate_argument_linter()
|
||||
, "empty_assignment" = lintr::empty_assignment_linter()
|
||||
, "equals_na" = lintr::equals_na_linter()
|
||||
, "fixed_regex" = lintr::fixed_regex_linter()
|
||||
, "for_loop_index" = lintr::for_loop_index_linter()
|
||||
, "function_left" = lintr::function_left_parentheses_linter()
|
||||
, "function_return" = lintr::function_return_linter()
|
||||
, "implicit_assignment" = lintr::implicit_assignment_linter()
|
||||
, "implicit_integers" = lintr::implicit_integer_linter()
|
||||
, "infix_spaces" = lintr::infix_spaces_linter()
|
||||
, "inner_combine" = lintr::inner_combine_linter()
|
||||
, "is_numeric" = lintr::is_numeric_linter()
|
||||
, "lengths" = lintr::lengths_linter()
|
||||
, "length_levels" = lintr::length_levels_linter()
|
||||
, "length_test" = lintr::length_test_linter()
|
||||
, "line_length" = lintr::line_length_linter(length = 120L)
|
||||
, "literal_coercion" = lintr::literal_coercion_linter()
|
||||
, "matrix" = lintr::matrix_apply_linter()
|
||||
, "missing_argument" = lintr::missing_argument_linter()
|
||||
, "non_portable_path" = lintr::nonportable_path_linter()
|
||||
, "numeric_leading_zero" = lintr::numeric_leading_zero_linter()
|
||||
, "outer_negation" = lintr::outer_negation_linter()
|
||||
, "package_hooks" = lintr::package_hooks_linter()
|
||||
, "paren_body" = lintr::paren_body_linter()
|
||||
, "paste" = lintr::paste_linter()
|
||||
, "quotes" = lintr::quotes_linter()
|
||||
, "redundant_equals" = lintr::redundant_equals_linter()
|
||||
, "regex_subset" = lintr::regex_subset_linter()
|
||||
, "routine_registration" = lintr::routine_registration_linter()
|
||||
, "scalar_in" = lintr::scalar_in_linter()
|
||||
, "semicolon" = lintr::semicolon_linter()
|
||||
, "seq" = lintr::seq_linter()
|
||||
, "spaces_inside" = lintr::spaces_inside_linter()
|
||||
, "spaces_left_parens" = lintr::spaces_left_parentheses_linter()
|
||||
, "sprintf" = lintr::sprintf_linter()
|
||||
, "string_boundary" = lintr::string_boundary_linter()
|
||||
, "todo_comments" = lintr::todo_comment_linter(c("todo", "fixme", "to-do"))
|
||||
, "trailing_blank" = lintr::trailing_blank_lines_linter()
|
||||
, "trailing_white" = lintr::trailing_whitespace_linter()
|
||||
, "true_false" = lintr::T_and_F_symbol_linter()
|
||||
, "undesirable_function" = lintr::undesirable_function_linter(
|
||||
fun = c(
|
||||
"cbind" = paste0(
|
||||
"cbind is an unsafe way to build up a data frame. merge() or direct "
|
||||
, "column assignment is preferred."
|
||||
)
|
||||
, "dyn.load" = "Directly loading or unloading .dll or .so files in package code should not be necessary."
|
||||
, "dyn.unload" = "Directly loading or unloading .dll or .so files in package code should not be necessary."
|
||||
, "help" = interactive_text
|
||||
, "ifelse" = "The use of ifelse() is dangerous because it will silently allow mixing types."
|
||||
, "install.packages" = interactive_text
|
||||
, "is.list" = paste0(
|
||||
"This project uses data.table, and is.list(x) is TRUE for a data.table. "
|
||||
, "identical(class(x), 'list') is a safer way to check that something is an R list object."
|
||||
)
|
||||
, "rbind" = "data.table::rbindlist() is faster and safer than rbind(), and is preferred in this project."
|
||||
, "require" = paste0(
|
||||
"library() is preferred to require() because it will raise an error immediately "
|
||||
, "if a package is missing."
|
||||
)
|
||||
)
|
||||
)
|
||||
, "undesirable_operator" = lintr::undesirable_operator_linter(
|
||||
op = c(
|
||||
"%>%" = pipe_text
|
||||
, "%.%" = pipe_text
|
||||
, "%..%" = pipe_text
|
||||
, "|>" = pipe_text
|
||||
, "?" = interactive_text
|
||||
, "??" = interactive_text
|
||||
)
|
||||
)
|
||||
, "unnecessary_concatenation" = lintr::unnecessary_concatenation_linter()
|
||||
, "unnecessary_lambda" = lintr::unnecessary_lambda_linter()
|
||||
, "unreachable_code" = lintr::unreachable_code_linter()
|
||||
, "unused_import" = lintr::unused_import_linter()
|
||||
, "vector_logic" = lintr::vector_logic_linter()
|
||||
, "whitespace" = lintr::whitespace_linter()
|
||||
)
|
||||
|
||||
noquote(paste0(length(FILES_TO_LINT), " R files need linting"))
|
||||
|
||||
results <- NULL
|
||||
|
||||
for (r_file in FILES_TO_LINT) {
|
||||
|
||||
this_result <- lintr::lint(
|
||||
filename = r_file
|
||||
, linters = LINTERS_TO_USE
|
||||
, cache = FALSE
|
||||
)
|
||||
|
||||
print(
|
||||
sprintf(
|
||||
"Found %i linting errors in %s"
|
||||
, length(this_result)
|
||||
, r_file
|
||||
)
|
||||
, quote = FALSE
|
||||
)
|
||||
|
||||
results <- c(results, this_result)
|
||||
|
||||
}
|
||||
|
||||
issues_found <- length(results)
|
||||
|
||||
noquote(paste0("Total linting issues found: ", issues_found))
|
||||
|
||||
if (issues_found > 0L) {
|
||||
print(results)
|
||||
}
|
||||
|
||||
quit(save = "no", status = issues_found)
|
||||
@@ -0,0 +1,408 @@
|
||||
# coding: utf-8
|
||||
"""Helper script for generating config file and parameters list.
|
||||
|
||||
This script generates LightGBM/src/io/config_auto.cpp file
|
||||
with list of all parameters, aliases table and other routines
|
||||
along with parameters description in LightGBM/docs/Parameters.rst file
|
||||
from the information in LightGBM/include/LightGBM/config.h file.
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
|
||||
def get_parameter_infos(config_hpp: Path) -> Tuple[List[Tuple[str, int]], List[List[Dict[str, List]]]]:
|
||||
"""Parse config header file.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config_hpp : pathlib.Path
|
||||
Path to the config header file.
|
||||
|
||||
Returns
|
||||
-------
|
||||
infos : tuple
|
||||
Tuple with names and content of sections.
|
||||
"""
|
||||
is_inparameter = False
|
||||
cur_key = None
|
||||
key_lvl = 0
|
||||
cur_info: Dict[str, List] = {}
|
||||
keys = []
|
||||
member_infos: List[List[Dict[str, List]]] = []
|
||||
with open(config_hpp) as config_hpp_file:
|
||||
for line in config_hpp_file:
|
||||
if line.strip() in {"#ifndef __NVCC__", "#endif // __NVCC__"}:
|
||||
continue
|
||||
if "#pragma region Parameters" in line:
|
||||
is_inparameter = True
|
||||
elif "#pragma region" in line and "Parameters" in line:
|
||||
key_lvl += 1
|
||||
cur_key = line.split("region")[1].strip()
|
||||
keys.append((cur_key, key_lvl))
|
||||
member_infos.append([])
|
||||
elif "#pragma endregion" in line:
|
||||
key_lvl -= 1
|
||||
if cur_key is not None:
|
||||
cur_key = None
|
||||
elif is_inparameter:
|
||||
is_inparameter = False
|
||||
elif cur_key is not None:
|
||||
line = line.strip()
|
||||
if line.startswith("//"):
|
||||
key, _, val = line[2:].partition("=")
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
if key not in cur_info:
|
||||
if key == "descl2" and "desc" not in cur_info:
|
||||
cur_info["desc"] = []
|
||||
elif key != "descl2":
|
||||
cur_info[key] = []
|
||||
if key == "desc":
|
||||
cur_info["desc"].append(("l1", val))
|
||||
elif key == "descl2":
|
||||
cur_info["desc"].append(("l2", val))
|
||||
else:
|
||||
cur_info[key].append(val)
|
||||
elif line:
|
||||
has_eqsgn = False
|
||||
tokens = line.split("=")
|
||||
if len(tokens) == 2:
|
||||
if "default" not in cur_info:
|
||||
cur_info["default"] = [tokens[1][:-1].strip()]
|
||||
has_eqsgn = True
|
||||
tokens = line.split()
|
||||
cur_info["inner_type"] = [tokens[0].strip()]
|
||||
if "name" not in cur_info:
|
||||
if has_eqsgn:
|
||||
cur_info["name"] = [tokens[1].strip()]
|
||||
else:
|
||||
cur_info["name"] = [tokens[1][:-1].strip()]
|
||||
member_infos[-1].append(cur_info)
|
||||
cur_info = {}
|
||||
|
||||
return keys, member_infos
|
||||
|
||||
|
||||
def get_names(infos: List[List[Dict[str, List]]]) -> List[str]:
|
||||
"""Get names of all parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
infos : list
|
||||
Content of the config header file.
|
||||
|
||||
Returns
|
||||
-------
|
||||
names : list
|
||||
Names of all parameters.
|
||||
"""
|
||||
names = []
|
||||
for x in infos:
|
||||
for y in x:
|
||||
names.append(y["name"][0])
|
||||
return names
|
||||
|
||||
|
||||
def get_alias(infos: List[List[Dict[str, List]]]) -> List[Tuple[str, str]]:
|
||||
"""Get aliases of all parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
infos : list
|
||||
Content of the config header file.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pairs : list
|
||||
List of tuples (param alias, param name).
|
||||
"""
|
||||
pairs = []
|
||||
for x in infos:
|
||||
for y in x:
|
||||
if "alias" in y:
|
||||
name = y["name"][0]
|
||||
alias = y["alias"][0].split(",")
|
||||
for name2 in alias:
|
||||
pairs.append((name2.strip(), name))
|
||||
return pairs
|
||||
|
||||
|
||||
def parse_check(check: str, reverse: bool = False) -> Tuple[str, str]:
|
||||
"""Parse the constraint.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
check : str
|
||||
String representation of the constraint.
|
||||
reverse : bool, optional (default=False)
|
||||
Whether to reverse the sign of the constraint.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pair : tuple
|
||||
Parsed constraint in the form of tuple (value, sign).
|
||||
"""
|
||||
try:
|
||||
idx = 1
|
||||
float(check[idx:])
|
||||
except ValueError:
|
||||
idx = 2
|
||||
float(check[idx:])
|
||||
if reverse:
|
||||
reversed_sign = {"<": ">", ">": "<", "<=": ">=", ">=": "<="}
|
||||
return check[idx:], reversed_sign[check[:idx]]
|
||||
else:
|
||||
return check[idx:], check[:idx]
|
||||
|
||||
|
||||
def set_one_var_from_string(name: str, param_type: str, checks: List[str]) -> str:
|
||||
"""Construct code for auto config file for one param value.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
Name of the parameter.
|
||||
param_type : str
|
||||
Type of the parameter.
|
||||
checks : list
|
||||
Constraints of the parameter.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ret : str
|
||||
Lines of auto config file with getting and checks of one parameter value.
|
||||
"""
|
||||
ret = ""
|
||||
univar_mapper = {"int": "GetInt", "double": "GetDouble", "bool": "GetBool", "std::string": "GetString"}
|
||||
if "vector" not in param_type:
|
||||
ret += f' {univar_mapper[param_type]}(params, "{name}", &{name});\n'
|
||||
if len(checks) > 0:
|
||||
check_mapper = {"<": "LT", ">": "GT", "<=": "LE", ">=": "GE"}
|
||||
for check in checks:
|
||||
value, sign = parse_check(check)
|
||||
ret += f" CHECK_{check_mapper[sign]}({name}, {value});\n"
|
||||
ret += "\n"
|
||||
else:
|
||||
ret += f' if (GetString(params, "{name}", &tmp_str)) {{\n'
|
||||
type2 = param_type.split("<")[1][:-1]
|
||||
if type2 == "std::string":
|
||||
ret += f" {name} = Common::Split(tmp_str.c_str(), ',');\n"
|
||||
else:
|
||||
ret += f" {name} = Common::StringToArray<{type2}>(tmp_str, ',');\n"
|
||||
ret += " }\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def gen_parameter_description(
|
||||
sections: List[Tuple[str, int]], descriptions: List[List[Dict[str, List]]], params_rst: Path
|
||||
) -> None:
|
||||
"""Write descriptions of parameters to the documentation file.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sections : list
|
||||
Names of parameters sections.
|
||||
descriptions : list
|
||||
Structured descriptions of parameters.
|
||||
params_rst : pathlib.Path
|
||||
Path to the file with parameters documentation.
|
||||
"""
|
||||
params_to_write = []
|
||||
lvl_mapper = {1: "-", 2: "~"}
|
||||
for (section_name, section_lvl), section_params in zip(sections, descriptions, strict=True):
|
||||
heading_sign = lvl_mapper[section_lvl]
|
||||
params_to_write.append(f"{section_name}\n{heading_sign * len(section_name)}")
|
||||
for param_desc in section_params:
|
||||
name = param_desc["name"][0]
|
||||
default_raw = param_desc["default"][0]
|
||||
default = default_raw.strip('"') if len(default_raw.strip('"')) > 0 else default_raw
|
||||
param_type = param_desc.get("type", param_desc["inner_type"])[0].split(":")[-1].split("<")[-1].strip(">")
|
||||
options = param_desc.get("options", [])
|
||||
if len(options) > 0:
|
||||
opts = "``, ``".join([x.strip() for x in options[0].split(",")])
|
||||
options_str = f", options: ``{opts}``"
|
||||
else:
|
||||
options_str = ""
|
||||
aliases = param_desc.get("alias", [])
|
||||
if len(aliases) > 0:
|
||||
aliases_joined = "``, ``".join([x.strip() for x in aliases[0].split(",")])
|
||||
aliases_str = f", aliases: ``{aliases_joined}``"
|
||||
else:
|
||||
aliases_str = ""
|
||||
checks = sorted(param_desc.get("check", []))
|
||||
checks_len = len(checks)
|
||||
if checks_len > 1:
|
||||
number1, sign1 = parse_check(checks[0])
|
||||
number2, sign2 = parse_check(checks[1], reverse=True)
|
||||
checks_str = f", constraints: ``{number2} {sign2} {name} {sign1} {number1}``"
|
||||
elif checks_len == 1:
|
||||
number, sign = parse_check(checks[0])
|
||||
checks_str = f", constraints: ``{name} {sign} {number}``"
|
||||
else:
|
||||
checks_str = ""
|
||||
main_desc = f'- ``{name}`` :raw-html:`<a id="{name}" title="Permalink to this parameter" href="#{name}">🔗︎</a>`, default = ``{default}``, type = {param_type}{options_str}{aliases_str}{checks_str}'
|
||||
params_to_write.append(main_desc)
|
||||
params_to_write.extend([f"{' ' * 3 * int(desc[0][-1])}- {desc[1]}" for desc in param_desc["desc"]])
|
||||
|
||||
with open(params_rst) as original_params_file:
|
||||
all_lines = original_params_file.read()
|
||||
before, start_sep, _ = all_lines.partition(".. start params list\n\n")
|
||||
_, end_sep, after = all_lines.partition("\n\n.. end params list")
|
||||
|
||||
with open(params_rst, "w") as new_params_file:
|
||||
new_params_file.write(before)
|
||||
new_params_file.write(start_sep)
|
||||
new_params_file.write("\n\n".join(params_to_write))
|
||||
new_params_file.write(end_sep)
|
||||
new_params_file.write(after)
|
||||
|
||||
|
||||
def gen_parameter_code(
|
||||
config_hpp: Path, config_out_cpp: Path
|
||||
) -> Tuple[List[Tuple[str, int]], List[List[Dict[str, List]]]]:
|
||||
"""Generate auto config file.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config_hpp : pathlib.Path
|
||||
Path to the config header file.
|
||||
config_out_cpp : pathlib.Path
|
||||
Path to the auto config file.
|
||||
|
||||
Returns
|
||||
-------
|
||||
infos : tuple
|
||||
Tuple with names and content of sections.
|
||||
"""
|
||||
keys, infos = get_parameter_infos(config_hpp)
|
||||
names = get_names(infos)
|
||||
alias = get_alias(infos)
|
||||
names_with_aliases = defaultdict(list)
|
||||
str_to_write = r"""/*!
|
||||
* Copyright (c) 2018-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2018-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*
|
||||
* \note
|
||||
* This file is auto generated by LightGBM\.ci\parameter-generator.py from LightGBM\include\LightGBM\config.h file.
|
||||
*/
|
||||
"""
|
||||
str_to_write += "#include <LightGBM/config.h>\n\n"
|
||||
str_to_write += "#include <string>\n"
|
||||
str_to_write += "#include <unordered_map>\n"
|
||||
str_to_write += "#include <unordered_set>\n"
|
||||
str_to_write += "#include <vector>\n\n"
|
||||
str_to_write += "namespace LightGBM {\n"
|
||||
|
||||
# alias table
|
||||
str_to_write += "const std::unordered_map<std::string, std::string>& Config::alias_table() {\n"
|
||||
str_to_write += " static std::unordered_map<std::string, std::string> aliases({\n"
|
||||
|
||||
for pair in alias:
|
||||
str_to_write += f' {{"{pair[0]}", "{pair[1]}"}},\n'
|
||||
names_with_aliases[pair[1]].append(pair[0])
|
||||
str_to_write += " });\n"
|
||||
str_to_write += " return aliases;\n"
|
||||
str_to_write += "}\n\n"
|
||||
|
||||
# names
|
||||
str_to_write += "const std::unordered_set<std::string>& Config::parameter_set() {\n"
|
||||
str_to_write += " static std::unordered_set<std::string> params({\n"
|
||||
|
||||
for name in names:
|
||||
str_to_write += f' "{name}",\n'
|
||||
str_to_write += " });\n"
|
||||
str_to_write += " return params;\n"
|
||||
str_to_write += "}\n\n"
|
||||
# from strings
|
||||
str_to_write += "void Config::GetMembersFromString(const std::unordered_map<std::string, std::string>& params) {\n"
|
||||
str_to_write += ' std::string tmp_str = "";\n'
|
||||
for x in infos:
|
||||
for y in x:
|
||||
if "[no-automatically-extract]" in y:
|
||||
continue
|
||||
param_type = y["inner_type"][0]
|
||||
name = y["name"][0]
|
||||
checks = []
|
||||
if "check" in y:
|
||||
checks = y["check"]
|
||||
tmp = set_one_var_from_string(name, param_type, checks)
|
||||
str_to_write += tmp
|
||||
# tails
|
||||
str_to_write = f"{str_to_write.strip()}\n}}\n\n"
|
||||
str_to_write += "std::string Config::SaveMembersToString() const {\n"
|
||||
str_to_write += " std::stringstream str_buf;\n"
|
||||
for x in infos:
|
||||
for y in x:
|
||||
if "[no-save]" in y:
|
||||
continue
|
||||
param_type = y["inner_type"][0]
|
||||
name = y["name"][0]
|
||||
if "vector" in param_type:
|
||||
if "int8" in param_type:
|
||||
str_to_write += f' str_buf << "[{name}: " << Common::Join(Common::ArrayCast<int8_t, int>({name}), ",") << "]\\n";\n'
|
||||
else:
|
||||
str_to_write += f' str_buf << "[{name}: " << Common::Join({name}, ",") << "]\\n";\n'
|
||||
else:
|
||||
str_to_write += f' str_buf << "[{name}: " << {name} << "]\\n";\n'
|
||||
# tails
|
||||
str_to_write += " return str_buf.str();\n"
|
||||
str_to_write += "}\n\n"
|
||||
|
||||
str_to_write += """const std::unordered_map<std::string, std::vector<std::string>>& Config::parameter2aliases() {
|
||||
static std::unordered_map<std::string, std::vector<std::string>> map({"""
|
||||
for name in names:
|
||||
str_to_write += '\n {"' + name + '", '
|
||||
if names_with_aliases[name]:
|
||||
str_to_write += '{"' + '", "'.join(names_with_aliases[name]) + '"}},'
|
||||
else:
|
||||
str_to_write += "{}},"
|
||||
str_to_write += """
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
"""
|
||||
str_to_write += """const std::unordered_map<std::string, std::string>& Config::ParameterTypes() {
|
||||
static std::unordered_map<std::string, std::string> map({"""
|
||||
int_t_pat = re.compile(r"int\d+_t")
|
||||
# the following are stored as comma separated strings but are arrays in the wrappers
|
||||
overrides = {
|
||||
"categorical_feature": "vector<int>",
|
||||
"ignore_column": "vector<int>",
|
||||
"interaction_constraints": "vector<vector<int>>",
|
||||
}
|
||||
for x in infos:
|
||||
for y in x:
|
||||
name = y["name"][0]
|
||||
if name == "task":
|
||||
continue
|
||||
if name in overrides:
|
||||
param_type = overrides[name]
|
||||
else:
|
||||
param_type = int_t_pat.sub("int", y["inner_type"][0]).replace("std::", "")
|
||||
str_to_write += '\n {"' + name + '", "' + param_type + '"},'
|
||||
str_to_write += """
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
str_to_write += "} // namespace LightGBM\n"
|
||||
with open(config_out_cpp, "w") as config_out_cpp_file:
|
||||
config_out_cpp_file.write(str_to_write)
|
||||
|
||||
return keys, infos
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
current_dir = Path(__file__).absolute().parent
|
||||
config_hpp = current_dir.parent / "include" / "LightGBM" / "config.h"
|
||||
config_out_cpp = current_dir.parent / "src" / "io" / "config_auto.cpp"
|
||||
params_rst = current_dir.parent / "docs" / "Parameters.rst"
|
||||
sections, descriptions = gen_parameter_code(config_hpp, config_out_cpp)
|
||||
gen_parameter_description(sections, descriptions, params_rst)
|
||||
@@ -0,0 +1,24 @@
|
||||
# nightlies for: matplotlib, numpy, pandas, pyarrow, scipy, scikit-learn
|
||||
--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
|
||||
|
||||
# runtime dependencies (using `.dev0` suffix to allow nightlies)
|
||||
#
|
||||
# pinning rules:
|
||||
#
|
||||
# * latest versions of lightgbm's dependencies,
|
||||
# * including pre-releases and nightlies
|
||||
#
|
||||
cffi>=2.0.0
|
||||
matplotlib>=3.11.0.dev0
|
||||
narwhals>=2.21.0.dev0
|
||||
numpy>=2.5.0.dev0
|
||||
pandas>=3.1.0.dev0
|
||||
polars>=1.40.1.dev0
|
||||
pyarrow>=24.0.0.dev0
|
||||
scikit-learn>=1.9.dev0
|
||||
scipy>=1.18.0.dev0
|
||||
|
||||
# testing-only dependencies
|
||||
cloudpickle>=3.1.2
|
||||
psutil>=7.2.2
|
||||
pytest>=9.0.2
|
||||
@@ -0,0 +1,15 @@
|
||||
# oldest versions of dependencies published after
|
||||
# minimum supported Python version's first release,
|
||||
# for which there are wheels compatible with the
|
||||
# python:{version} image
|
||||
#
|
||||
# see https://devguide.python.org/versions/
|
||||
#
|
||||
cffi==1.15.1
|
||||
narwhals==1.15.0
|
||||
numpy==1.22.4
|
||||
pandas==1.3.4
|
||||
polars==1.0.0
|
||||
pyarrow==16.0.0
|
||||
scikit-learn==1.0.2
|
||||
scipy==1.7.2
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# [description]
|
||||
# Rerun specified workflow for given pull request.
|
||||
#
|
||||
# [usage]
|
||||
# rerun-workflow.sh <WORKFLOW_ID> <PR_BRANCH>
|
||||
#
|
||||
# WORKFLOW_ID: Identifier (config name of ID) of a workflow to be rerun.
|
||||
#
|
||||
# PR_BRANCH: Name of pull request's branch.
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
if [ -z "$GITHUB_ACTIONS" ]; then
|
||||
echo "Must be run inside GitHub Actions CI"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "Usage: $0 <WORKFLOW_ID> <PR_BRANCH>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
workflow_id=$1
|
||||
pr_branch=$2
|
||||
|
||||
# --branch for some GitHub GLI commands does not respect the difference between forks and branches
|
||||
# on the main repo. While some parts of the GitHub API refer to the branch of a workflow
|
||||
# as '{org}:{branch}' for branches from forks, others expect only '{branch}'.
|
||||
#
|
||||
# This expansion trims a leading '{org}:' from 'pr_branch' if one is present.
|
||||
pr_branch_no_fork_prefix="${pr_branch/*:/}"
|
||||
|
||||
RUN_ID=$(
|
||||
gh run list \
|
||||
--repo 'lightgbm-org/LightGBM' \
|
||||
--workflow "${workflow_id}" \
|
||||
--event "pull_request" \
|
||||
--branch "${pr_branch_no_fork_prefix}" \
|
||||
--json 'createdAt,databaseId' \
|
||||
--jq 'sort_by(.createdAt) | reverse | .[0] | .databaseId'
|
||||
)
|
||||
|
||||
if [[ -z "${RUN_ID}" ]]; then
|
||||
echo "ERROR: failed to find a run of workflow '${workflow_id}' for branch '${pr_branch}'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Re-running workflow '${workflow_id}' (run ID ${RUN_ID})"
|
||||
|
||||
gh run rerun \
|
||||
--repo 'lightgbm-org/LightGBM' \
|
||||
"${RUN_ID}"
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -u -o pipefail
|
||||
|
||||
PKG_TARBALL="${1}"
|
||||
declare -i ALLOWED_CHECK_NOTES=${2}
|
||||
|
||||
# 'R CMD check' redirects installation logs to a file, and returns
|
||||
# a non-0 exit code if ERRORs are raised.
|
||||
#
|
||||
# The '||' here gives us an opportunity to echo out the installation
|
||||
# logs prior to exiting the script.
|
||||
check_succeeded="yes"
|
||||
R CMD check "${PKG_TARBALL}" \
|
||||
--as-cran \
|
||||
--run-donttest \
|
||||
|| check_succeeded="no"
|
||||
|
||||
CHECK_LOG_FILE=lightgbm.Rcheck/00check.log
|
||||
BUILD_LOG_FILE=lightgbm.Rcheck/00install.out
|
||||
|
||||
echo "R CMD check build logs:"
|
||||
cat "${BUILD_LOG_FILE}"
|
||||
|
||||
if [[ $check_succeeded == "no" ]]; then
|
||||
echo "R CMD check failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# WARNINGs or ERRORs should be treated as a failure
|
||||
if grep -q -E "WARNING|ERROR" "${CHECK_LOG_FILE}"; then
|
||||
echo "WARNINGs or ERRORs have been found by R CMD check"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Allow a configurable number of NOTEs.
|
||||
# Sometimes NOTEs are raised in CI that wouldn't show up on an actual CRAN submission.
|
||||
set +e
|
||||
NUM_CHECK_NOTES=$(
|
||||
grep -o -E '[0-9]+ NOTE' "${CHECK_LOG_FILE}" \
|
||||
| sed 's/[^0-9]*//g'
|
||||
)
|
||||
if [[ ${NUM_CHECK_NOTES} -gt ${ALLOWED_CHECK_NOTES} ]]; then
|
||||
echo "Found ${NUM_CHECK_NOTES} NOTEs from R CMD check. Only ${ALLOWED_CHECK_NOTES} are allowed"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# [description]
|
||||
# Set a status with a given name to the specified commit.
|
||||
#
|
||||
# [usage]
|
||||
# set-commit-status.sh <NAME> <STATUS> <SHA>
|
||||
#
|
||||
# NAME: Name of status.
|
||||
# Status with existing name overwrites a previous one.
|
||||
#
|
||||
# STATUS: Status to be set.
|
||||
# Can be "error", "failure", "pending" or "success".
|
||||
#
|
||||
# SHA: SHA of a commit to set a status on.
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
if [ -z "$GITHUB_ACTIONS" ]; then
|
||||
echo "Must be run inside GitHub Actions CI"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $# -ne 3 ]; then
|
||||
echo "Usage: $0 <NAME> <STATUS> <SHA>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
name=$1
|
||||
|
||||
status=$2
|
||||
status=${status/error/failure}
|
||||
status=${status/cancelled/failure}
|
||||
status=${status/timed_out/failure}
|
||||
status=${status/in_progress/pending}
|
||||
status=${status/queued/pending}
|
||||
|
||||
sha=$3
|
||||
|
||||
data=$(
|
||||
jq -n \
|
||||
--arg state "${status}" \
|
||||
--arg url "${GITHUB_SERVER_URL}/lightgbm-org/LightGBM/actions/runs/${GITHUB_RUN_ID}" \
|
||||
--arg name "${name}" \
|
||||
'{"state":$state,"target_url":$url,"context":$name}'
|
||||
)
|
||||
|
||||
curl -sL \
|
||||
--fail \
|
||||
-X POST \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||
-d "$data" \
|
||||
"${GITHUB_API_URL}/repos/lightgbm-org/LightGBM/statuses/$sha"
|
||||
Executable
+192
@@ -0,0 +1,192 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
# defaults
|
||||
IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"}
|
||||
SETUP_CONDA=${SETUP_CONDA:-"true"}
|
||||
|
||||
ARCH=$(uname -m)
|
||||
|
||||
|
||||
if [[ $OS_NAME == "macos" ]]; then
|
||||
# Check https://github.com/actions/runner-images/tree/main/images/macos for available
|
||||
# versions of Xcode
|
||||
macos_ver=$(sw_vers --productVersion)
|
||||
if [[ "${macos_ver}" =~ 15. ]]; then
|
||||
xcode_path="/Applications/Xcode_16.0.app/Contents/Developer"
|
||||
else
|
||||
xcode_path="/Applications/Xcode_15.0.app/Contents/Developer"
|
||||
fi
|
||||
sudo xcode-select -s "${xcode_path}" || exit 1
|
||||
if [[ $COMPILER == "clang" ]]; then
|
||||
brew install libomp
|
||||
else # gcc
|
||||
brew install 'gcc@14'
|
||||
fi
|
||||
if [[ $TASK == "mpi" ]]; then
|
||||
brew install open-mpi
|
||||
fi
|
||||
if [[ $TASK == "swig" ]]; then
|
||||
brew install swig
|
||||
fi
|
||||
else # Linux
|
||||
if type -f apt > /dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
ca-certificates \
|
||||
curl
|
||||
else
|
||||
sudo yum update -y
|
||||
sudo yum install -y \
|
||||
ca-certificates \
|
||||
curl
|
||||
fi
|
||||
CMAKE_VERSION="3.30.0"
|
||||
curl -O -L \
|
||||
"https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.sh" \
|
||||
|| exit 1
|
||||
sudo mkdir /opt/cmake || exit 1
|
||||
sudo sh "cmake-${CMAKE_VERSION}-linux-${ARCH}.sh" --skip-license --prefix=/opt/cmake || exit 1
|
||||
sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1
|
||||
|
||||
if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
|
||||
# fixes error "unable to initialize frontend: Dialog"
|
||||
# https://github.com/moby/moby/issues/27988#issuecomment-462809153
|
||||
echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
software-properties-common
|
||||
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
build-essential \
|
||||
git \
|
||||
libcurl4 \
|
||||
libicu-dev \
|
||||
libssl-dev \
|
||||
locales \
|
||||
locales-all || exit 1
|
||||
if [[ $COMPILER == "clang" ]]; then
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
clang \
|
||||
libomp-dev
|
||||
elif [[ $COMPILER == "clang-17" ]]; then
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
wget
|
||||
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
|
||||
sudo apt-add-repository deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
|
||||
sudo apt-add-repository deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
clang-17 \
|
||||
libomp-17-dev
|
||||
fi
|
||||
|
||||
export LANG="en_US.UTF-8"
|
||||
sudo update-locale LANG=${LANG}
|
||||
export LC_ALL="${LANG}"
|
||||
fi
|
||||
if [[ $TASK == "r-package" ]] && [[ $COMPILER == "clang" ]]; then
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
libomp-dev
|
||||
fi
|
||||
if [[ $TASK == "mpi" ]]; then
|
||||
if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
libopenmpi-dev \
|
||||
openmpi-bin
|
||||
else # in manylinux image
|
||||
sudo yum update -y
|
||||
sudo yum install -y \
|
||||
openmpi-devel \
|
||||
|| exit 1
|
||||
fi
|
||||
fi
|
||||
if [[ $TASK == "gpu" ]]; then
|
||||
if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
libboost1.74-dev \
|
||||
libboost-filesystem1.74-dev \
|
||||
ocl-icd-opencl-dev
|
||||
else # in manylinux image
|
||||
sudo yum update -y
|
||||
sudo yum install -y \
|
||||
boost-devel \
|
||||
ocl-icd-devel \
|
||||
opencl-headers \
|
||||
|| exit 1
|
||||
fi
|
||||
fi
|
||||
if [[ $TASK == "gpu" || $TASK == "bdist" ]]; then
|
||||
if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
patchelf \
|
||||
pocl-opencl-icd
|
||||
elif [[ $(uname -m) == "x86_64" ]]; then
|
||||
sudo yum update -y
|
||||
sudo yum install -y \
|
||||
ocl-icd-devel \
|
||||
opencl-headers \
|
||||
|| exit 1
|
||||
fi
|
||||
fi
|
||||
if [[ $TASK == "cuda" ]]; then
|
||||
echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
|
||||
apt-get update
|
||||
APT_INSTALL_PACKAGES=()
|
||||
|
||||
# only re-install NCCL if there wasn't one already pre-installed in the image
|
||||
# (for example, nvidia/cuda 12.9.1 images excluded it)
|
||||
if ! apt list --installed | grep -E 'libnccl\-dev' >/dev/null 2>&1; then
|
||||
echo "libnccl-dev not found, manually installing it"
|
||||
|
||||
# find a compatible libnccl-dev for this CUDA version
|
||||
#
|
||||
# (just 'apt-get install libnccl-dev' can result in mixing across CUDA versions)
|
||||
IFS='.' read -r CUDA_MAJOR CUDA_MINOR _ <<< "${CI_CUDA_VERSION}"
|
||||
NCCL_VERSION=$(
|
||||
apt-cache madison libnccl-dev \
|
||||
| awk -F'|' '{print $2}' \
|
||||
| tr -d ' ' \
|
||||
| grep "+cuda${CUDA_MAJOR}.${CUDA_MINOR}$" \
|
||||
| sort -V \
|
||||
| tail -1
|
||||
)
|
||||
|
||||
APT_INSTALL_PACKAGES+=(
|
||||
"libnccl-dev=${NCCL_VERSION}"
|
||||
"libnccl2=${NCCL_VERSION}"
|
||||
)
|
||||
else
|
||||
echo "libnccl-dev already installed"
|
||||
fi
|
||||
if [[ $COMPILER == "clang" ]]; then
|
||||
APT_INSTALL_PACKAGES+=(
|
||||
clang
|
||||
libomp-dev
|
||||
)
|
||||
fi
|
||||
apt-get install --no-install-recommends -y \
|
||||
"${APT_INSTALL_PACKAGES[@]}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${TASK}" != "cpp-tests" ]] && [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "swig" ]]; then
|
||||
if [[ $SETUP_CONDA != "false" ]]; then
|
||||
curl \
|
||||
-sL \
|
||||
-o miniforge.sh \
|
||||
"https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh"
|
||||
sh miniforge.sh -b -p "${CONDA}"
|
||||
fi
|
||||
conda config --set always_yes yes --set changeps1 no
|
||||
conda update -q -y conda
|
||||
|
||||
# print output of 'conda info', to help in submitting bug reports
|
||||
echo "conda info:"
|
||||
conda info
|
||||
fi
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
echo "installing lightgbm and its dependencies"
|
||||
pip install \
|
||||
--prefer-binary \
|
||||
--upgrade \
|
||||
-r ./.ci/pip-envs/requirements-latest.txt \
|
||||
dist/*.whl
|
||||
|
||||
echo "installed package versions:"
|
||||
pip freeze
|
||||
|
||||
echo ""
|
||||
echo "running tests"
|
||||
pytest tests/c_api_test/
|
||||
pytest tests/python_package_test/
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
echo "installing lightgbm and its dependencies"
|
||||
pip install \
|
||||
--prefer-binary \
|
||||
--upgrade \
|
||||
--constraint ./.ci/pip-envs/requirements-oldest.txt \
|
||||
"$(echo dist/*.whl)[arrow,pandas,scikit-learn]"
|
||||
|
||||
echo "installed package versions:"
|
||||
pip freeze
|
||||
|
||||
echo ""
|
||||
echo "checking that examples run without error"
|
||||
|
||||
# run a few examples to test that Python-package minimally works
|
||||
echo ""
|
||||
echo "--- advanced_example.py ---"
|
||||
echo ""
|
||||
python ./examples/python-guide/advanced_example.py || exit 1
|
||||
|
||||
echo ""
|
||||
echo "--- logistic_regression.py ---"
|
||||
echo ""
|
||||
python ./examples/python-guide/logistic_regression.py || exit 1
|
||||
|
||||
echo ""
|
||||
echo "--- simple_example.py ---"
|
||||
echo ""
|
||||
python ./examples/python-guide/simple_example.py || exit 1
|
||||
|
||||
echo ""
|
||||
echo "--- sklearn_example.py ---"
|
||||
echo ""
|
||||
python ./examples/python-guide/sklearn_example.py || exit 1
|
||||
echo ""
|
||||
|
||||
echo "checking that imports work without any optional dependencies"
|
||||
pip uninstall --yes \
|
||||
cffi \
|
||||
dask \
|
||||
distributed \
|
||||
graphviz \
|
||||
joblib \
|
||||
matplotlib \
|
||||
pandas \
|
||||
psutil \
|
||||
pyarrow \
|
||||
scikit-learn
|
||||
|
||||
python -c "import lightgbm"
|
||||
|
||||
echo ""
|
||||
echo "done testing on oldest supported Python version"
|
||||
Executable
+100
@@ -0,0 +1,100 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
RDscriptvalgrind ./.ci/install-r-deps.R --test || exit 1
|
||||
sh build-cran-package.sh \
|
||||
--r-executable=RDvalgrind \
|
||||
--no-build-vignettes \
|
||||
|| exit 1
|
||||
|
||||
RDvalgrind CMD INSTALL --preclean --install-tests lightgbm_*.tar.gz || exit 1
|
||||
|
||||
cd R-package/tests
|
||||
|
||||
ALL_LOGS_FILE="out.log"
|
||||
VALGRIND_LOGS_FILE="valgrind-logs.log"
|
||||
|
||||
RDvalgrind \
|
||||
--no-readline \
|
||||
--vanilla \
|
||||
-d "valgrind --tool=memcheck --leak-check=full --track-origins=yes --gen-suppressions=all" \
|
||||
-f testthat.R \
|
||||
> ${ALL_LOGS_FILE} 2>&1 || exit 1
|
||||
|
||||
cat ${ALL_LOGS_FILE}
|
||||
|
||||
echo "writing valgrind output to ${VALGRIND_LOGS_FILE}"
|
||||
cat ${ALL_LOGS_FILE} | grep -E "^\=" > ${VALGRIND_LOGS_FILE}
|
||||
|
||||
bytes_definitely_lost=$(
|
||||
cat ${VALGRIND_LOGS_FILE} \
|
||||
| grep -E "definitely lost\: .*" \
|
||||
| sed 's/^.*definitely lost\: \(.*\) bytes.*$/\1/' \
|
||||
| tr -d ","
|
||||
)
|
||||
echo "valgrind found ${bytes_definitely_lost} bytes definitely lost"
|
||||
if [[ ${bytes_definitely_lost} -gt 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bytes_indirectly_lost=$(
|
||||
cat ${VALGRIND_LOGS_FILE} \
|
||||
| grep -E "indirectly lost\: .*" \
|
||||
| sed 's/^.*indirectly lost\: \(.*\) bytes.*$/\1/' \
|
||||
| tr -d ","
|
||||
)
|
||||
echo "valgrind found ${bytes_indirectly_lost} bytes indirectly lost"
|
||||
if [[ ${bytes_indirectly_lost} -gt 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# one error caused by a false positive between valgrind and openmp is allowed
|
||||
# ==2063== 352 bytes in 1 blocks are possibly lost in loss record 153 of 2,709
|
||||
# ==2063== at 0x483DD99: calloc (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_memcheck-amd64-linux.so)
|
||||
# ==2063== by 0x40149CA: allocate_dtv (dl-tls.c:286)
|
||||
# ==2063== by 0x40149CA: _dl_allocate_tls (dl-tls.c:532)
|
||||
# ==2063== by 0x5702322: allocate_stack (allocatestack.c:622)
|
||||
# ==2063== by 0x5702322: pthread_create@@GLIBC_2.2.5 (pthread_create.c:660)
|
||||
# ==2063== by 0x56D0DDA: ??? (in /usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0)
|
||||
# ==2063== by 0x56C88E0: GOMP_parallel (in /usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0)
|
||||
# ==2063== by 0x1544D29C: LGBM_DatasetCreateFromCSC (c_api.cpp:1286)
|
||||
# ==2063== by 0x1546F980: LGBM_DatasetCreateFromCSC_R (lightgbm_R.cpp:91)
|
||||
# ==2063== by 0x4941E2F: R_doDotCall (dotcode.c:634)
|
||||
# ==2063== by 0x494CCC6: do_dotcall (dotcode.c:1281)
|
||||
# ==2063== by 0x499FB01: bcEval (eval.c:7078)
|
||||
# ==2063== by 0x498B67F: Rf_eval (eval.c:727)
|
||||
# ==2063== by 0x498E414: R_execClosure (eval.c:1895)
|
||||
bytes_possibly_lost=$(
|
||||
cat ${VALGRIND_LOGS_FILE} \
|
||||
| grep -E "possibly lost\: .*" \
|
||||
| sed 's/^.*possibly lost\: \(.*\) bytes.*$/\1/' \
|
||||
| tr -d ","
|
||||
)
|
||||
echo "valgrind found ${bytes_possibly_lost} bytes possibly lost"
|
||||
if [[ ${bytes_possibly_lost} -gt 1104 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ensure 'grep --count' doesn't cause failures
|
||||
set +e
|
||||
|
||||
echo "checking for invalid reads"
|
||||
invalid_reads=$(
|
||||
cat ${VALGRIND_LOGS_FILE} \
|
||||
| grep --count -i "Invalid read"
|
||||
)
|
||||
if [[ ${invalid_reads} -gt 0 ]]; then
|
||||
echo "valgrind found invalid reads: ${invalid_reads}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "checking for invalid writes"
|
||||
invalid_writes=$(
|
||||
cat ${VALGRIND_LOGS_FILE} \
|
||||
| grep --count -i "Invalid write"
|
||||
)
|
||||
if [[ ${invalid_writes} -gt 0 ]]; then
|
||||
echo "valgrind found invalid writes: ${invalid_writes}"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,337 @@
|
||||
# Download a file and retry upon failure. This looks like
|
||||
# an infinite loop but CI-level timeouts will kill it
|
||||
function Get-File-With-Tenacity {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$url,
|
||||
[Parameter(Mandatory = $true)][string]$destfile
|
||||
)
|
||||
$ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows down download speed
|
||||
do {
|
||||
Write-Output "Downloading ${url}"
|
||||
sleep 5
|
||||
Invoke-WebRequest -Uri $url -OutFile $destfile
|
||||
} while (-not $?)
|
||||
}
|
||||
|
||||
# External utilities like R.exe / Rscript.exe writing to stderr (even for harmless
|
||||
# status information) can cause failures in GitHub Actions PowerShell jobs.
|
||||
# See https://github.community/t/powershell-steps-fail-nondeterministically/115496
|
||||
#
|
||||
# Using standard PowerShell redirection does not work to avoid these errors.
|
||||
# This function uses R's built-in redirection mechanism, sink(). Any place where
|
||||
# this function is used is a command that writes harmless messages to stderr
|
||||
function Invoke-R-Code-Redirect-Stderr {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$rcode
|
||||
)
|
||||
$decorated_code = "out_file <- file(tempfile(), open = 'wt'); sink(out_file, type = 'message'); $rcode; sink()"
|
||||
Rscript --vanilla -e $decorated_code
|
||||
}
|
||||
|
||||
# Remove all items matching some pattern from PATH environment variable
|
||||
function Remove-From-Path {
|
||||
[CmdletBinding(SupportsShouldProcess)]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$pattern_to_remove
|
||||
)
|
||||
if ($PSCmdlet.ShouldProcess($env:PATH, "Removing ${pattern_to_remove}")) {
|
||||
$env:PATH = ($env:PATH.Split(';') | Where-Object { $_ -notmatch "$pattern_to_remove" }) -join ';'
|
||||
}
|
||||
}
|
||||
|
||||
# remove some details that exist in the GitHub Actions images which might
|
||||
# cause conflicts with R and other components installed by this script
|
||||
$env:RTOOLS40_HOME = ""
|
||||
Remove-From-Path ".*Amazon.*"
|
||||
Remove-From-Path ".*Anaconda.*"
|
||||
Remove-From-Path ".*android.*"
|
||||
Remove-From-Path ".*Android.*"
|
||||
Remove-From-Path ".*chocolatey.*"
|
||||
Remove-From-Path ".*Chocolatey.*"
|
||||
Remove-From-Path ".*cmake.*"
|
||||
Remove-From-Path ".*CMake.*"
|
||||
Remove-From-Path ".*\\Git\\.*"
|
||||
Remove-From-Path "(?!.*pandoc.*).*hostedtoolcache.*"
|
||||
Remove-From-Path ".*Microsoft SDKs.*"
|
||||
Remove-From-Path ".*mingw.*"
|
||||
Remove-From-Path ".*msys64.*"
|
||||
Remove-From-Path ".*PostgreSQL.*"
|
||||
Remove-From-Path ".*\\R\\.*"
|
||||
Remove-From-Path ".*R Client.*"
|
||||
Remove-From-Path ".*rtools40.*"
|
||||
Remove-From-Path ".*rtools42.*"
|
||||
Remove-From-Path ".*rtools43.*"
|
||||
Remove-From-Path ".*rtools44.*"
|
||||
Remove-From-Path ".*rtools45.*"
|
||||
Remove-From-Path ".*shells.*"
|
||||
Remove-From-Path ".*Strawberry.*"
|
||||
Remove-From-Path ".*tools.*"
|
||||
|
||||
Remove-Item C:\rtools40 -Force -Recurse -ErrorAction Ignore
|
||||
Remove-Item C:\rtools42 -Force -Recurse -ErrorAction Ignore
|
||||
Remove-Item C:\rtools43 -Force -Recurse -ErrorAction Ignore
|
||||
Remove-Item C:\rtools44 -Force -Recurse -ErrorAction Ignore
|
||||
Remove-Item C:\rtools45 -Force -Recurse -ErrorAction Ignore
|
||||
|
||||
# Get details needed for installing R components
|
||||
#
|
||||
# NOTES:
|
||||
# * some paths and file names are different on R4.0
|
||||
$env:R_MAJOR_VERSION = $env:R_VERSION.split('.')[0]
|
||||
if ($env:R_MAJOR_VERSION -eq "4") {
|
||||
$RTOOLS_INSTALL_PATH = "C:\rtools43"
|
||||
$env:RTOOLS_BIN = "$RTOOLS_INSTALL_PATH\usr\bin"
|
||||
$env:RTOOLS_MINGW_BIN = "$RTOOLS_INSTALL_PATH\x86_64-w64-mingw32.static.posix\bin"
|
||||
$env:RTOOLS_EXE_FILE = "rtools43-5550-5548.exe"
|
||||
$env:R_WINDOWS_VERSION = "4.3.1"
|
||||
} else {
|
||||
Write-Output "[ERROR] Unrecognized R version: $env:R_VERSION"
|
||||
Assert-Output $false
|
||||
}
|
||||
$env:CMAKE_VERSION = "3.30.0"
|
||||
|
||||
$env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/'
|
||||
$env:R_LIBS = "$env:R_LIB_PATH"
|
||||
$env:CMAKE_PATH = "$env:BUILD_SOURCESDIRECTORY/CMake_installation"
|
||||
$env:PATH = @(
|
||||
"$env:RTOOLS_BIN",
|
||||
"$env:RTOOLS_MINGW_BIN",
|
||||
"$env:R_LIB_PATH/R/bin/x64",
|
||||
"$env:CMAKE_PATH/cmake-$env:CMAKE_VERSION-windows-x86_64/bin",
|
||||
"$env:PATH"
|
||||
) -join ";"
|
||||
$env:CRAN_MIRROR = "https://cran.rstudio.com"
|
||||
$env:MIKTEX_EXCEPTION_PATH = "$env:TEMP\miktex"
|
||||
|
||||
# don't fail builds for long-running examples unless they're very long.
|
||||
# See https://github.com/lightgbm-org/LightGBM/issues/4049#issuecomment-793412254.
|
||||
if ($env:R_BUILD_TYPE -ne "cran") {
|
||||
$env:_R_CHECK_EXAMPLE_TIMING_THRESHOLD_ = 30
|
||||
}
|
||||
|
||||
if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
|
||||
$env:CXX = "$env:RTOOLS_MINGW_BIN/g++.exe"
|
||||
$env:CC = "$env:RTOOLS_MINGW_BIN/gcc.exe"
|
||||
}
|
||||
|
||||
Set-Location "$env:BUILD_SOURCESDIRECTORY"
|
||||
tzutil /s "GMT Standard Time"
|
||||
[Void][System.IO.Directory]::CreateDirectory("$env:R_LIB_PATH")
|
||||
[Void][System.IO.Directory]::CreateDirectory("$env:CMAKE_PATH")
|
||||
|
||||
# download R, RTools and CMake
|
||||
Write-Output "Downloading R, Rtools and CMake"
|
||||
$params = @{
|
||||
url = "$env:CRAN_MIRROR/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe"
|
||||
destfile = "R-win.exe"
|
||||
}
|
||||
Get-File-With-Tenacity @params
|
||||
|
||||
$params = @{
|
||||
url = "https://github.com/lightgbm-org/LightGBM/releases/download/v2.0.12/$env:RTOOLS_EXE_FILE"
|
||||
destfile = "Rtools.exe"
|
||||
}
|
||||
Get-File-With-Tenacity @params
|
||||
|
||||
$params = @{
|
||||
url = "https://github.com/Kitware/CMake/releases/download/v{0}/cmake-{0}-windows-x86_64.zip" -f $env:CMAKE_VERSION
|
||||
destfile = "$env:CMAKE_PATH/cmake.zip"
|
||||
}
|
||||
Get-File-With-Tenacity @params
|
||||
|
||||
# Install R
|
||||
Write-Output "Installing R"
|
||||
$params = @{
|
||||
FilePath = "R-win.exe"
|
||||
NoNewWindow = $true
|
||||
Wait = $true
|
||||
ArgumentList = "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64,i386"
|
||||
}
|
||||
Start-Process @params ; Assert-Output $?
|
||||
Write-Output "Done installing R"
|
||||
|
||||
Write-Output "Installing Rtools"
|
||||
$params = @{
|
||||
FilePath = "Rtools.exe"
|
||||
NoNewWindow = $true
|
||||
Wait = $true
|
||||
ArgumentList = "/VERYSILENT /SUPPRESSMSGBOXES /DIR=$RTOOLS_INSTALL_PATH"
|
||||
}
|
||||
Start-Process @params; Assert-Output $?
|
||||
Write-Output "Done installing Rtools"
|
||||
|
||||
Write-Output "Installing CMake"
|
||||
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
||||
[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:CMAKE_PATH/cmake.zip", "$env:CMAKE_PATH") ; Assert-Output $?
|
||||
# Remove old CMake shipped with RTools
|
||||
Remove-Item "$env:RTOOLS_MINGW_BIN/cmake.exe" -Force -ErrorAction Ignore
|
||||
Write-Output "Done installing CMake"
|
||||
|
||||
Write-Output "Installing dependencies"
|
||||
Rscript.exe --vanilla ".ci/install-r-deps.R" --build --include=processx --test ; Assert-Output $?
|
||||
|
||||
Write-Output "Building R-package"
|
||||
|
||||
# R CMD check is not used for MSVC builds
|
||||
if ($env:COMPILER -ne "MSVC") {
|
||||
|
||||
$PKG_FILE_NAME = "lightgbm_$env:LGB_VER.tar.gz"
|
||||
$LOG_FILE_NAME = "lightgbm.Rcheck/00check.log"
|
||||
|
||||
if ($env:R_BUILD_TYPE -eq "cmake") {
|
||||
if ($env:TOOLCHAIN -eq "MINGW") {
|
||||
Write-Output "Telling R to use MinGW"
|
||||
$env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw', '-j4')"
|
||||
} elseif ($env:TOOLCHAIN -eq "MSYS") {
|
||||
Write-Output "Telling R to use MSYS"
|
||||
$env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2', '-j4')"
|
||||
} elseif ($env:TOOLCHAIN -eq "MSVC") {
|
||||
$env:BUILD_R_FLAGS = "'--skip-install'"
|
||||
} else {
|
||||
Write-Output "[ERROR] Unrecognized toolchain: $env:TOOLCHAIN"
|
||||
Assert-Output $false
|
||||
}
|
||||
Invoke-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"
|
||||
Assert-Output $?
|
||||
} elseif ($env:R_BUILD_TYPE -eq "cran") {
|
||||
$params = -join @(
|
||||
"result <- processx::run(command = 'sh', args = 'build-cran-package.sh', ",
|
||||
"echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)"
|
||||
)
|
||||
Invoke-R-Code-Redirect-Stderr $params ; Assert-Output $?
|
||||
Remove-From-Path ".*msys64.*"
|
||||
# Test CRAN source .tar.gz in a directory that is not this repo or below it.
|
||||
# When people install.packages('lightgbm'), they won't have the LightGBM
|
||||
# git repo around. This is to protect against the use of relative paths
|
||||
# like ../../CMakeLists.txt that would only work if you are in the repoo
|
||||
$R_CMD_CHECK_DIR = "tmp-r-cmd-check"
|
||||
New-Item -Path "C:\" -Name $R_CMD_CHECK_DIR -ItemType "directory" > $null
|
||||
Move-Item -Path "$PKG_FILE_NAME" -Destination "C:\$R_CMD_CHECK_DIR\" > $null
|
||||
Set-Location "C:\$R_CMD_CHECK_DIR\"
|
||||
}
|
||||
|
||||
Write-Output "Running R CMD check"
|
||||
if ($env:R_BUILD_TYPE -eq "cran") {
|
||||
# CRAN packages must pass without --no-multiarch (build on 64-bit and 32-bit)
|
||||
$check_args = "c('CMD', 'check', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
|
||||
} else {
|
||||
$check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
|
||||
}
|
||||
$params = -join (
|
||||
"result <- processx::run(command = 'R.exe', args = $check_args, ",
|
||||
"echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)"
|
||||
)
|
||||
Invoke-R-Code-Redirect-Stderr $params ; $check_succeeded = $?
|
||||
|
||||
Write-Output "R CMD check build logs:"
|
||||
$INSTALL_LOG_FILE_NAME = "lightgbm.Rcheck\00install.out"
|
||||
Get-Content -Path "$INSTALL_LOG_FILE_NAME"
|
||||
|
||||
Assert-Output $check_succeeded
|
||||
|
||||
Write-Output "Looking for issues with R CMD check results"
|
||||
if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "NOTE|WARNING|ERROR" -CaseSensitive -Quiet) {
|
||||
Write-Output "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
|
||||
Assert-Output $False
|
||||
}
|
||||
} else {
|
||||
$INSTALL_LOG_FILE_NAME = "$env:BUILD_SOURCESDIRECTORY\00install_out.txt"
|
||||
Invoke-R-Code-Redirect-Stderr "source('build_r.R')" 1> $INSTALL_LOG_FILE_NAME ; $install_succeeded = $?
|
||||
Write-Output "----- build and install logs -----"
|
||||
Get-Content -Path "$INSTALL_LOG_FILE_NAME"
|
||||
Write-Output "----- end of build and install logs -----"
|
||||
Assert-Output $install_succeeded
|
||||
# some errors are not raised above, but can be found in the logs
|
||||
if (Get-Content "$INSTALL_LOG_FILE_NAME" | Select-String -Pattern "ERROR" -CaseSensitive -Quiet) {
|
||||
Write-Output "ERRORs have been found installing lightgbm"
|
||||
Assert-Output $False
|
||||
}
|
||||
}
|
||||
|
||||
# Checking that the correct R version was used
|
||||
if ($env:TOOLCHAIN -ne "MSVC") {
|
||||
$checks = Select-String -Path "${LOG_FILE_NAME}" -Pattern "using R version $env:R_WINDOWS_VERSION"
|
||||
$checks_cnt = $checks.Matches.length
|
||||
} else {
|
||||
$checksParams = @{
|
||||
Path = "${INSTALL_LOG_FILE_NAME}"
|
||||
Pattern = "R version passed into FindLibR.* $env:R_WINDOWS_VERSION"
|
||||
}
|
||||
$checks = Select-String @checksParams
|
||||
$checks_cnt = $checks.Matches.length
|
||||
}
|
||||
if ($checks_cnt -eq 0) {
|
||||
Write-Output "Wrong R version was found (expected '$env:R_WINDOWS_VERSION'). Check the build logs."
|
||||
Assert-Output $False
|
||||
}
|
||||
|
||||
# Checking that we actually got the expected compiler. The R-package has some logic
|
||||
# to fail back to MinGW if MSVC fails, but for CI builds we need to check that the correct
|
||||
# compiler was used.
|
||||
if ($env:R_BUILD_TYPE -eq "cmake") {
|
||||
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Check for working CXX compiler.*$env:COMPILER"
|
||||
if ($checks.Matches.length -eq 0) {
|
||||
Write-Output "The wrong compiler was used. Check the build logs."
|
||||
Assert-Output $False
|
||||
}
|
||||
}
|
||||
|
||||
# Checking that we got the right toolchain for MinGW. If using MinGW, both
|
||||
# MinGW and MSYS toolchains are supported
|
||||
if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
|
||||
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Trying to build with.*$env:TOOLCHAIN"
|
||||
if ($checks.Matches.length -eq 0) {
|
||||
Write-Output "The wrong toolchain was used. Check the build logs."
|
||||
Assert-Output $False
|
||||
}
|
||||
}
|
||||
|
||||
# Checking that MM_PREFETCH preprocessor definition is actually used in CI builds.
|
||||
if ($env:R_BUILD_TYPE -eq "cran") {
|
||||
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "checking whether MM_PREFETCH work.*yes"
|
||||
$checks_cnt = $checks.Matches.length
|
||||
} elseif ($env:TOOLCHAIN -ne "MSVC") {
|
||||
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Performing Test MM_PREFETCH - Success"
|
||||
$checks_cnt = $checks.Matches.length
|
||||
} else {
|
||||
$checks_cnt = 1
|
||||
}
|
||||
if ($checks_cnt -eq 0) {
|
||||
Write-Output "MM_PREFETCH preprocessor definition wasn't used. Check the build logs."
|
||||
Assert-Output $False
|
||||
}
|
||||
|
||||
# Checking that MM_MALLOC preprocessor definition is actually used in CI builds.
|
||||
if ($env:R_BUILD_TYPE -eq "cran") {
|
||||
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "checking whether MM_MALLOC work.*yes"
|
||||
$checks_cnt = $checks.Matches.length
|
||||
} elseif ($env:TOOLCHAIN -ne "MSVC") {
|
||||
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Performing Test MM_MALLOC - Success"
|
||||
$checks_cnt = $checks.Matches.length
|
||||
} else {
|
||||
$checks_cnt = 1
|
||||
}
|
||||
if ($checks_cnt -eq 0) {
|
||||
Write-Output "MM_MALLOC preprocessor definition wasn't used. Check the build logs."
|
||||
Assert-Output $False
|
||||
}
|
||||
|
||||
# Checking that OpenMP is actually used in CMake builds.
|
||||
if ($env:R_BUILD_TYPE -eq "cmake") {
|
||||
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Found OpenMP: TRUE.*"
|
||||
if ($checks.Matches.length -eq 0) {
|
||||
Write-Output "OpenMP wasn't found. Check the build logs."
|
||||
Assert-Output $False
|
||||
}
|
||||
}
|
||||
|
||||
if ($env:COMPILER -eq "MSVC") {
|
||||
Write-Output "Running tests with testthat.R"
|
||||
Set-Location R-package/tests
|
||||
# NOTE: using Rscript.exe intentionally here, instead of Invoke-R-Code-Redirect-Stderr,
|
||||
# because something about the interaction between Invoke-R-Code-Redirect-Stderr
|
||||
# and testthat results in failing tests not exiting with a non-0 exit code.
|
||||
Rscript.exe --vanilla "testthat.R" ; Assert-Output $?
|
||||
}
|
||||
|
||||
Write-Output "No issues were found checking the R-package"
|
||||
Executable
+260
@@ -0,0 +1,260 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
# defaults
|
||||
ARCH=$(uname -m)
|
||||
|
||||
# set up R environment
|
||||
export CRAN_MIRROR="https://cran.rstudio.com"
|
||||
export R_LIB_PATH=~/Rlib
|
||||
mkdir -p $R_LIB_PATH
|
||||
export R_LIBS=$R_LIB_PATH
|
||||
export PATH="$R_LIB_PATH/R/bin:$PATH"
|
||||
|
||||
# don't fail builds for long-running examples unless they're very long.
|
||||
# See https://github.com/lightgbm-org/LightGBM/issues/4049#issuecomment-793412254.
|
||||
if [[ $R_BUILD_TYPE != "cran" ]]; then
|
||||
export _R_CHECK_EXAMPLE_TIMING_THRESHOLD_=30
|
||||
fi
|
||||
|
||||
# Get details needed for installing R components
|
||||
R_MAJOR_VERSION="${R_VERSION%.*}"
|
||||
if [[ "${R_MAJOR_VERSION}" == "4" ]]; then
|
||||
export R_MAC_VERSION=4.3.1
|
||||
export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/big-sur-${ARCH}/base/R-${R_MAC_VERSION}-${ARCH}.pkg
|
||||
export R_LINUX_VERSION="4.3.1-1.2204.0"
|
||||
export R_APT_REPO="jammy-cran40/"
|
||||
else
|
||||
echo "Unrecognized R version: ${R_VERSION}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# installing precompiled R for Ubuntu
|
||||
# https://cran.r-project.org/bin/linux/ubuntu/#installation
|
||||
# adding steps from https://stackoverflow.com/a/56378217/3986677 to get latest version
|
||||
#
|
||||
# `devscripts` is required for 'checkbashisms' (https://github.com/r-lib/actions/issues/111)
|
||||
if [[ $OS_NAME == "linux" ]]; then
|
||||
mkdir -p ~/.gnupg
|
||||
echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf
|
||||
sudo apt-key adv \
|
||||
--homedir ~/.gnupg \
|
||||
--keyserver keyserver.ubuntu.com \
|
||||
--recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 || exit 1
|
||||
sudo add-apt-repository \
|
||||
"deb ${CRAN_MIRROR}/bin/linux/ubuntu ${R_APT_REPO}" || exit 1
|
||||
sudo apt-get update
|
||||
sudo apt-get install \
|
||||
--no-install-recommends \
|
||||
-y \
|
||||
devscripts \
|
||||
libuv1-dev \
|
||||
r-base-core=${R_LINUX_VERSION} \
|
||||
r-base-dev=${R_LINUX_VERSION} \
|
||||
texinfo \
|
||||
texlive-latex-extra \
|
||||
texlive-latex-recommended \
|
||||
texlive-fonts-recommended \
|
||||
texlive-fonts-extra \
|
||||
tidy \
|
||||
qpdf \
|
||||
|| exit 1
|
||||
|
||||
if [[ $R_BUILD_TYPE == "cran" ]]; then
|
||||
sudo apt-get install \
|
||||
--no-install-recommends \
|
||||
-y \
|
||||
"autoconf=$(cat R-package/AUTOCONF_UBUNTU_VERSION)" \
|
||||
automake \
|
||||
|| exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Installing R precompiled for Mac OS 10.11 or higher
|
||||
if [[ $OS_NAME == "macos" ]]; then
|
||||
brew update-reset --auto-update
|
||||
brew update --auto-update
|
||||
if [[ $R_BUILD_TYPE == "cran" ]]; then
|
||||
brew install automake || exit 1
|
||||
fi
|
||||
brew install \
|
||||
checkbashisms \
|
||||
qpdf || exit 1
|
||||
brew install basictex || exit 1
|
||||
export PATH="/Library/TeX/texbin:$PATH"
|
||||
sudo tlmgr --verify-repo=none update --self || exit 1
|
||||
sudo tlmgr --verify-repo=none install inconsolata helvetic rsfs || exit 1
|
||||
|
||||
curl -sL "${R_MAC_PKG_URL}" -o R.pkg || exit 1
|
||||
sudo installer \
|
||||
-pkg "$(pwd)/R.pkg" \
|
||||
-target / || exit 1
|
||||
|
||||
# install tidy v5.8.0
|
||||
# ref: https://groups.google.com/g/r-sig-mac/c/7u_ivEj4zhM
|
||||
TIDY_URL=https://github.com/htacg/tidy-html5/releases/download/5.8.0/tidy-5.8.0-macos-x86_64+arm64.pkg
|
||||
curl -sL ${TIDY_URL} -o tidy.pkg
|
||||
sudo installer \
|
||||
-pkg "$(pwd)/tidy.pkg" \
|
||||
-target /
|
||||
|
||||
# ensure that this newer version of 'tidy' is used by 'R CMD check'
|
||||
# ref: https://cran.r-project.org/doc/manuals/R-exts.html#Checking-packages
|
||||
export R_TIDYCMD=/usr/local/bin/tidy
|
||||
fi
|
||||
|
||||
# {Matrix} needs {lattice}, so this needs to run before manually installing {Matrix}.
|
||||
# This should be unnecessary on R >=4.4.0
|
||||
# ref: https://github.com/lightgbm-org/LightGBM/issues/6433
|
||||
Rscript --vanilla -e "install.packages('lattice', repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}')"
|
||||
|
||||
# manually install {Matrix}, as {Matrix}=1.7-0 raised its R floor all the way to R 4.4.0
|
||||
# ref: https://github.com/lightgbm-org/LightGBM/issues/6433
|
||||
Rscript --vanilla -e "install.packages('https://cran.r-project.org/src/contrib/Archive/Matrix/Matrix_1.6-5.tar.gz', repos = NULL, lib = '${R_LIB_PATH}')"
|
||||
|
||||
# Manually install dependencies to avoid a CI-time dependency on devtools (for devtools::install_deps())
|
||||
Rscript --vanilla ./.ci/install-r-deps.R --build --test --exclude=Matrix || exit 1
|
||||
|
||||
cd "${BUILD_DIRECTORY}"
|
||||
PKG_TARBALL="lightgbm_$(head -1 VERSION.txt).tar.gz"
|
||||
BUILD_LOG_FILE="lightgbm.Rcheck/00install.out"
|
||||
LOG_FILE_NAME="lightgbm.Rcheck/00check.log"
|
||||
if [[ $R_BUILD_TYPE == "cmake" ]]; then
|
||||
Rscript build_r.R -j4 --skip-install || exit 1
|
||||
elif [[ $R_BUILD_TYPE == "cran" ]]; then
|
||||
|
||||
# on Linux, we recreate configure in CI to test if
|
||||
# a change in a PR has changed configure.ac
|
||||
if [[ $OS_NAME == "linux" ]]; then
|
||||
./R-package/recreate-configure.sh
|
||||
|
||||
num_files_changed=$(
|
||||
git diff --name-only | wc -l
|
||||
)
|
||||
if [[ ${num_files_changed} -gt 0 ]]; then
|
||||
echo "'configure' in the R-package has changed. Please recreate it and commit the changes."
|
||||
echo "Changed files:"
|
||||
git diff --compact-summary
|
||||
echo "See R-package/README.md for details on how to recreate this script."
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
./build-cran-package.sh || exit 1
|
||||
|
||||
# Test CRAN source .tar.gz in a directory that is not this repo or below it.
|
||||
# When people install.packages('lightgbm'), they won't have the LightGBM
|
||||
# git repo around. This is to protect against the use of relative paths
|
||||
# like ../../CMakeLists.txt that would only work if you are in the repo
|
||||
R_CMD_CHECK_DIR="${HOME}/tmp-r-cmd-check/"
|
||||
mkdir -p "${R_CMD_CHECK_DIR}"
|
||||
mv "${PKG_TARBALL}" "${R_CMD_CHECK_DIR}"
|
||||
cd "${R_CMD_CHECK_DIR}"
|
||||
fi
|
||||
|
||||
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
|
||||
cp "${PKG_TARBALL}" "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbm-${LGB_VER}-r-cran.tar.gz"
|
||||
fi
|
||||
|
||||
declare -i allowed_notes=0
|
||||
bash "${BUILD_DIRECTORY}/.ci/run-r-cmd-check.sh" \
|
||||
"${PKG_TARBALL}" \
|
||||
"${allowed_notes}"
|
||||
|
||||
# ensure 'grep --count' doesn't cause failures
|
||||
set +e
|
||||
|
||||
used_correct_r_version=$(
|
||||
cat $LOG_FILE_NAME \
|
||||
| grep --count "using R version ${R_VERSION}"
|
||||
)
|
||||
if [[ $used_correct_r_version -ne 1 ]]; then
|
||||
echo "Unexpected R version was used. Expected '${R_VERSION}'."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $R_BUILD_TYPE == "cmake" ]]; then
|
||||
passed_correct_r_version_to_cmake=$(
|
||||
cat $BUILD_LOG_FILE \
|
||||
| grep --count "R version passed into FindLibR.cmake: ${R_VERSION}"
|
||||
)
|
||||
if [[ $passed_correct_r_version_to_cmake -ne 1 ]]; then
|
||||
echo "Unexpected R version was passed into cmake. Expected '${R_VERSION}'."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# this check makes sure that CI builds of the package actually use OpenMP
|
||||
if [[ $OS_NAME == "macos" ]] && [[ $R_BUILD_TYPE == "cran" ]]; then
|
||||
omp_working=$(
|
||||
cat $BUILD_LOG_FILE \
|
||||
| grep --count -E "checking whether OpenMP will work .*yes"
|
||||
)
|
||||
elif [[ $R_BUILD_TYPE == "cmake" ]]; then
|
||||
omp_working=$(
|
||||
cat $BUILD_LOG_FILE \
|
||||
| grep --count -E ".*Found OpenMP: TRUE.*"
|
||||
)
|
||||
else
|
||||
omp_working=1
|
||||
fi
|
||||
if [[ $omp_working -ne 1 ]]; then
|
||||
echo "OpenMP was not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# this check makes sure that CI builds of the package
|
||||
# actually use MM_PREFETCH preprocessor definition
|
||||
#
|
||||
# _mm_prefetch will not work on arm64 architecture
|
||||
# ref: https://github.com/lightgbm-org/LightGBM/issues/4124
|
||||
if [[ $ARCH != "arm64" ]]; then
|
||||
if [[ $R_BUILD_TYPE == "cran" ]]; then
|
||||
mm_prefetch_working=$(
|
||||
cat $BUILD_LOG_FILE \
|
||||
| grep --count -E "checking whether MM_PREFETCH work.*yes"
|
||||
)
|
||||
else
|
||||
mm_prefetch_working=$(
|
||||
cat $BUILD_LOG_FILE \
|
||||
| grep --count -E ".*Performing Test MM_PREFETCH - Success"
|
||||
)
|
||||
fi
|
||||
if [[ $mm_prefetch_working -ne 1 ]]; then
|
||||
echo "MM_PREFETCH test was not passed"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# this check makes sure that CI builds of the package
|
||||
# actually use MM_MALLOC preprocessor definition
|
||||
if [[ $R_BUILD_TYPE == "cran" ]]; then
|
||||
mm_malloc_working=$(
|
||||
cat $BUILD_LOG_FILE \
|
||||
| grep --count -E "checking whether MM_MALLOC work.*yes"
|
||||
)
|
||||
else
|
||||
mm_malloc_working=$(
|
||||
cat $BUILD_LOG_FILE \
|
||||
| grep --count -E ".*Performing Test MM_MALLOC - Success"
|
||||
)
|
||||
fi
|
||||
if [[ $mm_malloc_working -ne 1 ]]; then
|
||||
echo "MM_MALLOC test was not passed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# this check makes sure that no "warning: unknown pragma ignored" logs
|
||||
# reach the user leading them to believe that something went wrong
|
||||
if [[ $R_BUILD_TYPE == "cran" ]]; then
|
||||
pragma_warning_present=$(
|
||||
cat $BUILD_LOG_FILE \
|
||||
| grep --count -E "warning: unknown pragma ignored"
|
||||
)
|
||||
if [[ $pragma_warning_present -ne 0 ]]; then
|
||||
echo "Unknown pragma warning is present, pragmas should have been removed before build"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,190 @@
|
||||
function Assert-Output {
|
||||
param( [Parameter(Mandatory = $true)][bool]$success )
|
||||
if (-not $success) {
|
||||
$host.SetShouldExit(-1)
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
$env:LGB_VER = (Get-Content $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim()
|
||||
# Use custom temp directory to avoid
|
||||
# > warning MSB8029: The Intermediate directory or Output directory cannot reside under the Temporary directory
|
||||
# > as it could lead to issues with incremental build.
|
||||
# And make sure this directory is always clean
|
||||
$env:TMPDIR = "$env:USERPROFILE\tmp"
|
||||
Remove-Item $env:TMPDIR -Force -Recurse -ErrorAction Ignore
|
||||
[Void][System.IO.Directory]::CreateDirectory($env:TMPDIR)
|
||||
|
||||
# create the artifact upload directory if it doesn't exist yet
|
||||
[Void][System.IO.Directory]::CreateDirectory($env:BUILD_ARTIFACTSTAGINGDIRECTORY)
|
||||
|
||||
if ($env:TASK -eq "r-package") {
|
||||
& .\.ci\test-r-package-windows.ps1 ; Assert-Output $?
|
||||
exit 0
|
||||
}
|
||||
|
||||
if ($env:TASK -eq "cpp-tests") {
|
||||
cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_DEBUG=ON -A x64
|
||||
cmake --build build --target testlightgbm --config Debug ; Assert-Output $?
|
||||
.\Debug\testlightgbm.exe ; Assert-Output $?
|
||||
exit 0
|
||||
}
|
||||
|
||||
if ($env:TASK -eq "swig") {
|
||||
$env:JAVA_HOME = $env:JAVA_HOME_8_X64 # there is pre-installed Eclipse Temurin 8 somewhere
|
||||
$ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows down download speed
|
||||
$params = @{
|
||||
Uri = "https://sourceforge.net/projects/swig/files/latest/download"
|
||||
OutFile = "$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip"
|
||||
UserAgent = "curl"
|
||||
}
|
||||
Invoke-WebRequest @params
|
||||
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
||||
[System.IO.Compression.ZipFile]::ExtractToDirectory(
|
||||
"$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip",
|
||||
"$env:BUILD_SOURCESDIRECTORY/swig"
|
||||
) ; Assert-Output $?
|
||||
$SwigFolder = Get-ChildItem -Name -Path "$env:BUILD_SOURCESDIRECTORY/swig" -Attributes Directory
|
||||
$env:PATH = @("$env:BUILD_SOURCESDIRECTORY/swig/$SwigFolder", "$env:PATH") -join ";"
|
||||
$BuildLogFileName = "$env:BUILD_SOURCESDIRECTORY\cmake_build.log"
|
||||
cmake -B build -S . -A x64 -DUSE_SWIG=ON *> "$BuildLogFileName" ; $build_succeeded = $?
|
||||
Write-Output "CMake build logs:"
|
||||
Get-Content -Path "$BuildLogFileName"
|
||||
Assert-Output $build_succeeded
|
||||
$checks = Select-String -Path "${BuildLogFileName}" -Pattern "-- Found SWIG.*${SwigFolder}/swig.exe"
|
||||
$checks_cnt = $checks.Matches.length
|
||||
if ($checks_cnt -eq 0) {
|
||||
Write-Output "Wrong SWIG version was found (expected '${SwigFolder}'). Check the build logs."
|
||||
Assert-Output $False
|
||||
}
|
||||
cmake --build build --target ALL_BUILD --config Release ; Assert-Output $?
|
||||
if ($env:PRODUCES_ARTIFACTS -eq "true") {
|
||||
cp ./build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar ; Assert-Output $?
|
||||
}
|
||||
exit 0
|
||||
}
|
||||
|
||||
# 'pixi' is used for end-of-life Python versions
|
||||
if ($env:PYTHON_VERSION -eq "3.10") {
|
||||
$activation = ((& pixi shell-hook --locked -e py310 --shell powershell) -join "`n")
|
||||
Invoke-Expression $activation ; Assert-Output $?
|
||||
} else {
|
||||
# update conda env
|
||||
$env:CONDA_ENV = "test-env"
|
||||
conda activate ; Assert-Output $?
|
||||
conda config --set always_yes yes --set changeps1 no ; Assert-Output $?
|
||||
conda config --remove channels defaults ; Assert-Output $?
|
||||
conda config --add channels nodefaults ; Assert-Output $?
|
||||
conda config --add channels conda-forge ; Assert-Output $?
|
||||
conda config --set channel_priority strict ; Assert-Output $?
|
||||
conda install -q -y conda "python=$env:PYTHON_VERSION[build=*_cp*]" ; Assert-Output $?
|
||||
|
||||
# print output of 'conda info', to help in submitting bug reports
|
||||
Write-Output "conda info:"
|
||||
conda info
|
||||
|
||||
$env:CONDA_REQUIREMENT_FILE = "$env:BUILD_SOURCESDIRECTORY/.ci/conda-envs/ci-core.txt"
|
||||
|
||||
$condaParams = @(
|
||||
"-y",
|
||||
"-n", "$env:CONDA_ENV",
|
||||
"--file", "$env:CONDA_REQUIREMENT_FILE",
|
||||
"python=$env:PYTHON_VERSION[build=*_cp*]"
|
||||
)
|
||||
conda create @condaParams ; Assert-Output $?
|
||||
|
||||
# print output of 'conda list', to help in submitting bug reports
|
||||
Write-Output "conda list:"
|
||||
conda list -n $env:CONDA_ENV
|
||||
|
||||
# 'bdist' job invokes 'RefreshEnv' to update PATH from the registry (which may have been modified
|
||||
# by building in OpenCL support), so defer activating the conda environment until later for those builds.
|
||||
if ($env:TASK -ne "bdist") {
|
||||
conda activate $env:CONDA_ENV
|
||||
}
|
||||
}
|
||||
|
||||
Set-Location "$env:BUILD_SOURCESDIRECTORY"
|
||||
if ($env:TASK -eq "regular") {
|
||||
cmake -B build -S . -A x64 ; Assert-Output $?
|
||||
cmake --build build --target ALL_BUILD --config Release ; Assert-Output $?
|
||||
sh ./build-python.sh install --precompile ; Assert-Output $?
|
||||
cp ./Release/lib_lightgbm.dll "$env:BUILD_ARTIFACTSTAGINGDIRECTORY"
|
||||
cp ./Release/lightgbm.exe "$env:BUILD_ARTIFACTSTAGINGDIRECTORY"
|
||||
} elseif ($env:TASK -eq "sdist") {
|
||||
sh ./build-python.sh sdist ; Assert-Output $?
|
||||
sh ./.ci/check-python-dists.sh ./dist ; Assert-Output $?
|
||||
Set-Location dist; pip install --no-deps @(Get-ChildItem *.gz) -v ; Assert-Output $?
|
||||
} elseif ($env:TASK -eq "bdist") {
|
||||
# Import the Chocolatey profile module so that the RefreshEnv command
|
||||
# invoked below properly updates the current PowerShell session environment.
|
||||
$module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
|
||||
Import-Module "$module" ; Assert-Output $?
|
||||
RefreshEnv
|
||||
|
||||
Write-Output "Current OpenCL drivers:"
|
||||
Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors
|
||||
|
||||
# (re-) activate conda environment, in case any activation logic was overridden by that 'RefreshEnv' call above
|
||||
conda activate $env:CONDA_ENV
|
||||
|
||||
# TODO: restore --integrated-opencl as part of https://github.com/lightgbm-org/LightGBM/issues/6968
|
||||
sh "build-python.sh" bdist_wheel ; Assert-Output $?
|
||||
sh ./.ci/check-python-dists.sh ./dist ; Assert-Output $?
|
||||
Set-Location dist; pip install --no-deps @(Get-ChildItem *py3-none-win_amd64.whl) ; Assert-Output $?
|
||||
cp @(Get-ChildItem *py3-none-win_amd64.whl) "$env:BUILD_ARTIFACTSTAGINGDIRECTORY"
|
||||
} elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
|
||||
if ($env:COMPILER -eq "MINGW") {
|
||||
sh ./build-python.sh install --mingw ; Assert-Output $?
|
||||
} else {
|
||||
sh ./build-python.sh install; Assert-Output $?
|
||||
}
|
||||
}
|
||||
|
||||
if (($env:TASK -eq "sdist") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
|
||||
# cannot test C API with "sdist" task
|
||||
$tests = "$env:BUILD_SOURCESDIRECTORY/tests/python_package_test"
|
||||
} else {
|
||||
$tests = "$env:BUILD_SOURCESDIRECTORY/tests"
|
||||
}
|
||||
if ($env:TASK -eq "bdist") {
|
||||
# Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
|
||||
# TODO: set LIGHTGBM_TEST_DUAL_CPU_GPU back to "1" as part of https://github.com/lightgbm-org/LightGBM/issues/6968
|
||||
$env:LIGHTGBM_TEST_DUAL_CPU_GPU = "0"
|
||||
}
|
||||
|
||||
pytest -ra $tests ; Assert-Output $?
|
||||
|
||||
if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
|
||||
Set-Location "$env:BUILD_SOURCESDIRECTORY/examples/python-guide"
|
||||
@("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"
|
||||
# Prevent interactive window mode
|
||||
(Get-Content "plot_example.py").replace(
|
||||
'graph.render(view=True)',
|
||||
'graph.render(view=False)'
|
||||
) | Set-Content "plot_example.py"
|
||||
|
||||
# install optional plotting libraries
|
||||
# (not necessary for pixi-managed environments, where they're just installed by default)
|
||||
if ($env:PYTHON_VERSION -ne "3.10") {
|
||||
conda install -y -n $env:CONDA_ENV "h5py>=3.10" "ipywidgets>=8.1.2" "notebook>=7.1.2"
|
||||
}
|
||||
# Run all examples
|
||||
foreach ($file in @(Get-ChildItem *.py)) {
|
||||
@(
|
||||
"import sys, warnings",
|
||||
-join @(
|
||||
"warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: ",
|
||||
"sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))"
|
||||
)
|
||||
) + (Get-Content $file) | Set-Content $file
|
||||
python $file ; Assert-Output $?
|
||||
}
|
||||
# Run all notebooks
|
||||
Set-Location "$env:BUILD_SOURCESDIRECTORY/examples/python-guide/notebooks"
|
||||
(Get-Content "interactive_plot_example.ipynb").replace(
|
||||
'INTERACTIVE = False',
|
||||
'assert False, \"Interactive mode disabled\"'
|
||||
) | Set-Content "interactive_plot_example.ipynb"
|
||||
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb ; Assert-Output $?
|
||||
}
|
||||
Executable
+321
@@ -0,0 +1,321 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -E -o -u pipefail
|
||||
|
||||
# defaults
|
||||
CONDA_ENV="test-env"
|
||||
IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"}
|
||||
METHOD=${METHOD:-""}
|
||||
PRODUCES_ARTIFACTS=${PRODUCES_ARTIFACTS:-"false"}
|
||||
SANITIZERS=${SANITIZERS:-""}
|
||||
|
||||
ARCH=$(uname -m)
|
||||
|
||||
LGB_VER=$(head -n 1 "${BUILD_DIRECTORY}/VERSION.txt")
|
||||
|
||||
# create the artifact upload directory if it doesn't exist yet
|
||||
mkdir -p "${BUILD_ARTIFACTSTAGINGDIRECTORY}"
|
||||
|
||||
if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then
|
||||
export CXX=g++-14
|
||||
export CC=gcc-14
|
||||
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang" ]]; then
|
||||
export CXX=clang++
|
||||
export CC=clang
|
||||
elif [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "clang-17" ]]; then
|
||||
export CXX=clang++-17
|
||||
export CC=clang-17
|
||||
fi
|
||||
|
||||
if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
|
||||
export LANG="en_US.UTF-8"
|
||||
export LC_ALL="en_US.UTF-8"
|
||||
fi
|
||||
|
||||
# Setting MACOSX_DEPLOYMENT_TARGET prevents CMake from building against too-new
|
||||
# macOS features, and helps tools like Python build tools determine the appropriate
|
||||
# wheel compatibility tags.
|
||||
#
|
||||
# ref:
|
||||
# * https://cmake.org/cmake/help/latest/envvar/MACOSX_DEPLOYMENT_TARGET.html
|
||||
# * https://github.com/scikit-build/scikit-build-core/blob/acb7d0346e4a05bcb47a4ea3939c705ab71e3145/src/scikit_build_core/builder/macos.py#L36
|
||||
if [[ $ARCH == "x86_64" ]]; then
|
||||
export MACOSX_DEPLOYMENT_TARGET=10.15
|
||||
else
|
||||
export MACOSX_DEPLOYMENT_TARGET=12.0
|
||||
fi
|
||||
|
||||
if [[ "${TASK}" == "r-package" ]]; then
|
||||
bash "${BUILD_DIRECTORY}/.ci/test-r-package.sh" || exit 1
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd "${BUILD_DIRECTORY}"
|
||||
|
||||
if [[ $TASK == "swig" ]]; then
|
||||
cmake -B build -S . -DUSE_SWIG=ON
|
||||
cmake --build build -j4 || exit 1
|
||||
if [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "gcc" ]]; then
|
||||
objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1
|
||||
objdump -T ./lib_lightgbm_swig.so >> ./objdump.log || exit 1
|
||||
./.ci/check-dynamic-dependencies.sh ./objdump.log || exit 1
|
||||
fi
|
||||
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
|
||||
cp ./build/lightgbmlib.jar "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbmlib_${OS_NAME}.jar"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$TASK" == "cpp-tests" ]]; then
|
||||
cmake_args=(
|
||||
-DBUILD_CPP_TEST=ON
|
||||
-DUSE_DEBUG=ON
|
||||
)
|
||||
if [[ $METHOD == "with-sanitizers" ]]; then
|
||||
cmake_args+=("-DUSE_SANITIZER=ON")
|
||||
if [[ -n $SANITIZERS ]]; then
|
||||
cmake_args+=("-DENABLED_SANITIZERS=$SANITIZERS")
|
||||
fi
|
||||
fi
|
||||
cmake -B build -S . "${cmake_args[@]}"
|
||||
cmake --build build --target testlightgbm -j4 || exit 1
|
||||
./testlightgbm || exit 1
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# including python=version=[build=*_cp*] to ensure that conda prefers CPython and doesn't fall back to
|
||||
# other implementations like pypy
|
||||
CONDA_PYTHON_REQUIREMENT="python=${PYTHON_VERSION}[build=*_cp*]"
|
||||
|
||||
if [[ $TASK == "if-else" ]]; then
|
||||
conda create -q -y -n "${CONDA_ENV}" "${CONDA_PYTHON_REQUIREMENT}" numpy
|
||||
# shellcheck disable=SC1091
|
||||
source activate "${CONDA_ENV}"
|
||||
cmake -B build -S . || exit 1
|
||||
cmake --build build --target lightgbm -j4 || exit 1
|
||||
cd "$BUILD_DIRECTORY/tests/cpp_tests"
|
||||
../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp
|
||||
../../lightgbm config=predict.conf output_result=origin.pred
|
||||
../../lightgbm config=predict.conf output_result=ifelse.pred
|
||||
python test.py
|
||||
exit 0
|
||||
fi
|
||||
|
||||
PYTHON_ENV_MANAGER="conda"
|
||||
if [[ "${PYTHON_VERSION}" == "3.10" ]]; then
|
||||
PYTHON_ENV_MANAGER="pixi"
|
||||
CI_PIXI_ENV="py310"
|
||||
fi
|
||||
|
||||
# 'pixi' is used for end-of-life Python versions
|
||||
if [[ "${PYTHON_ENV_MANAGER}" == "pixi" ]]; then
|
||||
eval "$(pixi shell-hook --locked -e "${CI_PIXI_ENV}")"
|
||||
else
|
||||
CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt"
|
||||
conda create \
|
||||
-y \
|
||||
-n "${CONDA_ENV}" \
|
||||
--file "${CONDA_REQUIREMENT_FILE}" \
|
||||
"${CONDA_PYTHON_REQUIREMENT}" \
|
||||
|| exit 1
|
||||
|
||||
# print output of 'conda list', to help in submitting bug reports
|
||||
echo "conda list:"
|
||||
conda list -n ${CONDA_ENV}
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source activate $CONDA_ENV
|
||||
fi
|
||||
|
||||
cd "${BUILD_DIRECTORY}"
|
||||
|
||||
if [[ $TASK == "sdist" ]]; then
|
||||
sh ./build-python.sh sdist || exit 1
|
||||
sh .ci/check-python-dists.sh ./dist || exit 1
|
||||
pip install -v --no-deps "./dist/lightgbm-${LGB_VER}.tar.gz" || exit 1
|
||||
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
|
||||
cp "./dist/lightgbm-${LGB_VER}.tar.gz" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
|
||||
fi
|
||||
pytest -ra ./tests/python_package_test || exit 1
|
||||
exit 0
|
||||
elif [[ $TASK == "bdist" ]]; then
|
||||
if [[ $OS_NAME == "macos" ]]; then
|
||||
sh ./build-python.sh bdist_wheel || exit 1
|
||||
sh .ci/check-python-dists.sh ./dist || exit 1
|
||||
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
|
||||
cp "$(echo "dist/lightgbm-${LGB_VER}-py3-none-macosx"*.whl)" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
|
||||
fi
|
||||
else
|
||||
sh ./build-python.sh bdist_wheel --integrated-opencl || exit 1
|
||||
|
||||
# print some debugging logs about the wheel's GLIBC version and dependencies on shared libraries
|
||||
pip install 'auditwheel>=6.5.1'
|
||||
auditwheel show ./dist/lightgbm*.whl
|
||||
|
||||
# pass through 'auditwheel repair' to set the appropriate wheel tags.
|
||||
#
|
||||
# intentionally avoid vendoring libgomp, to reduce the risk of multiple OpenMP libraries
|
||||
# being loaded in the same process.
|
||||
auditwheel repair \
|
||||
--exclude 'libgomp.so*' \
|
||||
--lib-sdir '' \
|
||||
--wheel-dir dist-fixed/ \
|
||||
./dist/lightgbm*.whl
|
||||
|
||||
# overwrite the original wheel with the new one
|
||||
rm ./dist/lightgbm*.whl
|
||||
mv ./dist-fixed/lightgbm*.whl ./dist
|
||||
|
||||
# check wheel properties
|
||||
sh .ci/check-python-dists.sh ./dist || exit 1
|
||||
|
||||
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
|
||||
# hard-code expected tag so CI will fail if 'auditwheel repair' has a surprising result (e.g. newer
|
||||
# manylinux tag than we intended)
|
||||
if [[ $ARCH == "x86_64" ]]; then
|
||||
PLATFORM="manylinux_2_27_x86_64.manylinux_2_28_x86_64"
|
||||
else
|
||||
PLATFORM="manylinux2014_aarch64.manylinux_2_17_aarch64"
|
||||
fi
|
||||
cp "dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1
|
||||
fi
|
||||
# Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
|
||||
export LIGHTGBM_TEST_DUAL_CPU_GPU=1
|
||||
fi
|
||||
pip install -v --no-deps ./dist/*.whl || exit 1
|
||||
pytest -ra ./tests || exit 1
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ $TASK == "gpu" ]]; then
|
||||
sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' ./include/LightGBM/config.h
|
||||
grep -q 'std::string device_type = "gpu"' ./include/LightGBM/config.h || exit 1 # make sure that changes were really done
|
||||
if [[ $METHOD == "pip" ]]; then
|
||||
sh ./build-python.sh sdist || exit 1
|
||||
sh .ci/check-python-dists.sh ./dist || exit 1
|
||||
pip install \
|
||||
-v \
|
||||
--no-deps \
|
||||
--config-settings=cmake.define.USE_GPU=ON \
|
||||
"./dist/lightgbm-${LGB_VER}.tar.gz" \
|
||||
|| exit 1
|
||||
pytest -ra ./tests/python_package_test || exit 1
|
||||
exit 0
|
||||
elif [[ $METHOD == "wheel" ]]; then
|
||||
sh ./build-python.sh bdist_wheel --gpu || exit 1
|
||||
sh ./.ci/check-python-dists.sh ./dist || exit 1
|
||||
pip install -v --no-deps "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" || exit 1
|
||||
pytest -ra ./tests || exit 1
|
||||
exit 0
|
||||
elif [[ $METHOD == "source" ]]; then
|
||||
cmake -B build -S . -DUSE_GPU=ON
|
||||
fi
|
||||
elif [[ $TASK == "cuda" ]]; then
|
||||
sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "cuda";/' ./include/LightGBM/config.h
|
||||
grep -q 'std::string device_type = "cuda"' ./include/LightGBM/config.h || exit 1 # make sure that changes were really done
|
||||
# by default ``gpu_use_dp=false`` for efficiency. change to ``true`` here for exact results in ci tests
|
||||
sed -i'.bak' 's/gpu_use_dp = false;/gpu_use_dp = true;/' ./include/LightGBM/config.h
|
||||
grep -q 'gpu_use_dp = true' ./include/LightGBM/config.h || exit 1 # make sure that changes were really done
|
||||
if [[ $METHOD == "pip" ]]; then
|
||||
sh ./build-python.sh sdist || exit 1
|
||||
sh ./.ci/check-python-dists.sh ./dist || exit 1
|
||||
pip install \
|
||||
-v \
|
||||
--no-deps \
|
||||
--config-settings=cmake.define.USE_CUDA=ON \
|
||||
"./dist/lightgbm-${LGB_VER}.tar.gz" \
|
||||
|| exit 1
|
||||
pytest -ra ./tests/python_package_test || exit 1
|
||||
exit 0
|
||||
elif [[ $METHOD == "wheel" ]]; then
|
||||
sh ./build-python.sh bdist_wheel --cuda || exit 1
|
||||
sh ./.ci/check-python-dists.sh ./dist || exit 1
|
||||
pip install -v --no-deps "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" || exit 1
|
||||
pytest -ra ./tests || exit 1
|
||||
exit 0
|
||||
elif [[ $METHOD == "source" ]]; then
|
||||
# we want at least 1 CI job testing that manual override of CMAKE_CUDA_ARCHITECTURES works
|
||||
cmake \
|
||||
-B build \
|
||||
-S . \
|
||||
-DUSE_CUDA=ON \
|
||||
-DCMAKE_CUDA_ARCHITECTURES="native"
|
||||
fi
|
||||
elif [[ $TASK == "mpi" ]]; then
|
||||
if [[ $METHOD == "pip" ]]; then
|
||||
sh ./build-python.sh sdist || exit 1
|
||||
sh ./.ci/check-python-dists.sh ./dist || exit 1
|
||||
pip install \
|
||||
-v \
|
||||
--no-deps \
|
||||
--config-settings=cmake.define.USE_MPI=ON \
|
||||
"./dist/lightgbm-${LGB_VER}.tar.gz" \
|
||||
|| exit 1
|
||||
pytest -ra ./tests/python_package_test || exit 1
|
||||
exit 0
|
||||
elif [[ $METHOD == "wheel" ]]; then
|
||||
sh ./build-python.sh bdist_wheel --mpi || exit 1
|
||||
sh ./.ci/check-python-dists.sh ./dist || exit 1
|
||||
pip install -v --no-deps "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" || exit 1
|
||||
pytest -ra ./tests || exit 1
|
||||
exit 0
|
||||
elif [[ $METHOD == "source" ]]; then
|
||||
cmake -B build -S . -DUSE_MPI=ON -DUSE_DEBUG=ON
|
||||
fi
|
||||
else
|
||||
cmake -B build -S .
|
||||
fi
|
||||
|
||||
cmake --build build --target _lightgbm -j4 || exit 1
|
||||
|
||||
sh ./build-python.sh install --precompile || exit 1
|
||||
pytest -ra ./tests || exit 1
|
||||
|
||||
if [[ $TASK == "regular" ]]; then
|
||||
if [[ $PRODUCES_ARTIFACTS == "true" ]]; then
|
||||
if [[ $OS_NAME == "macos" ]]; then
|
||||
cp ./lib_lightgbm.dylib "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.dylib"
|
||||
else
|
||||
if [[ $COMPILER == "gcc" ]]; then
|
||||
objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1
|
||||
./.ci/check-dynamic-dependencies.sh ./objdump.log || exit 1
|
||||
fi
|
||||
cp ./lib_lightgbm.so "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.so"
|
||||
fi
|
||||
fi
|
||||
cd "$BUILD_DIRECTORY/examples/python-guide"
|
||||
sed -i'.bak' '/import lightgbm as lgb/a\
|
||||
import matplotlib\
|
||||
matplotlib.use\(\"Agg\"\)\
|
||||
' plot_example.py # prevent interactive window mode
|
||||
sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
|
||||
# install optional plotting libraries
|
||||
# (not necessary for pixi-managed environments, where they're just installed by default)
|
||||
if [[ "${PYTHON_ENV_MANAGER}" != "pixi" ]]; then
|
||||
conda install -y -n $CONDA_ENV \
|
||||
'h5py>=3.10' \
|
||||
'ipywidgets>=8.1.2' \
|
||||
'notebook>=7.1.2'
|
||||
fi
|
||||
for f in *.py **/*.py; do python "${f}" || exit 1; done # run all examples
|
||||
cd "$BUILD_DIRECTORY/examples/python-guide/notebooks"
|
||||
sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb
|
||||
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace ./*.ipynb || exit 1 # run all notebooks
|
||||
|
||||
# importing the library should succeed even if all optional dependencies are not present
|
||||
if [[ "${PYTHON_ENV_MANAGER}" != "pixi" ]]; then
|
||||
conda uninstall -n $CONDA_ENV --force --yes \
|
||||
cffi \
|
||||
dask \
|
||||
distributed \
|
||||
joblib \
|
||||
matplotlib-base \
|
||||
pandas \
|
||||
polars \
|
||||
psutil \
|
||||
pyarrow \
|
||||
python-graphviz \
|
||||
scikit-learn || exit 1
|
||||
fi
|
||||
python -c "import lightgbm" || exit 1
|
||||
fi
|
||||
@@ -0,0 +1,19 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
end_of_line = lf
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
|
||||
[*.{py,sh,ps1,js,json}]
|
||||
indent_size = 4
|
||||
max_line_length = 120
|
||||
skip = external_libs
|
||||
known_first_party = lightgbm
|
||||
|
||||
# Tabs matter for Makefile and .gitmodules
|
||||
[{makefile*,Makefile*,*.mk,*.mak,*.makefile,*.Makefile,GNUmakefile,BSDmakefile,make.bat,Makevars*,*.gitmodules}]
|
||||
indent_style = tab
|
||||
@@ -0,0 +1,8 @@
|
||||
# introduce ruff-format (#6308)
|
||||
6330d6269c81dfd4c96e664b99239b8ff39ccf91
|
||||
# enable ruff format on tests and examples (#6317)
|
||||
1b792e716682254c33ddb5eb845357e84018636d
|
||||
# enable ruff-format on main library Python code (#6336)
|
||||
dd31208ab7a7aea86762830697b00666f843ded9
|
||||
# enable whitespace/indent_namespace rule from cpplint (#7056)
|
||||
50f11a9f3c066eadee475ea567f9e9d49e6bb827
|
||||
@@ -0,0 +1,34 @@
|
||||
* text=auto eol=lf
|
||||
|
||||
# Explicitly declare text files you want to always be normalized and converted
|
||||
# to native line endings on checkout.
|
||||
*.c text
|
||||
*.cc text
|
||||
*.cmake text
|
||||
*.conf text
|
||||
*.cpp text
|
||||
*.cu text
|
||||
*.cuh text
|
||||
*.h text
|
||||
*.hpp text
|
||||
*.i text
|
||||
*.js text
|
||||
*.md text
|
||||
*.py text
|
||||
*.R text
|
||||
*.rst text
|
||||
*.sh text
|
||||
*.svg text
|
||||
*.toml text
|
||||
*.tpp text
|
||||
*.txt text
|
||||
*.yaml text
|
||||
*.yml text
|
||||
|
||||
# help git identify which files should be treated as binary
|
||||
*.ico binary
|
||||
*.png binary
|
||||
|
||||
# SCM syntax highlighting & preventing 3-way merges
|
||||
# (added automatically by 'pixi init')
|
||||
pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff
|
||||
@@ -0,0 +1,10 @@
|
||||
# This file controls default reviewers for LightGBM code.
|
||||
# See https://help.github.com/en/articles/about-code-owners
|
||||
# for details
|
||||
#
|
||||
# Maintainers are encouraged to use their best discretion in
|
||||
# setting reviewers on PRs manually, but this file should
|
||||
# offer a reasonable automatic best-guess
|
||||
|
||||
# catch-all rule (this only gets matched if no rules below match)
|
||||
* @guolinke @jameslamb @shiyu1994 @jmoralez @borchero @StrikerRUS @mayer79
|
||||
@@ -0,0 +1,26 @@
|
||||
---
|
||||
name: Bug Report 🐞
|
||||
about: Something isn't working as expected? Here is the right place to report.
|
||||
---
|
||||
|
||||
## Description
|
||||
<!-- A clear description of the bug -->
|
||||
|
||||
## Reproducible example
|
||||
<!-- Minimal code that exhibits this behavior -->
|
||||
|
||||
## Environment info
|
||||
|
||||
LightGBM version or commit hash:
|
||||
|
||||
Command(s) you used to install LightGBM
|
||||
|
||||
```shell
|
||||
|
||||
```
|
||||
|
||||
<!-- Put any additional environment information here -->
|
||||
|
||||
|
||||
## Additional Comments
|
||||
<!-- What else should we know? -->
|
||||
@@ -0,0 +1,25 @@
|
||||
---
|
||||
name: Feature Request 💡
|
||||
about: Suggest a new idea for the project.
|
||||
labels: enhancement, feature-request
|
||||
---
|
||||
|
||||
<!--
|
||||
Please search your feature on previous issues and our feature requests consolidation hub (https://github.com/lightgbm-org/LightGBM/issues/2302) before you open a new one.
|
||||
-->
|
||||
|
||||
## Summary
|
||||
|
||||
<!-- Briefly explain your feature proposal. -->
|
||||
|
||||
## Motivation
|
||||
|
||||
<!-- Why is it useful to have this feature in the LightGBM project? -->
|
||||
|
||||
## Description
|
||||
|
||||
<!-- Detailed description of the new feature. -->
|
||||
|
||||
## References
|
||||
|
||||
<!-- Any useful references, for instance, papers, implementations in other projects, draft code snippets, etc. -->
|
||||
@@ -0,0 +1,17 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: monthly
|
||||
groups:
|
||||
ci-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
cooldown:
|
||||
# only accept releases that have been out for at least 15 days
|
||||
default-days: 15
|
||||
commit-message:
|
||||
prefix: "[ci]"
|
||||
labels:
|
||||
- maintenance
|
||||
@@ -0,0 +1,20 @@
|
||||
name-template: 'v$NEXT_PATCH_VERSION'
|
||||
tag-template: 'v$NEXT_PATCH_VERSION'
|
||||
categories:
|
||||
- title: '💡 New Features'
|
||||
label: 'feature'
|
||||
- title: '🔨 Breaking'
|
||||
label: 'breaking'
|
||||
- title: '🚀 Efficiency Improvement'
|
||||
label: 'efficiency'
|
||||
- title: '🐛 Bug Fixes'
|
||||
label: 'fix'
|
||||
- title: '📖 Documentation'
|
||||
label: 'doc'
|
||||
- title: '🧰 Maintenance'
|
||||
label: 'maintenance'
|
||||
change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
|
||||
template: |
|
||||
## Changes
|
||||
|
||||
$CHANGES
|
||||
@@ -0,0 +1,81 @@
|
||||
# builds core artifacts, intended to be attached to releases
|
||||
# or used by other workflows
|
||||
name: Build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
env:
|
||||
# tell scripts where to put artifacts
|
||||
BUILD_ARTIFACTSTAGINGDIRECTORY: '${{ github.workspace }}/artifacts'
|
||||
|
||||
jobs:
|
||||
archive:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Create source archive
|
||||
run: |
|
||||
mkdir -p "${BUILD_ARTIFACTSTAGINGDIRECTORY}"
|
||||
tar \
|
||||
-czvf \
|
||||
/tmp/LightGBM-complete_source_code_tar_gz.tar.gz \
|
||||
.
|
||||
mv \
|
||||
/tmp/LightGBM-complete_source_code_tar_gz.tar.gz \
|
||||
${BUILD_ARTIFACTSTAGINGDIRECTORY}/
|
||||
- name: Create commit.txt
|
||||
shell: bash
|
||||
run: |
|
||||
# for pull requests, github.sha refers to the merge commit from merging the PR and
|
||||
# target branch... we want the actual commit that was pushed
|
||||
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
|
||||
COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
|
||||
else
|
||||
COMMIT_SHA="${{ github.sha }}"
|
||||
fi
|
||||
echo "${COMMIT_SHA}" > "${BUILD_ARTIFACTSTAGINGDIRECTORY}/commit.txt"
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: source-archive
|
||||
path: |
|
||||
${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}/commit.txt
|
||||
${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}/LightGBM-complete_source_code_tar_gz.tar.gz
|
||||
if-no-files-found: error
|
||||
all-build-jobs-successful:
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- archive
|
||||
permissions:
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Note that all tests succeeded
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
@@ -0,0 +1,158 @@
|
||||
name: C++
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
env:
|
||||
# tell scripts where to put artifacts
|
||||
# (this variable name is left over from when jobs ran on Azure DevOps)
|
||||
BUILD_ARTIFACTSTAGINGDIRECTORY: '${{ github.workspace }}/artifacts'
|
||||
# where repo sources are cloned to
|
||||
BUILD_DIRECTORY: '${{ github.workspace }}'
|
||||
# in CMake-driven builds, parallelize compilation
|
||||
CMAKE_BUILD_PARALLEL_LEVEL: 4
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: ${{ matrix.task }} (${{ matrix.os-display-name || matrix.os }}, ${{ matrix.compiler }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
container: ${{ matrix.container }}
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
#############
|
||||
# C++ tests #
|
||||
#############
|
||||
- os: ubuntu-latest
|
||||
task: cpp-tests
|
||||
compiler: clang-17
|
||||
method: with-sanitizers
|
||||
container: 'ubuntu:22.04'
|
||||
os-display-name: 'ubuntu22.04'
|
||||
- os: macos-15-intel
|
||||
task: cpp-tests
|
||||
compiler: clang
|
||||
method: with-sanitizers
|
||||
sanitizers: "address;undefined"
|
||||
container: null
|
||||
- os: windows-2022
|
||||
task: cpp-tests
|
||||
compiler: msvc
|
||||
container: null
|
||||
###########
|
||||
# if-else #
|
||||
###########
|
||||
# These test CLI-generated C++ inference code.
|
||||
#
|
||||
# They need Python because they're written with pytest, but they
|
||||
# do not need the LightGBM Python package
|
||||
- os: ubuntu-latest
|
||||
task: if-else
|
||||
compiler: clang-17
|
||||
container: 'ubuntu:22.04'
|
||||
os-display-name: 'ubuntu22.04'
|
||||
python_version: '3.14'
|
||||
setup-conda: 'true'
|
||||
- os: macos-15-intel
|
||||
task: if-else
|
||||
compiler: clang
|
||||
python_version: '3.10'
|
||||
container: null
|
||||
- os: ubuntu-latest
|
||||
task: if-else
|
||||
compiler: gcc
|
||||
python_version: '3.11'
|
||||
container: 'ghcr.io/lightgbm-org/lightgbm-vsts-agent:manylinux_2_28_x86_64'
|
||||
os-display-name: 'manylinux_2_28'
|
||||
steps:
|
||||
- name: Install packages used by third-party actions
|
||||
if: matrix.container == 'ubuntu:22.04'
|
||||
shell: bash
|
||||
run: |
|
||||
apt-get update -y
|
||||
apt-get install --no-install-recommends -y \
|
||||
dirmngr \
|
||||
gpg \
|
||||
gpg-agent \
|
||||
software-properties-common \
|
||||
sudo
|
||||
# install newest version of git
|
||||
# ref:
|
||||
# - https://unix.stackexchange.com/a/170831/550004
|
||||
# - https://git-scm.com/download/linux
|
||||
add-apt-repository ppa:git-core/ppa -y
|
||||
apt-get update -y
|
||||
apt-get install --no-install-recommends -y \
|
||||
git
|
||||
- name: Trust git cloning LightGBM
|
||||
if: startsWith(matrix.os, 'ubuntu')
|
||||
run: |
|
||||
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Setup and run tests on Linux and macOS
|
||||
if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu')
|
||||
shell: bash
|
||||
run: |
|
||||
export COMPILER="${{ matrix.compiler }}"
|
||||
export CONDA="${HOME}/miniforge"
|
||||
export METHOD="${{ matrix.method }}"
|
||||
export PATH=${CONDA}/bin:${PATH}
|
||||
export PYTHON_VERSION="${{ matrix.python_version }}"
|
||||
export SANITIZERS="${{ matrix.sanitizers }}"
|
||||
export SETUP_CONDA="${{ matrix.setup-conda }}"
|
||||
export TASK="${{ matrix.task }}"
|
||||
if [[ "${{ matrix.os }}" =~ ^macos ]]; then
|
||||
export OS_NAME="macos"
|
||||
elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
|
||||
export OS_NAME="linux"
|
||||
fi
|
||||
if [[ "${{ matrix.container }}" == "ubuntu:22.04" ]]; then
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export IN_UBUNTU_BASE_CONTAINER="true"
|
||||
fi
|
||||
$GITHUB_WORKSPACE/.ci/setup.sh
|
||||
$GITHUB_WORKSPACE/.ci/test.sh
|
||||
- name: Setup and run tests on Windows
|
||||
if: startsWith(matrix.os, 'windows')
|
||||
shell: pwsh -command ". {0}"
|
||||
run: |
|
||||
$env:BUILD_SOURCESDIRECTORY = $env:BUILD_DIRECTORY
|
||||
$env:TASK = "${{ matrix.task }}"
|
||||
& "$env:GITHUB_WORKSPACE/.ci/test-windows.ps1"
|
||||
all-cpp-jobs-successful:
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- test
|
||||
permissions:
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Note that all tests succeeded
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
@@ -0,0 +1,115 @@
|
||||
name: CUDA Version
|
||||
|
||||
on:
|
||||
push:
|
||||
# run on pushes to LightGBM branches matching certain names
|
||||
branches:
|
||||
- cuda-*
|
||||
# Run manually by clicking a button in the UI
|
||||
workflow_dispatch:
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
jobs:
|
||||
test:
|
||||
# yamllint disable-line rule:line-length
|
||||
name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }})
|
||||
# yamllint disable-line rule:line-length
|
||||
# ref: https://docs.github.com/en/actions/concepts/runners/about-larger-runners#specifications-for-gpu-larger-runners
|
||||
runs-on: github-hosted-linux-gpu-amd64
|
||||
container:
|
||||
image: nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel-${{ matrix.linux_version }}
|
||||
env:
|
||||
CMAKE_BUILD_PARALLEL_LEVEL: 4
|
||||
COMPILER: ${{ matrix.compiler }}
|
||||
CONDA: /tmp/miniforge
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
METHOD: ${{ matrix.method }}
|
||||
OS_NAME: linux
|
||||
PYTHON_VERSION: ${{ matrix.python_version }}
|
||||
TASK: ${{ matrix.task }}
|
||||
SKBUILD_STRICT_CONFIG: true
|
||||
options: --gpus all
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- method: wheel
|
||||
compiler: gcc
|
||||
python_version: "3.11"
|
||||
cuda_version: "13.2.1"
|
||||
linux_version: "ubuntu24.04"
|
||||
task: cuda
|
||||
- method: wheel
|
||||
compiler: gcc
|
||||
python_version: "3.11"
|
||||
cuda_version: "12.9.1"
|
||||
linux_version: "ubuntu22.04"
|
||||
task: cuda
|
||||
- method: source
|
||||
compiler: gcc
|
||||
python_version: "3.14"
|
||||
cuda_version: "12.2.2"
|
||||
linux_version: "ubuntu22.04"
|
||||
task: cuda
|
||||
- method: pip
|
||||
compiler: gcc
|
||||
python_version: "3.12"
|
||||
cuda_version: "11.8.0"
|
||||
linux_version: "ubuntu20.04"
|
||||
task: cuda
|
||||
steps:
|
||||
- name: Install latest git and sudo
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install --no-install-recommends -y \
|
||||
ca-certificates \
|
||||
software-properties-common
|
||||
add-apt-repository ppa:git-core/ppa -y
|
||||
apt-get update
|
||||
apt-get install --no-install-recommends -y \
|
||||
git \
|
||||
sudo
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Setup and run tests
|
||||
env:
|
||||
CI_CUDA_VERSION: ${{ matrix.cuda_version }}
|
||||
run: |
|
||||
export BUILD_ARTIFACTSTAGINGDIRECTORY="${{ github.workspace }}/artifacts"
|
||||
export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
|
||||
export PATH=$CONDA/bin:$PATH
|
||||
|
||||
# check GPU usage
|
||||
nvidia-smi
|
||||
|
||||
# build and test
|
||||
$GITHUB_WORKSPACE/.ci/setup.sh
|
||||
$GITHUB_WORKSPACE/.ci/test.sh
|
||||
all-cuda-jobs-successful:
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- test
|
||||
permissions:
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Note that all tests succeeded
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
@@ -0,0 +1,58 @@
|
||||
name: 'Lock Inactive Threads'
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# midnight UTC, every Wednesday, for Issues
|
||||
- cron: '0 0 * * 3'
|
||||
# midnight UTC, every Thursday, for PRs
|
||||
- cron: '0 0 * * 4'
|
||||
# allow manual triggering from GitHub UI
|
||||
workflow_dispatch:
|
||||
|
||||
# only 1 job running in the repo at any time
|
||||
concurrency:
|
||||
group: lock
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
jobs:
|
||||
action:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: dessant/lock-threads@89ae32b08ed1a541efecbab17912962a5e38981c # v6.0.2
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
# after how many days of inactivity should a closed issue/PR be locked?
|
||||
issue-inactive-days: '365'
|
||||
pr-inactive-days: '365'
|
||||
# do not close feature request issues...
|
||||
# we close those but track them in https://github.com/lightgbm-org/LightGBM/issues/2302
|
||||
exclude-any-issue-labels: 'feature request'
|
||||
# what labels should be removed prior to locking?
|
||||
remove-issue-labels: 'awaiting response,awaiting review,blocking,in progress'
|
||||
remove-pr-labels: 'awaiting response,awaiting review,blocking,in progress'
|
||||
# what message should be posted prior to locking?
|
||||
issue-comment: >
|
||||
This issue has been automatically locked
|
||||
since there has not been any recent activity since it was closed.
|
||||
|
||||
To start a new related discussion,
|
||||
open a new issue at https://github.com/lightgbm-org/LightGBM/issues
|
||||
including a reference to this.
|
||||
pr-comment: >
|
||||
This pull request has been automatically locked
|
||||
since there has not been any recent activity since it was closed.
|
||||
|
||||
To start a new related discussion,
|
||||
open a new issue at https://github.com/lightgbm-org/LightGBM/issues
|
||||
including a reference to this.
|
||||
# what should the locking status be?
|
||||
issue-lock-reason: 'resolved'
|
||||
pr-lock-reason: 'resolved'
|
||||
process-only: ${{ github.event.schedule == '0 0 * * 3' && 'issues' || 'prs' }}
|
||||
@@ -0,0 +1,54 @@
|
||||
name: Link checks
|
||||
|
||||
on:
|
||||
# Run manually by clicking a button in the UI
|
||||
workflow_dispatch:
|
||||
# Run once a day at 8:00am UTC
|
||||
schedule:
|
||||
- cron: '0 8 * * *'
|
||||
|
||||
# only 1 job running in the repo at any time
|
||||
concurrency:
|
||||
group: lock
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
env:
|
||||
COMPILER: gcc
|
||||
OS_NAME: 'linux'
|
||||
TASK: 'check-links'
|
||||
|
||||
jobs:
|
||||
check-links:
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: false
|
||||
- name: Build docs
|
||||
run: |
|
||||
export CONDA=${HOME}/miniforge
|
||||
export PATH=${CONDA}/bin:${HOME}/.local/bin:${PATH}
|
||||
$GITHUB_WORKSPACE/.ci/setup.sh || exit 1
|
||||
$GITHUB_WORKSPACE/.ci/build-docs.sh || exit 1
|
||||
- name: Check links
|
||||
uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2.8.0
|
||||
with:
|
||||
args: >-
|
||||
--config=./docs/.lychee.toml
|
||||
--
|
||||
"**/*.rst"
|
||||
"**/*.md"
|
||||
"./R-package/**/*.Rd"
|
||||
"./docs/_build/html/*.html"
|
||||
fail: true
|
||||
failIfEmpty: true
|
||||
@@ -0,0 +1,40 @@
|
||||
name: No Response Bot
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
schedule:
|
||||
# "every day at 04:00 UTC"
|
||||
- cron: '0 4 * * *'
|
||||
|
||||
# only 1 job running in the repo at any time
|
||||
concurrency:
|
||||
group: lock
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
jobs:
|
||||
no-response:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: lee-dohm/no-response@9bb0a4b5e6a45046f00353d5de7d90fb8bd773bb # v0.5.0
|
||||
with:
|
||||
closeComment: >
|
||||
This issue has been automatically closed
|
||||
because it has been awaiting a response for too long.
|
||||
|
||||
When you have time to to work with the maintainers to resolve this issue,
|
||||
please post a new comment and it will be re-opened.
|
||||
If the issue has been locked for editing by the time you return to it,
|
||||
please open a new issue and reference this one.
|
||||
|
||||
Thank you for taking the time to improve LightGBM!
|
||||
daysUntilClose: 30
|
||||
responseRequiredLabel: awaiting response
|
||||
token: ${{ github.token }}
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Optional checks
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
jobs:
|
||||
all-optional-checks-successful:
|
||||
timeout-minutes: 30
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: false
|
||||
- name: Check valgrind workflow
|
||||
shell: bash
|
||||
run: |
|
||||
# ref: https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#github-context
|
||||
PR_BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
|
||||
echo "checking status for branch '${PR_BRANCH}'"
|
||||
./.ci/check-workflow-status.sh \
|
||||
"${PR_BRANCH}" \
|
||||
'r_valgrind.yml' \
|
||||
${{ github.event.number }}
|
||||
@@ -0,0 +1,534 @@
|
||||
name: Python-package
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
env:
|
||||
# tell scripts where to put artifacts
|
||||
# (this variable name is left over from when jobs ran on Azure DevOps)
|
||||
BUILD_ARTIFACTSTAGINGDIRECTORY: '${{ github.workspace }}/artifacts'
|
||||
# where repo sources are cloned to
|
||||
BUILD_SOURCESDIRECTORY: '${{ github.workspace }}'
|
||||
CMAKE_BUILD_PARALLEL_LEVEL: 4
|
||||
# avoid interactive prompts in Debian-based distributions
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
# On runners using nvidia-container-runtime with Docker, ensure GPUs are available to running processes.
|
||||
#
|
||||
# ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html
|
||||
NVIDIA_VISIBLE_DEVICES: 'all'
|
||||
SKBUILD_STRICT_CONFIG: true
|
||||
|
||||
jobs:
|
||||
test-linux-aarch64:
|
||||
name: bdist wheel (manylinux2014-arm, gcc, Python 3.14)
|
||||
runs-on: ubuntu-24.04-arm
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Setup and run tests
|
||||
shell: bash
|
||||
# this uses 'docker run' instead of just setting 'container:'
|
||||
# because actions/checkout requires GLIBC 2.28 and that is too
|
||||
# new for manylinux2014
|
||||
env:
|
||||
BUILD_DIRECTORY: /LightGBM
|
||||
run: |
|
||||
cat > ./docker-script.sh <<EOF
|
||||
mkdir -p \$BUILD_ARTIFACTSTAGINGDIRECTORY
|
||||
export CONDA=\$HOME/miniforge
|
||||
export PATH=\$CONDA/bin:\$PATH
|
||||
${{ env.BUILD_DIRECTORY }}/.ci/setup.sh || exit 1
|
||||
${{ env.BUILD_DIRECTORY }}/.ci/test.sh || exit 1
|
||||
EOF
|
||||
IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
|
||||
docker pull "${IMAGE_URI}" || exit 1
|
||||
docker run \
|
||||
--platform "${PLATFORM}" \
|
||||
--rm \
|
||||
--env BUILD_DIRECTORY=${{ env.BUILD_DIRECTORY }} \
|
||||
--env BUILD_ARTIFACTSTAGINGDIRECTORY=${{ env.BUILD_DIRECTORY }}/artifacts/ \
|
||||
--env COMPILER=gcc \
|
||||
--env METHOD=wheel \
|
||||
--env OS_NAME=linux \
|
||||
--env PRODUCES_ARTIFACTS=true \
|
||||
--env PYTHON_VERSION="3.14" \
|
||||
--env TASK=bdist \
|
||||
-v "${PWD}":"${{ env.BUILD_DIRECTORY }}" \
|
||||
-w ${{ env.BUILD_DIRECTORY }} \
|
||||
"${IMAGE_URI}" \
|
||||
/bin/bash ./docker-script.sh
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: linux-aarch64-wheel
|
||||
path: artifacts/*.whl
|
||||
if-no-files-found: error
|
||||
test:
|
||||
# yamllint disable-line rule:line-length
|
||||
name: ${{ matrix.task }} ${{ matrix.method }} (${{ matrix.os-display-name || matrix.os }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
container: ${{ matrix.container }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-15-intel
|
||||
task: sdist
|
||||
compiler: clang
|
||||
python_version: '3.10'
|
||||
pixi-env: 'py310'
|
||||
- os: windows-2022
|
||||
task: sdist
|
||||
compiler: MSVC
|
||||
python_version: '3.10'
|
||||
pixi-env: 'py310'
|
||||
- os: ubuntu-latest
|
||||
container: &ubuntu_latest_image ubuntu:22.04
|
||||
os-display-name: &ubuntu_latest_display_name 'ubuntu22.04'
|
||||
task: regular
|
||||
compiler: clang-17
|
||||
python_version: '3.13'
|
||||
in-ubuntu-base-container: 'true'
|
||||
- os: ubuntu-latest
|
||||
container: *ubuntu_latest_image
|
||||
os-display-name: *ubuntu_latest_display_name
|
||||
task: sdist
|
||||
compiler: clang-17
|
||||
python_version: '3.14'
|
||||
in-ubuntu-base-container: 'true'
|
||||
#########################
|
||||
# OpenCL-based GPU jobs #
|
||||
#########################
|
||||
# clang-17 jobs are run on machines with actual GPUs because
|
||||
# the package built with that compiler toolchain can't target the
|
||||
# CPUs on GitHub's runners.
|
||||
#
|
||||
# ref: https://github.com/lightgbm-org/LightGBM/pull/7096/files#r2590879590
|
||||
# TODO(jameslamb): restore these GPU jobs when we've clarified handling of GPU CI
|
||||
# ref: https://github.com/lightgbm-org/LightGBM/issues/7187
|
||||
# - os: github-hosted-linux-gpu-amd64
|
||||
# container: nvcr.io/nvidia/cuda:12.9.1-devel-ubuntu22.04
|
||||
# os-display-name: ubuntu22.04-with-gpu
|
||||
# task: bdist
|
||||
# compiler: clang-17
|
||||
# method: wheel
|
||||
# python_version: '3.11'
|
||||
# in-ubuntu-base-container: 'true'
|
||||
# - os: github-hosted-linux-gpu-amd64
|
||||
# container: nvcr.io/nvidia/cuda:12.9.1-devel-ubuntu22.04
|
||||
# os-display-name: ubuntu22.04-with-gpu
|
||||
# task: gpu
|
||||
# compiler: clang-17
|
||||
# method: source
|
||||
# python_version: '3.12'
|
||||
# in-ubuntu-base-container: 'true'
|
||||
- os: ubuntu-latest
|
||||
container: &manylinux_amd64_image ghcr.io/lightgbm-org/lightgbm-vsts-agent:manylinux_2_28_x86_64
|
||||
os-display-name: &manylinux_amd64_display_name 'manylinux_2_28'
|
||||
task: gpu
|
||||
method: source
|
||||
compiler: gcc
|
||||
python_version: '3.14'
|
||||
in-ubuntu-base-container: 'false'
|
||||
setup-conda: 'false'
|
||||
############
|
||||
# MPI jobs #
|
||||
############
|
||||
- os: macos-15-intel
|
||||
task: mpi
|
||||
compiler: gcc
|
||||
method: source
|
||||
python_version: '3.12'
|
||||
- os: ubuntu-latest
|
||||
container: *manylinux_amd64_image
|
||||
os-display-name: *manylinux_amd64_display_name
|
||||
task: mpi
|
||||
compiler: gcc
|
||||
method: source
|
||||
python_version: '3.11'
|
||||
in-ubuntu-base-container: 'false'
|
||||
setup-conda: 'false'
|
||||
- os: ubuntu-latest
|
||||
container: *ubuntu_latest_image
|
||||
os-display-name: *ubuntu_latest_display_name
|
||||
task: mpi
|
||||
compiler: clang-17
|
||||
method: source
|
||||
python_version: '3.13'
|
||||
in-ubuntu-base-container: 'true'
|
||||
- os: ubuntu-latest
|
||||
container: *ubuntu_latest_image
|
||||
os-display-name: *ubuntu_latest_display_name
|
||||
task: mpi
|
||||
compiler: clang-17
|
||||
method: pip
|
||||
python_version: '3.12'
|
||||
in-ubuntu-base-container: 'true'
|
||||
- os: ubuntu-latest
|
||||
container: *ubuntu_latest_image
|
||||
os-display-name: *ubuntu_latest_display_name
|
||||
task: mpi
|
||||
compiler: clang-17
|
||||
method: wheel
|
||||
python_version: '3.10'
|
||||
pixi-env: 'py310'
|
||||
in-ubuntu-base-container: 'true'
|
||||
#####################
|
||||
# jobs that create #
|
||||
# release artifacts #
|
||||
#####################
|
||||
- os: macos-15-intel
|
||||
task: bdist
|
||||
compiler: clang
|
||||
method: wheel
|
||||
produces-artifacts: 'true'
|
||||
artifact-name: 'macosx-amd64-wheel'
|
||||
python_version: '3.13'
|
||||
- os: macos-14
|
||||
task: bdist
|
||||
compiler: clang
|
||||
method: wheel
|
||||
produces-artifacts: 'true'
|
||||
artifact-name: 'macosx-arm64-wheel'
|
||||
python_version: '3.11'
|
||||
- os: macos-15-intel
|
||||
task: regular
|
||||
compiler: clang
|
||||
produces-artifacts: 'true'
|
||||
artifact-name: 'macosx-amd64-liblightgbm'
|
||||
python_version: '3.11'
|
||||
- os: ubuntu-latest
|
||||
container: *manylinux_amd64_image
|
||||
os-display-name: *manylinux_amd64_display_name
|
||||
task: regular
|
||||
compiler: gcc
|
||||
python_version: '3.11'
|
||||
in-ubuntu-base-container: 'false'
|
||||
setup-conda: 'false'
|
||||
produces-artifacts: 'true'
|
||||
artifact-name: 'linux-amd64-liblightgbm'
|
||||
- os: ubuntu-latest
|
||||
container: *manylinux_amd64_image
|
||||
os-display-name: *manylinux_amd64_display_name
|
||||
task: bdist
|
||||
compiler: gcc
|
||||
method: wheel
|
||||
python_version: '3.10'
|
||||
pixi-env: 'py310'
|
||||
in-ubuntu-base-container: 'false'
|
||||
setup-conda: 'false'
|
||||
produces-artifacts: 'true'
|
||||
artifact-name: 'linux-amd64-wheel'
|
||||
- os: ubuntu-latest
|
||||
container: *manylinux_amd64_image
|
||||
os-display-name: *manylinux_amd64_display_name
|
||||
task: sdist
|
||||
compiler: gcc
|
||||
python_version: '3.10'
|
||||
pixi-env: 'py310'
|
||||
in-ubuntu-base-container: 'false'
|
||||
setup-conda: 'false'
|
||||
produces-artifacts: 'true'
|
||||
artifact-name: 'sdist'
|
||||
- os: windows-2022
|
||||
task: bdist
|
||||
compiler: MSVC
|
||||
method: wheel
|
||||
produces-artifacts: 'true'
|
||||
artifact-name: 'windows-amd64-wheel'
|
||||
python_version: '3.13'
|
||||
- os: windows-2022
|
||||
task: regular
|
||||
compiler: MSVC
|
||||
produces-artifacts: 'true'
|
||||
artifact-name: 'windows-amd64-cli-and-liblightgbm'
|
||||
python_version: '3.11'
|
||||
steps:
|
||||
- name: Install packages used by third-party actions
|
||||
if: ${{ matrix.in-ubuntu-base-container == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
apt-get update -y
|
||||
apt-get install --no-install-recommends -y \
|
||||
ca-certificates \
|
||||
dirmngr \
|
||||
gpg \
|
||||
gpg-agent \
|
||||
software-properties-common \
|
||||
sudo
|
||||
# install newest version of git
|
||||
# ref:
|
||||
# - https://unix.stackexchange.com/a/170831/550004
|
||||
# - https://git-scm.com/download/linux
|
||||
add-apt-repository ppa:git-core/ppa -y
|
||||
apt-get update -y
|
||||
apt-get install --no-install-recommends -y \
|
||||
git
|
||||
- name: Trust git cloning LightGBM
|
||||
if: startsWith(matrix.os, 'ubuntu')
|
||||
run: |
|
||||
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Set up 'pixi' for jobs that need it
|
||||
if: matrix.pixi-env != ''
|
||||
uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
|
||||
with:
|
||||
environments: ${{ matrix.pixi-env }}
|
||||
- name: Setup and run tests on Linux and macOS
|
||||
if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu') || endsWith(matrix.os, 'gpu-amd64')
|
||||
shell: bash
|
||||
run: |
|
||||
export COMPILER="${{ matrix.compiler }}"
|
||||
export IN_UBUNTU_BASE_CONTAINER="${{ matrix.in-ubuntu-base-container }}"
|
||||
export SETUP_CONDA="${{ matrix.setup-conda }}"
|
||||
export TASK="${{ matrix.task }}"
|
||||
export METHOD="${{ matrix.method }}"
|
||||
if [[ "${{ matrix.os }}" =~ ^macos ]]; then
|
||||
export OS_NAME="macos"
|
||||
elif \
|
||||
[[ "${{ matrix.os }}" == "ubuntu-latest" ]] \
|
||||
|| [[ "${{ matrix.os }}" == "github-hosted-linux-gpu-amd64" ]];
|
||||
then
|
||||
export OS_NAME="linux"
|
||||
export PATH="/usr/lib64/openmpi/bin:${PATH}"
|
||||
fi
|
||||
export PYTHON_VERSION="${{ matrix.python_version }}"
|
||||
export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
|
||||
# some pre-built images already have 'CONDA' set
|
||||
if [[ -z "${CONDA:-}" ]]; then
|
||||
export CONDA=${HOME}/miniforge
|
||||
fi
|
||||
export PATH=${CONDA}/bin:${PATH}
|
||||
export PRODUCES_ARTIFACTS="${{ matrix.produces-artifacts }}"
|
||||
$GITHUB_WORKSPACE/.ci/setup.sh || exit 1
|
||||
$GITHUB_WORKSPACE/.ci/test.sh || exit 1
|
||||
- name: Install OpenCL on Windows
|
||||
if: startsWith(matrix.os, 'windows') && (matrix.task == 'bdist')
|
||||
shell: pwsh -command ". {0}"
|
||||
run: |
|
||||
& "$env:GITHUB_WORKSPACE/.ci/install-opencl.ps1"
|
||||
# 'conda init powershell' needs to be run in a separate process
|
||||
# ref: https://docs.conda.io/projects/conda/en/stable/dev-guide/deep-dives/activation.html
|
||||
- name: Initialize conda on Windows
|
||||
if: startsWith(matrix.os, 'windows')
|
||||
shell: pwsh -command ". {0}"
|
||||
run: |
|
||||
$env:PATH = "$env:CONDA/Scripts;$env:PATH"
|
||||
conda init powershell
|
||||
- name: Setup and run tests on Windows
|
||||
if: startsWith(matrix.os, 'windows')
|
||||
shell: pwsh -command ". {0}"
|
||||
run: |
|
||||
$env:COMPILER = "${{ matrix.compiler }}"
|
||||
$env:METHOD = "${{ matrix.method }}"
|
||||
$env:PRODUCES_ARTIFACTS = "${{ matrix.produces-artifacts }}"
|
||||
$env:PYTHON_VERSION = "${{ matrix.python_version }}"
|
||||
$env:TASK = "${{ matrix.task }}"
|
||||
& "$env:GITHUB_WORKSPACE/.ci/test-windows.ps1"
|
||||
- name: Upload artifacts
|
||||
if: ${{ matrix.produces-artifacts == 'true' }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ matrix.artifact-name }}
|
||||
path: |
|
||||
artifacts/*.dll
|
||||
artifacts/*.dylib
|
||||
artifacts/*.exe
|
||||
artifacts/*.so
|
||||
artifacts/*.tar.gz
|
||||
artifacts/*.whl
|
||||
if-no-files-found: error
|
||||
create-nuget:
|
||||
name: NuGet package
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs:
|
||||
- test
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: false
|
||||
- name: Download lib_lightgbm (all operating systems) and Windows CLI
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
merge-multiple: true
|
||||
path: ${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}
|
||||
pattern: '*amd64*-liblightgbm'
|
||||
run-id: ${{ github.run_id }}
|
||||
- name: Setup NuGet CLI
|
||||
uses: NuGet/setup-nuget@fd55a6f3b34392fa83fde1454582407d8c714123 # v4.0
|
||||
# ref: https://github.com/NuGet/setup-nuget/issues/168
|
||||
- name: Setup mono
|
||||
run: |
|
||||
sudo apt-get update -y
|
||||
sudo apt-get install --no-install-recommends -y \
|
||||
mono-devel
|
||||
- name: Create NuGet package
|
||||
run: |
|
||||
python .ci/create-nuget.py "${BUILD_ARTIFACTSTAGINGDIRECTORY}"
|
||||
nuget pack \
|
||||
$(pwd)/.ci/nuget/LightGBM.nuspec \
|
||||
-NonInteractive \
|
||||
-Verbosity detailed \
|
||||
-OutputDirectory "${BUILD_ARTIFACTSTAGINGDIRECTORY}"
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: nuget-package
|
||||
path: artifacts/*.nupkg
|
||||
if-no-files-found: error
|
||||
|
||||
test-latest-versions:
|
||||
name: Python - latest versions (manylinux_2_28)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Create wheel
|
||||
run: |
|
||||
docker run \
|
||||
--rm \
|
||||
--env CMAKE_BUILD_PARALLEL_LEVEL=${{ env.CMAKE_BUILD_PARALLEL_LEVEL }} \
|
||||
-v $(pwd):/opt/lgb-build \
|
||||
-w /opt/lgb-build \
|
||||
lightgbm/vsts-agent:manylinux_2_28_x86_64 \
|
||||
/bin/bash -c 'PATH=/opt/miniforge/bin:$PATH sh ./build-python.sh bdist_wheel --nomp'
|
||||
- name: Test compatibility
|
||||
run: |
|
||||
docker run \
|
||||
--rm \
|
||||
-v $(pwd):/opt/lgb-build \
|
||||
-w /opt/lgb-build \
|
||||
python:3.14 \
|
||||
/bin/bash ./.ci/test-python-latest.sh
|
||||
|
||||
test-old-versions:
|
||||
name: Python - oldest supported versions (manylinux_2_28, Python ${{ matrix.python_version }})
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# This should always include at least the oldest
|
||||
# not-yet-end-of-life Python version
|
||||
python_version:
|
||||
- '3.10'
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Create wheel
|
||||
run: |
|
||||
docker run \
|
||||
--rm \
|
||||
--env CMAKE_BUILD_PARALLEL_LEVEL=${{ env.CMAKE_BUILD_PARALLEL_LEVEL }} \
|
||||
-v $(pwd):/opt/lgb-build \
|
||||
-w /opt/lgb-build \
|
||||
lightgbm/vsts-agent:manylinux_2_28_x86_64 \
|
||||
/bin/bash -c 'PATH=/opt/miniforge/bin:$PATH sh ./build-python.sh bdist_wheel --nomp'
|
||||
- name: Test compatibility
|
||||
run: |
|
||||
docker run \
|
||||
--rm \
|
||||
-v $(pwd):/opt/lgb-build \
|
||||
-w /opt/lgb-build \
|
||||
python:${{ matrix.python_version }} \
|
||||
/bin/bash ./.ci/test-python-oldest.sh
|
||||
|
||||
upload-nightlies:
|
||||
needs:
|
||||
- test
|
||||
- test-linux-aarch64
|
||||
runs-on: ubuntu-latest
|
||||
# publish whenever a new commit is pushed to 'main'
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
environment:
|
||||
name: nightly-python-uploads
|
||||
url: https://anaconda.org/lightgbm-packages/lightgbm
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: setup
|
||||
run: |
|
||||
mkdir -p "${BUILD_ARTIFACTSTAGINGDIRECTORY}"
|
||||
pip install --prefer-binary \
|
||||
anaconda-auth \
|
||||
anaconda-client
|
||||
- name: download wheels
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
merge-multiple: true
|
||||
path: ${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}
|
||||
pattern: '*-wheel'
|
||||
run-id: ${{ github.run_id }}
|
||||
- name: download sdists
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: sdist
|
||||
path: ${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}
|
||||
run-id: ${{ github.run_id }}
|
||||
- name: upload to anaconda.org
|
||||
env:
|
||||
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN_LIGHTGBM_PACKAGES }}
|
||||
run: |
|
||||
anaconda upload \
|
||||
--package lightgbm \
|
||||
--force \
|
||||
-t pypi \
|
||||
${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbm-*.tar.gz \
|
||||
${BUILD_ARTIFACTSTAGINGDIRECTORY}/*.whl
|
||||
|
||||
all-python-package-jobs-successful:
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- test-latest-versions
|
||||
- test
|
||||
- test-linux-aarch64
|
||||
- test-old-versions
|
||||
permissions:
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Note that all tests succeeded
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
@@ -0,0 +1,60 @@
|
||||
name: R generate configure
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr-branch:
|
||||
type: string
|
||||
description: |
|
||||
Branch in lightgbm-org/LightGBM to update.
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
jobs:
|
||||
r-configure:
|
||||
name: r-configure
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
container: "ubuntu:22.04"
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Install essential software before checkout
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install --no-install-recommends -y \
|
||||
ca-certificates \
|
||||
git
|
||||
- name: Trust git cloning LightGBM
|
||||
run: |
|
||||
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
submodules: false
|
||||
repository: lightgbm-org/LightGBM
|
||||
ref: "refs/heads/${{ inputs.pr-branch }}"
|
||||
token: ${{ github.token }}
|
||||
persist-credentials: true
|
||||
- name: Update configure
|
||||
shell: bash
|
||||
run: ./R-package/recreate-configure.sh || exit 1
|
||||
- name: Push changes
|
||||
run: |
|
||||
# source for this user and email: https://github.com/orgs/community/discussions/160496
|
||||
git config --global user.name "github-actions[bot]"
|
||||
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add "./R-package/configure"
|
||||
git commit --allow-empty -m "Auto-update configure"
|
||||
git push
|
||||
@@ -0,0 +1,395 @@
|
||||
name: R-package
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
env:
|
||||
# tell scripts where to put artifacts
|
||||
# (this variable name is left over from when jobs ran on Azure DevOps)
|
||||
BUILD_ARTIFACTSTAGINGDIRECTORY: '${{ github.workspace }}/artifacts'
|
||||
# in CMake-driven builds, parallelize compilation
|
||||
CMAKE_BUILD_PARALLEL_LEVEL: 4
|
||||
# on Debian-based images, avoid interactive prompts
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
# Fix issues like the following that can show up running 'R CMD check' on
|
||||
# specific clang versions:
|
||||
#
|
||||
# * checking for detritus in the temp directory ... NOTE
|
||||
# Found the following files/directories:
|
||||
# ‘dsymutil-63923a’ ‘dsymutil-9aa721’ ‘dsymutil-b7e1bb’
|
||||
#
|
||||
# These are unlikely to show up in CRAN's checks. They come from
|
||||
# 'dsymutil ---gen-reproducer' being run (not something LightGBM explicitly does).
|
||||
#
|
||||
# ref:
|
||||
# - https://github.com/llvm/llvm-project/issues/61920
|
||||
# - https://github.com/golang/go/issues/59026#issuecomment-1520487072
|
||||
DSYMUTIL_REPRODUCER_PATH: /dev/null
|
||||
# parallelize compilation (extra important for Linux, where CRAN doesn't supply pre-compiled binaries)
|
||||
MAKEFLAGS: "-j4"
|
||||
# hack to get around this:
|
||||
# https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html
|
||||
_R_CHECK_SYSTEM_CLOCK_: 0
|
||||
# ignore R CMD CHECK NOTE checking how long it has
|
||||
# been since the last submission
|
||||
_R_CHECK_CRAN_INCOMING_REMOTE_: 0
|
||||
# CRAN ignores the "installed size is too large" NOTE,
|
||||
# so our CI can too. Setting to a large value here just
|
||||
# to catch extreme problems
|
||||
_R_CHECK_PKG_SIZES_THRESHOLD_: 100
|
||||
|
||||
jobs:
|
||||
test:
|
||||
# yamllint disable-line rule:line-length
|
||||
name: ${{ matrix.task }} (${{ matrix.os-display-name || matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
container: ${{ matrix.container }}
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
################
|
||||
# CMake builds #
|
||||
################
|
||||
- os: ubuntu-latest
|
||||
task: r-package
|
||||
compiler: gcc
|
||||
r_version: 4.3
|
||||
build_type: cmake
|
||||
container: 'ubuntu:22.04'
|
||||
os-display-name: 'ubuntu22.04'
|
||||
- os: ubuntu-latest
|
||||
task: r-package
|
||||
compiler: clang
|
||||
r_version: 4.3
|
||||
build_type: cmake
|
||||
container: 'ubuntu:22.04'
|
||||
os-display-name: 'ubuntu22.04'
|
||||
- os: macos-15-intel
|
||||
task: r-package
|
||||
compiler: clang
|
||||
r_version: 4.3
|
||||
build_type: cmake
|
||||
container: null
|
||||
- os: windows-latest
|
||||
task: r-package
|
||||
compiler: MINGW
|
||||
toolchain: MSYS
|
||||
r_version: 4.3
|
||||
build_type: cmake
|
||||
container: null
|
||||
# Visual Studio 2022
|
||||
- os: windows-2022
|
||||
task: r-package
|
||||
compiler: MSVC
|
||||
toolchain: MSVC
|
||||
r_version: 4.3
|
||||
build_type: cmake
|
||||
container: null
|
||||
###############
|
||||
# CRAN builds #
|
||||
###############
|
||||
- os: windows-latest
|
||||
task: r-package
|
||||
compiler: MINGW
|
||||
toolchain: MSYS
|
||||
r_version: 4.3
|
||||
build_type: cran
|
||||
container: null
|
||||
- os: ubuntu-latest
|
||||
task: r-package
|
||||
compiler: gcc
|
||||
r_version: 4.3
|
||||
build_type: cran
|
||||
container: 'ubuntu:22.04'
|
||||
os-display-name: 'ubuntu22.04'
|
||||
produces-artifacts: 'true'
|
||||
artifact-name: r-cran-package
|
||||
- os: macos-15-intel
|
||||
task: r-package
|
||||
compiler: clang
|
||||
r_version: 4.3
|
||||
build_type: cran
|
||||
container: null
|
||||
# macos-14 = arm64
|
||||
- os: macos-14
|
||||
task: r-package
|
||||
compiler: clang
|
||||
r_version: 4.3
|
||||
build_type: cran
|
||||
container: null
|
||||
steps:
|
||||
- name: Prevent conversion of line endings on Windows
|
||||
if: startsWith(matrix.os, 'windows')
|
||||
shell: pwsh
|
||||
run: git config --global core.autocrlf false
|
||||
- name: Install packages used by third-party actions
|
||||
if: startsWith(matrix.os, 'ubuntu')
|
||||
shell: bash
|
||||
run: |
|
||||
apt-get update -y
|
||||
apt-get install --no-install-recommends -y \
|
||||
ca-certificates \
|
||||
dirmngr \
|
||||
gpg \
|
||||
gpg-agent \
|
||||
software-properties-common \
|
||||
sudo
|
||||
# install newest version of git
|
||||
# ref:
|
||||
# - https://unix.stackexchange.com/a/170831/550004
|
||||
# - https://git-scm.com/download/linux
|
||||
add-apt-repository ppa:git-core/ppa -y
|
||||
apt-get update -y
|
||||
apt-get install --no-install-recommends -y \
|
||||
git
|
||||
- name: Trust git cloning LightGBM
|
||||
if: startsWith(matrix.os, 'ubuntu')
|
||||
run: |
|
||||
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Install pandoc
|
||||
uses: r-lib/actions/setup-pandoc@a51a8012b0aab7c32ef9d19bf54da93f3254335e # v2.12.0
|
||||
- name: Install tinytex
|
||||
if: startsWith(matrix.os, 'windows')
|
||||
uses: r-lib/actions/setup-tinytex@a51a8012b0aab7c32ef9d19bf54da93f3254335e # v2.12.0
|
||||
env:
|
||||
CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex
|
||||
TINYTEX_INSTALLER: TinyTeX
|
||||
- name: Setup and run tests on Linux and macOS
|
||||
if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu')
|
||||
shell: bash
|
||||
run: |
|
||||
export TASK="${{ matrix.task }}"
|
||||
export COMPILER="${{ matrix.compiler }}"
|
||||
if [[ "${{ matrix.os }}" =~ ^macos ]]; then
|
||||
export OS_NAME="macos"
|
||||
elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
|
||||
export OS_NAME="linux"
|
||||
export IN_UBUNTU_BASE_CONTAINER="true"
|
||||
fi
|
||||
export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
|
||||
export LGB_VER=$(head -n 1 "${BUILD_DIRECTORY}/VERSION.txt")
|
||||
export PRODUCES_ARTIFACTS="${{ matrix.produces-artifacts }}"
|
||||
export R_VERSION="${{ matrix.r_version }}"
|
||||
export R_BUILD_TYPE="${{ matrix.build_type }}"
|
||||
$GITHUB_WORKSPACE/.ci/setup.sh
|
||||
$GITHUB_WORKSPACE/.ci/test.sh
|
||||
- name: Setup and run tests on Windows
|
||||
if: startsWith(matrix.os, 'windows')
|
||||
shell: pwsh -command ". {0}"
|
||||
run: |
|
||||
$env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE
|
||||
$env:LGB_VER = (Get-Content -TotalCount 1 $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim().replace('rc', '-')
|
||||
$env:TOOLCHAIN = "${{ matrix.toolchain }}"
|
||||
$env:R_VERSION = "${{ matrix.r_version }}"
|
||||
$env:R_BUILD_TYPE = "${{ matrix.build_type }}"
|
||||
$env:COMPILER = "${{ matrix.compiler }}"
|
||||
$env:TASK = "${{ matrix.task }}"
|
||||
& "$env:GITHUB_WORKSPACE/.ci/test-windows.ps1"
|
||||
- name: Upload artifacts
|
||||
if: ${{ matrix.produces-artifacts == 'true' }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ matrix.artifact-name }}
|
||||
path: ${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}/*.tar.gz
|
||||
if-no-files-found: error
|
||||
test-r-sanitizers:
|
||||
name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN)
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
container: wch1/r-debug # zizmor: ignore[unpinned-images]
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- r_customization: san
|
||||
compiler: gcc
|
||||
- r_customization: csan
|
||||
compiler: clang
|
||||
steps:
|
||||
- name: Trust git cloning LightGBM
|
||||
run: |
|
||||
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Install packages
|
||||
shell: bash
|
||||
run: |
|
||||
RDscript${{ matrix.r_customization }} ./.ci/install-r-deps.R --build --test
|
||||
sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }}
|
||||
RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1
|
||||
- name: Run tests with sanitizers
|
||||
shell: bash
|
||||
run: |
|
||||
cd R-package/tests
|
||||
exit_code=0
|
||||
RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1
|
||||
cat ./tests.log
|
||||
exit ${exit_code}
|
||||
test-r-extra-checks:
|
||||
name: r-package (${{ matrix.image }}, R-devel)
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# references:
|
||||
# * CRAN "additional checks": https://cran.r-project.org/web/checks/check_issue_kinds.html
|
||||
# * images: https://r-hub.github.io/containers/containers.html
|
||||
#
|
||||
# Generally, only images that are still supported, listed at
|
||||
# https://r-hub.github.io/containers/#available-containers, should be used.
|
||||
image:
|
||||
- clang21
|
||||
- clang22
|
||||
- gcc16
|
||||
- rchk
|
||||
- ubuntu-clang
|
||||
- ubuntu-gcc12
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/r-hub/containers/${{ matrix.image }}:latest # zizmor: ignore[unpinned-images]
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Install pandoc
|
||||
uses: r-lib/actions/setup-pandoc@a51a8012b0aab7c32ef9d19bf54da93f3254335e # v2.12.0
|
||||
- name: Install system dependencies
|
||||
shell: bash
|
||||
run: |
|
||||
# libuv is needed for {fs} (https://github.com/lightgbm-org/LightGBM/issues/7208)
|
||||
if type -f apt 2>&1 > /dev/null; then
|
||||
apt-get update
|
||||
apt-get install --no-install-recommends -y \
|
||||
cmake \
|
||||
devscripts \
|
||||
libcurl4-openssl-dev \
|
||||
libuv1-dev \
|
||||
texinfo \
|
||||
texlive-latex-extra \
|
||||
texlive-latex-recommended \
|
||||
texlive-fonts-recommended \
|
||||
texlive-fonts-extra \
|
||||
tidy \
|
||||
qpdf
|
||||
else
|
||||
yum update -y
|
||||
yum install -y \
|
||||
cmake \
|
||||
devscripts \
|
||||
libcurl-devel \
|
||||
libuv-devel \
|
||||
qpdf \
|
||||
texinfo \
|
||||
texinfo-tex \
|
||||
texlive-amsmath \
|
||||
texlive-amsfonts \
|
||||
texlive-collection-fontsrecommended \
|
||||
texlive-collection-latexrecommended \
|
||||
texlive-latex \
|
||||
tidy
|
||||
fi
|
||||
- name: Install packages and run tests
|
||||
shell: bash
|
||||
run: |
|
||||
Rscript ./.ci/install-r-deps.R --build --test --exclude=testthat
|
||||
sh build-cran-package.sh
|
||||
|
||||
# 'rchk' isn't run through 'R CMD check', use the approach documented at
|
||||
# https://r-hub.github.io/containers/local.html
|
||||
if [[ "${{ matrix.image }}" =~ "rchk" ]]; then
|
||||
r-check "$(pwd)" \
|
||||
| tee ./rchk-logs.txt 2>&1
|
||||
|
||||
# the '-v' exceptions below are from R/rchk itself and not LightGBM:
|
||||
# https://github.com/kalibera/rchk/issues/22#issuecomment-656036156
|
||||
if grep -E '\[PB\]|ERROR' ./rchk-logs.txt \
|
||||
| grep -v 'too many states' \
|
||||
> /dev/null; \
|
||||
then
|
||||
echo "rchk found issues"
|
||||
exit 1
|
||||
else
|
||||
echo "rchk did not find any issues"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# why these packages?
|
||||
#
|
||||
# - 'curl' is not a dependency of {lightgbm}, but it's needed for 'R CMD check' URL checks
|
||||
# - 'testthat' is not needed by 'rchk', so avoid installing it until here
|
||||
#
|
||||
Rscript -e "
|
||||
install.packages(c('curl', 'testthat'),
|
||||
repos = 'https://cran.rstudio.com',
|
||||
Ncpus = parallel::detectCores())
|
||||
"
|
||||
|
||||
if [[ "${{ matrix.image }}" =~ "clang" ]]; then
|
||||
# allowing the following NOTEs (produced by default in the clang images):
|
||||
#
|
||||
# * checking compilation flags used ... NOTE
|
||||
# Compilation used the following non-portable flag(s):
|
||||
# ‘-Wp,-D_FORTIFY_SOURCE=3’
|
||||
#
|
||||
# even though CRAN itself sets that:
|
||||
# https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang
|
||||
#
|
||||
declare -i allowed_notes=1
|
||||
else
|
||||
declare -i allowed_notes=0
|
||||
fi
|
||||
|
||||
bash .ci/run-r-cmd-check.sh \
|
||||
"$(echo lightgbm_$(head -1 VERSION.txt).tar.gz)" \
|
||||
"${allowed_notes}"
|
||||
all-r-package-jobs-successful:
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- test
|
||||
- test-r-sanitizers
|
||||
- test-r-extra-checks
|
||||
permissions:
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Note that all tests succeeded
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
@@ -0,0 +1,106 @@
|
||||
name: R valgrind tests
|
||||
|
||||
# 'run-name' is used here to distinguish in the API between different runs from forks.
|
||||
#
|
||||
# When this was added, it was the only way to feed workflow inputs into something that
|
||||
# would show up in the output of 'gh run list'.
|
||||
#
|
||||
# See https://github.com/orgs/community/discussions/73223#discussioncomment-11862624
|
||||
run-name: R valgrind tests (pr=${{ inputs.pr-number }})
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr-branch:
|
||||
type: string
|
||||
description: |
|
||||
Branch the PR was submitted from.
|
||||
Branches from forks should be prefixed with the user/org they originate from,
|
||||
like '{user}:{branch}'.
|
||||
pr-number:
|
||||
type: string
|
||||
description: Pull request ID, found in the PR URL.
|
||||
|
||||
jobs:
|
||||
test-r-valgrind:
|
||||
name: r-package (ubuntu-latest, R-devel, valgrind)
|
||||
timeout-minutes: 360
|
||||
runs-on: ubuntu-latest
|
||||
container: wch1/r-debug # zizmor: ignore[unpinned-images]
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
permissions:
|
||||
actions: write
|
||||
checks: write
|
||||
contents: read
|
||||
id-token: write
|
||||
pull-requests: write
|
||||
statuses: write
|
||||
steps:
|
||||
- name: Install essential software before checkout
|
||||
shell: bash
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install --no-install-recommends -y \
|
||||
curl \
|
||||
jq
|
||||
- name: Install GitHub CLI
|
||||
run: |
|
||||
GH_CLI_VERSION="2.83.0"
|
||||
curl \
|
||||
--fail \
|
||||
-O \
|
||||
-L \
|
||||
https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/gh_${GH_CLI_VERSION}_linux_amd64.tar.gz
|
||||
tar -xvf ./gh_${GH_CLI_VERSION}_linux_amd64.tar.gz
|
||||
mv ./gh_${GH_CLI_VERSION}_linux_amd64/bin/gh /usr/local/bin/
|
||||
- name: Trust git cloning LightGBM
|
||||
run: |
|
||||
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
submodules: true
|
||||
persist-credentials: false
|
||||
repository: lightgbm-org/LightGBM
|
||||
ref: "refs/pull/${{ inputs.pr-number }}/merge"
|
||||
- name: Run tests with valgrind
|
||||
shell: bash
|
||||
run: ./.ci/test-r-package-valgrind.sh
|
||||
- name: Send final status
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
GITHUB__WORKFLOW: ${{ github.workflow }}
|
||||
INPUTS__PR_NUMBER: ${{ inputs.pr-number }}
|
||||
JOB__STATUS: ${{ job.status }}
|
||||
run: |
|
||||
$GITHUB_WORKSPACE/.ci/set-commit-status.sh \
|
||||
"${GITHUB_WORKFLOW}" \
|
||||
"${JOB__STATUS}" \
|
||||
"${{ github.sha }}"
|
||||
comment="Workflow **${GITHUB__WORKFLOW}** has been triggered! 🚀\r\n"
|
||||
comment="${comment}\r\n${GITHUB_SERVER_URL}/lightgbm-org/LightGBM/actions/runs/${GITHUB_RUN_ID} \r\n"
|
||||
comment="${comment}\r\nStatus: ${JOB__STATUS}"
|
||||
$GITHUB_WORKSPACE/.ci/append-comment.sh \
|
||||
"${INPUTS__PR_NUMBER}" \
|
||||
"${comment}"
|
||||
- name: Rerun workflow-indicator
|
||||
if: ${{ always() }}
|
||||
env:
|
||||
INPUTS__PR_BRANCH: ${{ inputs.pr-branch }}
|
||||
run: |
|
||||
bash $GITHUB_WORKSPACE/.ci/rerun-workflow.sh \
|
||||
"optional_checks.yml" \
|
||||
"${INPUTS__PR_BRANCH}" \
|
||||
|| true
|
||||
@@ -0,0 +1,29 @@
|
||||
name: Release Drafter
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
# only 1 job running in the repo at any time
|
||||
concurrency:
|
||||
group: lock
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
jobs:
|
||||
updateReleaseDraft:
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: read
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: release-drafter/release-drafter@ed4bc48ec97379be2258e7b7ac2624a3e26ab809 # v7.4.0
|
||||
with:
|
||||
config-name: release-drafter.yml
|
||||
disable-autolabeler: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
@@ -0,0 +1,122 @@
|
||||
# contains non-functional tests, like checks on docs
|
||||
# and code style
|
||||
name: Static Analysis
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
env:
|
||||
COMPILER: 'gcc'
|
||||
MAKEFLAGS: '-j4'
|
||||
OS_NAME: 'linux'
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: false
|
||||
- name: Setup and run tests
|
||||
shell: bash
|
||||
run: |
|
||||
export TASK=lint
|
||||
export CONDA=${HOME}/miniforge
|
||||
export PATH=${CONDA}/bin:$HOME/.local/bin:${PATH}
|
||||
$GITHUB_WORKSPACE/.ci/setup.sh || exit 1
|
||||
$GITHUB_WORKSPACE/.ci/lint-all.sh || exit 1
|
||||
check-docs:
|
||||
name: check-docs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Setup and run tests
|
||||
shell: bash
|
||||
run: |
|
||||
export TASK=check-docs
|
||||
export CONDA=${HOME}/miniforge
|
||||
export PATH=${CONDA}/bin:$HOME/.local/bin:${PATH}
|
||||
$GITHUB_WORKSPACE/.ci/setup.sh || exit 1
|
||||
$GITHUB_WORKSPACE/.ci/build-docs.sh || exit 1
|
||||
r-check-docs:
|
||||
name: r-package-check-docs
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
container: rocker/verse # zizmor: ignore[unpinned-images]
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Trust git cloning LightGBM
|
||||
run: |
|
||||
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Install packages
|
||||
shell: bash
|
||||
run: |
|
||||
Rscript ./.ci/install-r-deps.R --build --include=roxygen2
|
||||
sh build-cran-package.sh || exit 1
|
||||
R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit 1
|
||||
- name: Test documentation
|
||||
shell: bash --noprofile --norc {0}
|
||||
run: |
|
||||
Rscript .ci/build-docs.R || exit 1
|
||||
num_doc_files_changed=$(
|
||||
git diff --name-only | grep --count -E "\.Rd|NAMESPACE"
|
||||
)
|
||||
if [[ ${num_doc_files_changed} -gt 0 ]]; then
|
||||
echo "Some R documentation files have changed. Please re-generate them and commit those changes."
|
||||
echo ""
|
||||
echo " sh build-cran-package.sh"
|
||||
echo " R CMD INSTALL --with-keep.source lightgbm_*.tar.gz"
|
||||
echo " Rscript -e \"roxygen2::roxygenize('R-package/', load = 'installed')\""
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
all-static-analysis-jobs-successful:
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- lint
|
||||
- check-docs
|
||||
- r-check-docs
|
||||
permissions:
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Note that all tests succeeded
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
@@ -0,0 +1,106 @@
|
||||
name: SWIG
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
# automatically cancel in-progress builds if another commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# default to 0 permissions
|
||||
# (job-level overrides add the minimal permissions needed)
|
||||
permissions:
|
||||
contents: none
|
||||
|
||||
env:
|
||||
# tell scripts where to put artifacts
|
||||
# (this variable name is left over from when jobs ran on Azure DevOps)
|
||||
BUILD_ARTIFACTSTAGINGDIRECTORY: '${{ github.workspace }}/artifacts'
|
||||
# in CMake-driven builds, parallelize compilation
|
||||
CMAKE_BUILD_PARALLEL_LEVEL: 4
|
||||
# all SWIG jobs produce artifacts
|
||||
PRODUCES_ARTIFACTS: 'true'
|
||||
# all jobs here have the same 'TASK'
|
||||
TASK: swig
|
||||
|
||||
jobs:
|
||||
test-swig:
|
||||
name: swig (${{ matrix.os-display-name || matrix.os }}, ${{ matrix.compiler }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
container: ${{ matrix.container }}
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
compiler: gcc
|
||||
container: 'ghcr.io/lightgbm-org/lightgbm-vsts-agent:manylinux_2_28_x86_64'
|
||||
artifact-name: swig-linux-x86_64-jar
|
||||
os-display-name: 'manylinux_2_28'
|
||||
- os: macos-15-intel
|
||||
compiler: clang
|
||||
container: null
|
||||
artifact-name: swig-macos-x86_64-jar
|
||||
# Visual Studio 2022
|
||||
- os: windows-2022
|
||||
compiler: msvc
|
||||
container: null
|
||||
artifact-name: swig-windows-x86_64-jar
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
fetch-depth: 5
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
- name: Setup and run tests on Linux and macOS
|
||||
if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu')
|
||||
shell: bash
|
||||
run: |
|
||||
export BUILD_DIRECTORY="${GITHUB_WORKSPACE}"
|
||||
export COMPILER="${{ matrix.compiler }}"
|
||||
export CONDA="${HOME}/miniforge"
|
||||
export PATH=${CONDA}/bin:${PATH}
|
||||
if [[ "${{ matrix.os }}" =~ ^macos ]]; then
|
||||
export OS_NAME="macos"
|
||||
elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
|
||||
export OS_NAME="linux"
|
||||
fi
|
||||
$GITHUB_WORKSPACE/.ci/setup.sh
|
||||
$GITHUB_WORKSPACE/.ci/test.sh
|
||||
- name: Setup and run tests on Windows
|
||||
if: startsWith(matrix.os, 'windows')
|
||||
shell: pwsh -command ". {0}"
|
||||
run: |
|
||||
$env:BUILD_DIRECTORY = $env:GITHUB_WORKSPACE
|
||||
$env:BUILD_SOURCESDIRECTORY = $env:BUILD_DIRECTORY
|
||||
& "$env:GITHUB_WORKSPACE/.ci/test-windows.ps1"
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ matrix.artifact-name }}
|
||||
path: ${{ env.BUILD_ARTIFACTSTAGINGDIRECTORY }}/*.jar
|
||||
if-no-files-found: error
|
||||
all-swig-jobs-successful:
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- test-swig
|
||||
permissions:
|
||||
statuses: read
|
||||
steps:
|
||||
- name: Note that all tests succeeded
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
+486
@@ -0,0 +1,486 @@
|
||||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
|
||||
# User-specific files
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
[Bb]uild/
|
||||
|
||||
# Visual Studio 2015 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUNIT
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# DNX
|
||||
project.lock.json
|
||||
artifacts/
|
||||
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_i.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# JustCode is a .NET coding add-in
|
||||
.JustCode
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
nuget/
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/packages/
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/packages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/packages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignoreable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Microsoft Azure ApplicationInsights config file
|
||||
ApplicationInsights.config
|
||||
|
||||
# Windows Store app package directory
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
.*.swp
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
node_modules/
|
||||
orleans.codegen.cs
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
|
||||
# Compiled Object files
|
||||
*.slo
|
||||
*.lo
|
||||
*.o
|
||||
*.obj
|
||||
|
||||
# Precompiled Headers
|
||||
*.gch
|
||||
*.pch
|
||||
|
||||
# Compiled Dynamic libraries
|
||||
*.so
|
||||
*.dylib
|
||||
*.dll
|
||||
|
||||
# Fortran module files
|
||||
*.mod
|
||||
|
||||
# Compiled Static libraries
|
||||
*.lai
|
||||
*.la
|
||||
*.a
|
||||
*.lib
|
||||
|
||||
# Executables
|
||||
*.exe
|
||||
*.out
|
||||
*.app
|
||||
/windows/LightGBM.VC.db
|
||||
/lightgbm
|
||||
/testlightgbm
|
||||
|
||||
# Created by https://www.gitignore.io/api/python
|
||||
|
||||
### Python ###
|
||||
!/python-package/lightgbm/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
env/
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
prof/
|
||||
*.prof
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
cover/
|
||||
**/coverage.html
|
||||
**/coverage.html.zip
|
||||
**/Rplots.pdf
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
docs/pythonapi/
|
||||
*.flag
|
||||
|
||||
# Doxygen documentation
|
||||
docs/doxyoutput/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
Untitled*.ipynb
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# celery beat schedule file
|
||||
celerybeat-schedule
|
||||
|
||||
# dotenv
|
||||
.env
|
||||
|
||||
# virtualenv
|
||||
.venv/
|
||||
venv/
|
||||
ENV/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# R testing artefact
|
||||
lightgbm.model
|
||||
|
||||
# saved or dumped model/data
|
||||
*.model
|
||||
*.pkl
|
||||
*.bin
|
||||
*.h5
|
||||
|
||||
# macOS
|
||||
**/.DS_Store
|
||||
|
||||
# VSCode
|
||||
.vscode
|
||||
|
||||
# IntelliJ/CLion
|
||||
.idea
|
||||
*.iml
|
||||
/cmake-build-debug/
|
||||
|
||||
# Files from local Python install
|
||||
lightgbm-python/
|
||||
python-package/LICENSE
|
||||
python-package/build_cpp/
|
||||
python-package/compile/
|
||||
python-package/lightgbm/VERSION.txt
|
||||
|
||||
# R build artefacts
|
||||
**/autom4te.cache/
|
||||
R-package/conftest*
|
||||
R-package/config.status
|
||||
!R-package/data/agaricus.test.rda
|
||||
!R-package/data/agaricus.train.rda
|
||||
!R-package/data/bank.rda
|
||||
R-package/docs
|
||||
R-package/src/CMakeLists.txt
|
||||
R-package/src/Makevars
|
||||
R-package/src/lib_lightgbm.so.dSYM/
|
||||
R-package/src/src/
|
||||
R-package/src-x64
|
||||
R-package/src-i386
|
||||
R-package/**/VERSION.txt
|
||||
**/Makevars.win
|
||||
lightgbm_r/
|
||||
lightgbm*.tar.gz
|
||||
lightgbm*.tgz
|
||||
lightgbm.Rcheck/
|
||||
miktex*.zip
|
||||
*.def
|
||||
|
||||
# Files created by examples and tests
|
||||
*.buffer
|
||||
**/lgb-Dataset.data
|
||||
**/lgb.Dataset.data
|
||||
**/model.txt
|
||||
**/lgb-model.txt
|
||||
examples/**/*.txt
|
||||
tests/distributed/mlist.txt
|
||||
tests/distributed/train*
|
||||
tests/distributed/model*
|
||||
tests/distributed/predict*
|
||||
|
||||
|
||||
# Files from interactive R sessions
|
||||
.Rproj.user
|
||||
**/.Rapp.history
|
||||
**/.Rhistory
|
||||
*.rda
|
||||
*.RData
|
||||
*.rds
|
||||
|
||||
# Files generated by aspell
|
||||
**/*.bak
|
||||
|
||||
# GraphViz artifacts
|
||||
*.gv
|
||||
*.gv.*
|
||||
|
||||
# Files from local Dask work
|
||||
dask-worker-space/
|
||||
|
||||
# credentials and key material
|
||||
*.env
|
||||
*.pem
|
||||
*.pub
|
||||
*.rdp
|
||||
*_rsa
|
||||
|
||||
# hipify-perl -inplace leaves behind *.prehip files
|
||||
*.prehip
|
||||
|
||||
# pixi environments
|
||||
.pixi
|
||||
!.pixi/config.toml
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# files created by release tasks
|
||||
/release-artifacts
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
[submodule "include/boost/compute"]
|
||||
path = external_libs/compute
|
||||
url = https://github.com/boostorg/compute
|
||||
[submodule "eigen"]
|
||||
path = external_libs/eigen
|
||||
url = https://gitlab.com/libeigen/eigen.git
|
||||
[submodule "external_libs/fmt"]
|
||||
path = external_libs/fmt
|
||||
url = https://github.com/fmtlib/fmt.git
|
||||
[submodule "external_libs/fast_double_parser"]
|
||||
path = external_libs/fast_double_parser
|
||||
url = https://github.com/lemire/fast_double_parser.git
|
||||
[submodule "external_libs/nanoarrow"]
|
||||
path = external_libs/nanoarrow
|
||||
url = https://github.com/apache/arrow-nanoarrow.git
|
||||
@@ -0,0 +1,115 @@
|
||||
# exclude files which are auto-generated by build tools
|
||||
exclude: |
|
||||
(?x)^(
|
||||
build|
|
||||
external_libs|
|
||||
lightgbm-python|
|
||||
lightgbm_r|
|
||||
\.pixi|
|
||||
)$
|
||||
|R-package/configure$
|
||||
|R-package/inst/Makevars$
|
||||
|R-package/inst/Makevars.win$
|
||||
|R-package/man/.*Rd$
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
- id: check-toml
|
||||
- id: check-xml
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/cmake-lint/cmake-lint
|
||||
rev: '1.4.3'
|
||||
hooks:
|
||||
- id: cmakelint
|
||||
args: ["--linelength=120"]
|
||||
- repo: https://github.com/cpplint/cpplint
|
||||
rev: '2.0.2'
|
||||
hooks:
|
||||
- id: cpplint
|
||||
args:
|
||||
- --root=.. # workaround to get correct header guard pattern
|
||||
- --recursive
|
||||
- --filter=-build/include_subdir,-whitespace/line_length
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: check-omp-pragmas
|
||||
name: check-omp-pragmas
|
||||
entry: sh
|
||||
args:
|
||||
- .ci/check-omp-pragmas.sh
|
||||
language: system
|
||||
pass_filenames: false
|
||||
- repo: https://github.com/adrienverge/yamllint
|
||||
rev: v1.38.0
|
||||
hooks:
|
||||
- id: yamllint
|
||||
args: ["--strict"]
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: regenerate-parameters
|
||||
name: regenerate-parameters
|
||||
entry: python
|
||||
args:
|
||||
- ./.ci/parameter-generator.py
|
||||
language: python
|
||||
pass_filenames: false
|
||||
- repo: https://github.com/rstcheck/rstcheck
|
||||
rev: v6.2.5
|
||||
hooks:
|
||||
- id: rstcheck
|
||||
args: ["--config", "./python-package/pyproject.toml"]
|
||||
additional_dependencies:
|
||||
- breathe>=4.36.0
|
||||
- sphinx>=8.1.3
|
||||
- sphinx_rtd_theme>=3.0.1
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.20
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: ["--config", "python-package/pyproject.toml"]
|
||||
types_or: [python, jupyter]
|
||||
- id: ruff-format
|
||||
args: ["--config", "python-package/pyproject.toml"]
|
||||
types_or: [python, jupyter]
|
||||
- repo: https://github.com/biomejs/pre-commit
|
||||
rev: v2.5.2
|
||||
hooks:
|
||||
- id: biome-ci
|
||||
args:
|
||||
- --config-path=./biome.json
|
||||
- --diagnostic-level=info
|
||||
- --error-on-warnings
|
||||
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||
rev: v0.11.0.1
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
- repo: https://github.com/crate-ci/typos
|
||||
rev: v1.48.0
|
||||
hooks:
|
||||
- id: typos
|
||||
args: ["--force-exclude"]
|
||||
exclude: (\.gitignore$)|(^\.editorconfig$)
|
||||
- repo: https://github.com/henryiii/validate-pyproject-schema-store
|
||||
rev: 2026.06.30
|
||||
hooks:
|
||||
- id: validate-pyproject
|
||||
files: python-package/pyproject.toml$
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
rev: v2.1.0
|
||||
hooks:
|
||||
- id: mypy
|
||||
args: ["--config-file", "python-package/pyproject.toml", "python-package/"]
|
||||
pass_filenames: false
|
||||
verbose: true
|
||||
additional_dependencies:
|
||||
- matplotlib>=3.9.1
|
||||
- narwhals>=1.15
|
||||
- pandas>=2.0
|
||||
- scikit-learn>=1.5.2
|
||||
- repo: https://github.com/zizmorcore/zizmor-pre-commit
|
||||
rev: v1.26.1
|
||||
hooks:
|
||||
- id: zizmor
|
||||
@@ -0,0 +1,16 @@
|
||||
version: 2
|
||||
build:
|
||||
os: "ubuntu-24.04"
|
||||
tools:
|
||||
python: "miniforge3-latest"
|
||||
conda:
|
||||
environment: docs/env.yml
|
||||
formats:
|
||||
- pdf
|
||||
sphinx:
|
||||
builder: html
|
||||
configuration: docs/conf.py
|
||||
fail_on_warning: true
|
||||
submodules:
|
||||
include: all
|
||||
recursive: true
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
default.extend-ignore-re = [
|
||||
"/Ot",
|
||||
"mis-alignment",
|
||||
"mis-spelled",
|
||||
"posix-seh-rt",
|
||||
]
|
||||
|
||||
[default.extend-words]
|
||||
MAPE = "MAPE"
|
||||
datas = "datas"
|
||||
indx = "indx"
|
||||
interprete = "interprete"
|
||||
mape = "mape"
|
||||
splitted = "splitted"
|
||||
|
||||
[default.extend-identifiers]
|
||||
ERRORs = "ERRORs"
|
||||
GAM = "GAM"
|
||||
ND24s = "ND24s"
|
||||
WARNINGs = "WARNINGs"
|
||||
fullset = "fullset"
|
||||
thess = "thess"
|
||||
@@ -0,0 +1,10 @@
|
||||
# default config: https://yamllint.readthedocs.io/en/stable/configuration.html#default-configuration
|
||||
extends: default
|
||||
|
||||
rules:
|
||||
document-start: disable
|
||||
line-length:
|
||||
max: 120
|
||||
truthy:
|
||||
# prevent treating GitHub Workflow "on" key as boolean value
|
||||
check-keys: false
|
||||
+866
@@ -0,0 +1,866 @@
|
||||
# Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
|
||||
# Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
|
||||
# Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
|
||||
cmake_minimum_required(VERSION 3.28)
|
||||
|
||||
option(USE_MPI "Enable MPI-based distributed learning" OFF)
|
||||
option(USE_OPENMP "Enable OpenMP" ON)
|
||||
option(USE_GPU "Enable GPU-accelerated training" OFF)
|
||||
option(USE_SWIG "Enable SWIG to generate Java API" OFF)
|
||||
option(USE_TIMETAG "Set to ON to output time costs" OFF)
|
||||
option(USE_CUDA "Enable CUDA-accelerated training " OFF)
|
||||
option(USE_ROCM "Enable ROCm-accelerated training " OFF)
|
||||
option(USE_DEBUG "Set to ON for Debug mode" OFF)
|
||||
option(USE_SANITIZER "Use sanitizer flags" OFF)
|
||||
set(
|
||||
ENABLED_SANITIZERS
|
||||
"address" "leak" "undefined"
|
||||
CACHE
|
||||
STRING
|
||||
"Semicolon separated list of sanitizer names, e.g., 'address;leak'. \
|
||||
Supported sanitizers are address, leak, undefined and thread."
|
||||
)
|
||||
option(USE_HOMEBREW_FALLBACK "(macOS-only) also look in 'brew --prefix' for libraries (e.g. OpenMP)" ON)
|
||||
option(BUILD_CLI "Build the 'lightgbm' command-line interface in addition to lib_lightgbm" ON)
|
||||
option(BUILD_CPP_TEST "Build C++ tests with Google Test" OFF)
|
||||
option(BUILD_STATIC_LIB "Build static library" OFF)
|
||||
option(INSTALL_HEADERS "Install headers to CMAKE_INSTALL_PREFIX (e.g. '/usr/local/include')" ON)
|
||||
option(__BUILD_FOR_PYTHON "Set to ON if building lib_lightgbm for use with the Python-package" OFF)
|
||||
option(__BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R-package" OFF)
|
||||
option(__INTEGRATE_OPENCL "Set to ON if building LightGBM with the OpenCL ICD Loader and its dependencies included" OFF)
|
||||
|
||||
# If using Visual Studio generators, always target v10.x of the Windows SDK.
|
||||
# Doing this avoids lookups that could fall back to very old versions, e.g. by finding
|
||||
# outdated registry entries.
|
||||
# ref: https://cmake.org/cmake/help/latest/variable/CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION.html
|
||||
if(CMAKE_GENERATOR MATCHES "Visual Studio")
|
||||
set(CMAKE_SYSTEM_VERSION 10.0 CACHE INTERNAL "target Windows SDK version" FORCE)
|
||||
endif()
|
||||
|
||||
project(lightgbm LANGUAGES C CXX)
|
||||
|
||||
include(cmake/Utils.cmake)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules")
|
||||
|
||||
#-- Sanitizer
|
||||
if(USE_SANITIZER)
|
||||
if(MSVC)
|
||||
message(FATAL_ERROR "Sanitizers are not supported with MSVC.")
|
||||
endif()
|
||||
include(cmake/Sanitizer.cmake)
|
||||
enable_sanitizers("${ENABLED_SANITIZERS}")
|
||||
endif()
|
||||
|
||||
if(__INTEGRATE_OPENCL)
|
||||
set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE)
|
||||
set(USE_GPU OFF CACHE BOOL "" FORCE)
|
||||
message(STATUS "Building library with integrated OpenCL components")
|
||||
endif()
|
||||
|
||||
if(__BUILD_FOR_PYTHON OR __BUILD_FOR_R OR USE_SWIG)
|
||||
# the SWIG wrapper, the Python and R packages don't require the CLI
|
||||
set(BUILD_CLI OFF)
|
||||
# installing the SWIG wrapper, the R and Python packages shouldn't place LightGBM's headers
|
||||
# outside of where the package is installed
|
||||
set(INSTALL_HEADERS OFF)
|
||||
endif()
|
||||
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8.2")
|
||||
message(FATAL_ERROR "Insufficient gcc version (${CMAKE_CXX_COMPILER_VERSION})")
|
||||
endif()
|
||||
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
||||
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "3.8")
|
||||
message(FATAL_ERROR "Insufficient Clang version (${CMAKE_CXX_COMPILER_VERSION})")
|
||||
endif()
|
||||
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
|
||||
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "8.1.0")
|
||||
message(FATAL_ERROR "Insufficient AppleClang version (${CMAKE_CXX_COMPILER_VERSION})")
|
||||
endif()
|
||||
elseif(MSVC)
|
||||
if(MSVC_VERSION LESS 1900)
|
||||
message(FATAL_ERROR "Insufficient MSVC version (${MSVC_VERSION})")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(USE_SWIG)
|
||||
find_package(SWIG REQUIRED)
|
||||
find_package(Java REQUIRED)
|
||||
find_package(JNI REQUIRED)
|
||||
include(UseJava)
|
||||
include(UseSWIG)
|
||||
set(SWIG_CXX_EXTENSION "cxx")
|
||||
set(SWIG_EXTRA_LIBRARIES "")
|
||||
set(SWIG_JAVA_EXTRA_FILE_EXTENSIONS ".java" "JNI.java")
|
||||
set(SWIG_MODULE_JAVA_LANGUAGE "JAVA")
|
||||
set(SWIG_MODULE_JAVA_SWIG_LANGUAGE_FLAG "java")
|
||||
set(CMAKE_SWIG_OUTDIR "${CMAKE_CURRENT_BINARY_DIR}/java")
|
||||
include_directories(Java_INCLUDE_DIRS)
|
||||
include_directories(JNI_INCLUDE_DIRS)
|
||||
include_directories($ENV{JAVA_HOME}/include)
|
||||
if(WIN32)
|
||||
set(LGBM_SWIG_DESTINATION_DIR "${CMAKE_CURRENT_BINARY_DIR}/com/microsoft/ml/lightgbm/windows/x86_64")
|
||||
include_directories($ENV{JAVA_HOME}/include/win32)
|
||||
elseif(APPLE)
|
||||
set(LGBM_SWIG_DESTINATION_DIR "${CMAKE_CURRENT_BINARY_DIR}/com/microsoft/ml/lightgbm/osx/x86_64")
|
||||
include_directories($ENV{JAVA_HOME}/include/darwin)
|
||||
else()
|
||||
set(LGBM_SWIG_DESTINATION_DIR "${CMAKE_CURRENT_BINARY_DIR}/com/microsoft/ml/lightgbm/linux/x86_64")
|
||||
include_directories($ENV{JAVA_HOME}/include/linux)
|
||||
endif()
|
||||
file(MAKE_DIRECTORY "${LGBM_SWIG_DESTINATION_DIR}")
|
||||
endif()
|
||||
|
||||
set(EIGEN_DIR "${PROJECT_SOURCE_DIR}/external_libs/eigen")
|
||||
include_directories(${EIGEN_DIR})
|
||||
|
||||
# See https://gitlab.com/libeigen/eigen/-/blob/master/COPYING.README
|
||||
add_definitions(-DEIGEN_MPL2_ONLY)
|
||||
add_definitions(-DEIGEN_DONT_PARALLELIZE)
|
||||
|
||||
set(FAST_DOUBLE_PARSER_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/external_libs/fast_double_parser/include")
|
||||
include_directories(${FAST_DOUBLE_PARSER_INCLUDE_DIR})
|
||||
|
||||
set(FMT_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/external_libs/fmt/include")
|
||||
include_directories(${FMT_INCLUDE_DIR})
|
||||
|
||||
if(BUILD_CPP_TEST AND MSVC)
|
||||
# Use /MT flag to statically link the C runtime. Must be set before
|
||||
# add_subdirectory(nanoarrow) so the bundled library uses the same runtime
|
||||
# as the test executable; otherwise linking fails with unresolved __imp_*
|
||||
# symbols (e.g. __imp_realloc).
|
||||
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
|
||||
endif()
|
||||
|
||||
# nanoarrow is only needed by the Arrow-based C API entry points, which are
|
||||
# disabled for the R package (the R API never calls them). Skipping the
|
||||
# subdirectory keeps the package tarball free of nanoarrow sources, which
|
||||
# would otherwise trigger CRAN warnings about diagnostic-suppressing pragmas.
|
||||
if(NOT __BUILD_FOR_R)
|
||||
add_subdirectory("${PROJECT_SOURCE_DIR}/external_libs/nanoarrow" EXCLUDE_FROM_ALL)
|
||||
if(NOT BUILD_STATIC_LIB)
|
||||
set_target_properties(nanoarrow_static PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(__BUILD_FOR_R)
|
||||
find_package(LibR REQUIRED)
|
||||
message(STATUS "LIBR_EXECUTABLE: ${LIBR_EXECUTABLE}")
|
||||
message(STATUS "LIBR_INCLUDE_DIRS: ${LIBR_INCLUDE_DIRS}")
|
||||
message(STATUS "LIBR_LIBS_DIR: ${LIBR_LIBS_DIR}")
|
||||
message(STATUS "LIBR_CORE_LIBRARY: ${LIBR_CORE_LIBRARY}")
|
||||
include_directories(${LIBR_INCLUDE_DIRS})
|
||||
add_definitions(-DLGB_R_BUILD)
|
||||
endif()
|
||||
|
||||
if(USE_TIMETAG)
|
||||
add_definitions(-DTIMETAG)
|
||||
endif()
|
||||
|
||||
if(USE_DEBUG)
|
||||
add_definitions(-DDEBUG)
|
||||
endif()
|
||||
|
||||
if(USE_MPI)
|
||||
find_package(MPI REQUIRED)
|
||||
add_definitions(-DUSE_MPI)
|
||||
else()
|
||||
add_definitions(-DUSE_SOCKET)
|
||||
endif()
|
||||
|
||||
if(USE_CUDA)
|
||||
set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}")
|
||||
find_package(CUDAToolkit 11.8 REQUIRED)
|
||||
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
|
||||
compute_cmake_cuda_archs("${CUDAToolkit_VERSION}")
|
||||
endif()
|
||||
enable_language(CUDA)
|
||||
set(USE_OPENMP ON CACHE BOOL "CUDA requires OpenMP" FORCE)
|
||||
endif()
|
||||
|
||||
if(USE_ROCM)
|
||||
enable_language(HIP)
|
||||
set(USE_OPENMP ON CACHE BOOL "ROCm requires OpenMP" FORCE)
|
||||
endif()
|
||||
|
||||
if(USE_OPENMP)
|
||||
if(APPLE)
|
||||
find_package(OpenMP)
|
||||
if(NOT OpenMP_FOUND)
|
||||
if(USE_HOMEBREW_FALLBACK)
|
||||
# libomp 15.0+ from brew is keg-only, so have to search in other locations.
|
||||
# See https://github.com/Homebrew/homebrew-core/issues/112107#issuecomment-1278042927.
|
||||
execute_process(COMMAND brew --prefix libomp
|
||||
OUTPUT_VARIABLE HOMEBREW_LIBOMP_PREFIX
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_LIBOMP_PREFIX}/include")
|
||||
set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_LIBOMP_PREFIX}/include")
|
||||
set(OpenMP_C_LIB_NAMES omp)
|
||||
set(OpenMP_CXX_LIB_NAMES omp)
|
||||
set(OpenMP_omp_LIBRARY ${HOMEBREW_LIBOMP_PREFIX}/lib/libomp.dylib)
|
||||
endif()
|
||||
find_package(OpenMP REQUIRED)
|
||||
endif()
|
||||
else()
|
||||
find_package(OpenMP REQUIRED)
|
||||
endif()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
|
||||
endif()
|
||||
|
||||
if(USE_GPU)
|
||||
set(BOOST_COMPUTE_HEADER_DIR ${PROJECT_SOURCE_DIR}/external_libs/compute/include)
|
||||
include_directories(${BOOST_COMPUTE_HEADER_DIR})
|
||||
find_package(OpenCL REQUIRED)
|
||||
include_directories(${OpenCL_INCLUDE_DIRS})
|
||||
message(STATUS "OpenCL include directory: " ${OpenCL_INCLUDE_DIRS})
|
||||
if(WIN32)
|
||||
set(Boost_USE_STATIC_LIBS ON)
|
||||
endif()
|
||||
find_package(Boost 1.56.0 COMPONENTS filesystem system REQUIRED)
|
||||
if(WIN32)
|
||||
# disable autolinking in boost
|
||||
add_definitions(-DBOOST_ALL_NO_LIB)
|
||||
endif()
|
||||
include_directories(${Boost_INCLUDE_DIRS})
|
||||
add_definitions(-DUSE_GPU)
|
||||
endif()
|
||||
|
||||
if(__INTEGRATE_OPENCL)
|
||||
if(APPLE)
|
||||
message(FATAL_ERROR "Integrated OpenCL build is not available on macOS")
|
||||
else()
|
||||
include(cmake/IntegratedOpenCL.cmake)
|
||||
add_definitions(-DUSE_GPU)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(USE_CUDA)
|
||||
find_package(NCCL REQUIRED)
|
||||
include_directories(${CUDAToolkit_INCLUDE_DIRS})
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS} -Xcompiler=-fPIC -Xcompiler=-Wall")
|
||||
if(USE_DEBUG)
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -g")
|
||||
else()
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -O3 -lineinfo")
|
||||
endif()
|
||||
message(STATUS "CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
|
||||
|
||||
add_definitions(-DUSE_CUDA)
|
||||
|
||||
if(NOT DEFINED CMAKE_CUDA_STANDARD)
|
||||
set(CMAKE_CUDA_STANDARD 17)
|
||||
set(CMAKE_CUDA_STANDARD_REQUIRED ON)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(USE_ROCM)
|
||||
find_package(HIP)
|
||||
include_directories(${HIP_INCLUDE_DIRS})
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__HIP_PLATFORM_AMD__")
|
||||
set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} ${OpenMP_CXX_FLAGS} -fPIC -Wall")
|
||||
|
||||
# avoid warning: unused variable 'mask' due to __shfl_down_sync work-around
|
||||
set(DISABLED_WARNINGS "${DISABLED_WARNINGS} -Wno-unused-variable")
|
||||
# avoid warning: 'hipHostAlloc' is deprecated: use hipHostMalloc instead
|
||||
set(DISABLED_WARNINGS "${DISABLED_WARNINGS} -Wno-deprecated-declarations")
|
||||
# avoid many warnings about missing overrides
|
||||
set(DISABLED_WARNINGS "${DISABLED_WARNINGS} -Wno-inconsistent-missing-override")
|
||||
# avoid warning: shift count >= width of type in feature_histogram.hpp
|
||||
set(DISABLED_WARNINGS "${DISABLED_WARNINGS} -Wno-shift-count-overflow")
|
||||
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${DISABLED_WARNINGS}")
|
||||
set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} ${DISABLED_WARNINGS}")
|
||||
|
||||
if(USE_DEBUG)
|
||||
set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -g -O0")
|
||||
else()
|
||||
set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -O3")
|
||||
endif()
|
||||
message(STATUS "CMAKE_HIP_FLAGS: ${CMAKE_HIP_FLAGS}")
|
||||
|
||||
# Building for ROCm almost always means USE_CUDA.
|
||||
# Exceptions to this will be guarded by USE_ROCM.
|
||||
add_definitions(-DUSE_CUDA)
|
||||
add_definitions(-DUSE_ROCM)
|
||||
endif()
|
||||
|
||||
include(CheckCXXSourceCompiles)
|
||||
check_cxx_source_compiles("
|
||||
#include <xmmintrin.h>
|
||||
int main() {
|
||||
int a = 0;
|
||||
_mm_prefetch(&a, _MM_HINT_NTA);
|
||||
return 0;
|
||||
}
|
||||
" MM_PREFETCH)
|
||||
|
||||
if(${MM_PREFETCH})
|
||||
message(STATUS "Using _mm_prefetch")
|
||||
add_definitions(-DMM_PREFETCH)
|
||||
endif()
|
||||
|
||||
include(CheckCXXSourceCompiles)
|
||||
check_cxx_source_compiles("
|
||||
#include <mm_malloc.h>
|
||||
int main() {
|
||||
char *a = (char*)_mm_malloc(8, 16);
|
||||
_mm_free(a);
|
||||
return 0;
|
||||
}
|
||||
" MM_MALLOC)
|
||||
|
||||
if(${MM_MALLOC})
|
||||
message(STATUS "Using _mm_malloc")
|
||||
add_definitions(-DMM_MALLOC)
|
||||
endif()
|
||||
|
||||
if(UNIX OR MINGW OR CYGWIN)
|
||||
set(
|
||||
CMAKE_CXX_FLAGS
|
||||
"${CMAKE_CXX_FLAGS} -pthread -Wextra -Wall -Wno-ignored-attributes -Wno-unknown-pragmas -Wno-return-type"
|
||||
)
|
||||
if(MINGW)
|
||||
# ignore this warning: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95353
|
||||
set(
|
||||
CMAKE_CXX_FLAGS
|
||||
"${CMAKE_CXX_FLAGS} -Wno-stringop-overflow"
|
||||
)
|
||||
endif()
|
||||
if(USE_DEBUG)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
|
||||
endif()
|
||||
if(USE_SWIG)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing")
|
||||
endif()
|
||||
if(NOT USE_OPENMP)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-pragmas -Wno-unused-private-field")
|
||||
endif()
|
||||
if(__BUILD_FOR_R AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-cast-function-type")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(WIN32 AND MINGW)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libstdc++")
|
||||
endif()
|
||||
|
||||
# Check if inet_pton is already available, to avoid conflicts with the implementation in LightGBM.
|
||||
# As of 2022, MinGW started including a definition of inet_pton.
|
||||
if(WIN32)
|
||||
include(CheckSymbolExists)
|
||||
list(APPEND CMAKE_REQUIRED_LIBRARIES "ws2_32")
|
||||
check_symbol_exists(inet_pton "ws2tcpip.h" WIN_INET_PTON_FOUND)
|
||||
if(WIN_INET_PTON_FOUND)
|
||||
add_definitions(-DWIN_HAS_INET_PTON)
|
||||
endif()
|
||||
list(REMOVE_ITEM CMAKE_REQUIRED_LIBRARIES "ws2_32")
|
||||
endif()
|
||||
|
||||
if(MSVC)
|
||||
# compiling 'fmt' on MSVC: "Unicode support requires compiling with /utf-8"
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /MP /utf-8")
|
||||
if(__BUILD_FOR_R)
|
||||
# MSVC does not like this commit:
|
||||
# https://github.com/wch/r-source/commit/fb52ac1a610571fcb8ac92d886b9fefcffaa7d48
|
||||
#
|
||||
# and raises "error C3646: 'private_data_c': unknown override specifier"
|
||||
add_definitions(-DR_LEGACY_RCOMPLEX)
|
||||
endif()
|
||||
if(USE_DEBUG)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Od")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ob2 /Oi /Ot /Oy")
|
||||
endif()
|
||||
else()
|
||||
if(NOT BUILD_STATIC_LIB)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
|
||||
endif()
|
||||
if(NOT USE_DEBUG)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -funroll-loops")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(LightGBM_HEADER_DIR ${PROJECT_SOURCE_DIR}/include)
|
||||
|
||||
set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR})
|
||||
set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR})
|
||||
|
||||
include_directories(${LightGBM_HEADER_DIR})
|
||||
|
||||
if(USE_MPI)
|
||||
include_directories(${MPI_CXX_INCLUDE_PATH})
|
||||
endif()
|
||||
|
||||
set(
|
||||
LGBM_SOURCES
|
||||
src/boosting/boosting.cpp
|
||||
src/boosting/gbdt_model_text.cpp
|
||||
src/boosting/gbdt_prediction.cpp
|
||||
src/boosting/gbdt.cpp
|
||||
src/boosting/prediction_early_stop.cpp
|
||||
src/boosting/sample_strategy.cpp
|
||||
src/io/bin.cpp
|
||||
src/io/config_auto.cpp
|
||||
src/io/config.cpp
|
||||
src/io/dataset_loader.cpp
|
||||
src/io/dataset.cpp
|
||||
src/io/file_io.cpp
|
||||
src/io/json11.cpp
|
||||
src/io/metadata.cpp
|
||||
src/io/parser.cpp
|
||||
src/io/train_share_states.cpp
|
||||
src/io/tree.cpp
|
||||
src/metric/dcg_calculator.cpp
|
||||
src/metric/metric.cpp
|
||||
src/network/linker_topo.cpp
|
||||
src/network/linkers_mpi.cpp
|
||||
src/network/linkers_socket.cpp
|
||||
src/network/network.cpp
|
||||
src/objective/objective_function.cpp
|
||||
src/treelearner/data_parallel_tree_learner.cpp
|
||||
src/treelearner/feature_histogram.cpp
|
||||
src/treelearner/feature_parallel_tree_learner.cpp
|
||||
src/treelearner/gpu_tree_learner.cpp
|
||||
src/treelearner/gradient_discretizer.cpp
|
||||
src/treelearner/linear_tree_learner.cpp
|
||||
src/treelearner/serial_tree_learner.cpp
|
||||
src/treelearner/tree_learner.cpp
|
||||
src/treelearner/voting_parallel_tree_learner.cpp
|
||||
src/utils/openmp_wrapper.cpp
|
||||
)
|
||||
set(
|
||||
LGBM_CUDA_SOURCES
|
||||
src/boosting/cuda/cuda_score_updater.cpp
|
||||
src/boosting/cuda/cuda_score_updater.cu
|
||||
src/boosting/cuda/nccl_gbdt.cpp
|
||||
src/metric/cuda/cuda_binary_metric.cpp
|
||||
src/metric/cuda/cuda_pointwise_metric.cpp
|
||||
src/metric/cuda/cuda_regression_metric.cpp
|
||||
src/metric/cuda/cuda_pointwise_metric.cu
|
||||
src/objective/cuda/cuda_binary_objective.cpp
|
||||
src/objective/cuda/cuda_multiclass_objective.cpp
|
||||
src/objective/cuda/cuda_rank_objective.cpp
|
||||
src/objective/cuda/cuda_regression_objective.cpp
|
||||
src/objective/cuda/cuda_binary_objective.cu
|
||||
src/objective/cuda/cuda_multiclass_objective.cu
|
||||
src/objective/cuda/cuda_rank_objective.cu
|
||||
src/objective/cuda/cuda_regression_objective.cu
|
||||
src/treelearner/cuda/cuda_best_split_finder.cpp
|
||||
src/treelearner/cuda/cuda_data_partition.cpp
|
||||
src/treelearner/cuda/cuda_histogram_constructor.cpp
|
||||
src/treelearner/cuda/cuda_leaf_splits.cpp
|
||||
src/treelearner/cuda/cuda_single_gpu_tree_learner.cpp
|
||||
src/treelearner/cuda/cuda_best_split_finder.cu
|
||||
src/treelearner/cuda/cuda_data_partition.cu
|
||||
src/treelearner/cuda/cuda_gradient_discretizer.cu
|
||||
src/treelearner/cuda/cuda_histogram_constructor.cu
|
||||
src/treelearner/cuda/cuda_leaf_splits.cu
|
||||
src/treelearner/cuda/cuda_single_gpu_tree_learner.cu
|
||||
src/io/cuda/cuda_column_data.cu
|
||||
src/io/cuda/cuda_tree.cu
|
||||
src/io/cuda/cuda_column_data.cpp
|
||||
src/io/cuda/cuda_metadata.cpp
|
||||
src/io/cuda/cuda_row_data.cpp
|
||||
src/io/cuda/cuda_tree.cpp
|
||||
src/cuda/cuda_utils.cpp
|
||||
src/cuda/cuda_algorithms.cu
|
||||
)
|
||||
|
||||
if(USE_CUDA OR USE_ROCM)
|
||||
list(APPEND LGBM_SOURCES ${LGBM_CUDA_SOURCES})
|
||||
endif()
|
||||
|
||||
if(USE_ROCM)
|
||||
set(CU_FILES "")
|
||||
foreach(file IN LISTS LGBM_CUDA_SOURCES)
|
||||
string(REGEX MATCH "\\.cu$" is_cu_file "${file}")
|
||||
if(is_cu_file)
|
||||
list(APPEND CU_FILES "${file}")
|
||||
endif()
|
||||
endforeach()
|
||||
set_source_files_properties(${CU_FILES} PROPERTIES LANGUAGE HIP)
|
||||
endif()
|
||||
|
||||
add_library(lightgbm_objs OBJECT ${LGBM_SOURCES})
|
||||
if(NOT __BUILD_FOR_R)
|
||||
target_link_libraries(lightgbm_objs PRIVATE nanoarrow_static)
|
||||
endif()
|
||||
|
||||
if(BUILD_CLI)
|
||||
add_executable(lightgbm src/main.cpp src/application/application.cpp)
|
||||
target_link_libraries(lightgbm PRIVATE lightgbm_objs)
|
||||
endif()
|
||||
|
||||
set(API_SOURCES "src/c_api.cpp")
|
||||
# Only build the R part of the library if building for
|
||||
# use with the R-package
|
||||
if(__BUILD_FOR_R)
|
||||
list(APPEND API_SOURCES "src/lightgbm_R.cpp")
|
||||
endif()
|
||||
|
||||
add_library(lightgbm_capi_objs OBJECT ${API_SOURCES})
|
||||
if(NOT __BUILD_FOR_R)
|
||||
target_link_libraries(lightgbm_capi_objs PRIVATE nanoarrow_static)
|
||||
endif()
|
||||
|
||||
if(BUILD_STATIC_LIB)
|
||||
add_library(_lightgbm STATIC)
|
||||
else()
|
||||
add_library(_lightgbm SHARED)
|
||||
endif()
|
||||
|
||||
# R expects libraries of the form <project>.{dll,dylib,so}, not lib_<project>.{dll,dylib,so}
|
||||
if(__BUILD_FOR_R)
|
||||
set_target_properties(
|
||||
_lightgbm
|
||||
PROPERTIES
|
||||
PREFIX ""
|
||||
OUTPUT_NAME "lightgbm"
|
||||
)
|
||||
endif()
|
||||
|
||||
# LightGBM headers include openmp, cuda, R etc. headers,
|
||||
# thus PUBLIC is required for building _lightgbm_swig target.
|
||||
target_link_libraries(_lightgbm PUBLIC lightgbm_capi_objs lightgbm_objs)
|
||||
|
||||
if(MSVC AND NOT __BUILD_FOR_R)
|
||||
set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
|
||||
endif()
|
||||
|
||||
if(USE_SWIG)
|
||||
set_property(SOURCE swig/lightgbmlib.i PROPERTY CPLUSPLUS ON)
|
||||
list(APPEND swig_options -package com.microsoft.ml.lightgbm)
|
||||
set_property(SOURCE swig/lightgbmlib.i PROPERTY SWIG_FLAGS "${swig_options}")
|
||||
swig_add_library(_lightgbm_swig LANGUAGE java SOURCES swig/lightgbmlib.i)
|
||||
swig_link_libraries(_lightgbm_swig _lightgbm)
|
||||
set_target_properties(
|
||||
_lightgbm_swig
|
||||
PROPERTIES
|
||||
# needed to ensure Linux build does not have lib prefix specified twice, e.g. liblib_lightgbm_swig
|
||||
PREFIX ""
|
||||
# needed in some versions of CMake for VS and MinGW builds to ensure output dll has lib prefix
|
||||
OUTPUT_NAME "lib_lightgbm_swig"
|
||||
)
|
||||
if(WIN32)
|
||||
set(LGBM_SWIG_LIB_DESTINATION_PATH "${LGBM_SWIG_DESTINATION_DIR}/lib_lightgbm_swig.dll")
|
||||
if(MINGW OR CYGWIN)
|
||||
set(LGBM_LIB_SOURCE_PATH "${PROJECT_SOURCE_DIR}/lib_lightgbm.dll")
|
||||
set(LGBM_SWIG_LIB_SOURCE_PATH "${PROJECT_SOURCE_DIR}/lib_lightgbm_swig.dll")
|
||||
else()
|
||||
set(LGBM_LIB_SOURCE_PATH "${PROJECT_SOURCE_DIR}/Release/lib_lightgbm.dll")
|
||||
set(LGBM_SWIG_LIB_SOURCE_PATH "${PROJECT_SOURCE_DIR}/Release/lib_lightgbm_swig.dll")
|
||||
endif()
|
||||
elseif(APPLE)
|
||||
set(LGBM_LIB_SOURCE_PATH "${PROJECT_SOURCE_DIR}/lib_lightgbm.dylib")
|
||||
set(LGBM_SWIG_LIB_SOURCE_PATH "${PROJECT_SOURCE_DIR}/lib_lightgbm_swig.jnilib")
|
||||
set(LGBM_SWIG_LIB_DESTINATION_PATH "${LGBM_SWIG_DESTINATION_DIR}/lib_lightgbm_swig.dylib")
|
||||
else()
|
||||
set(LGBM_LIB_SOURCE_PATH "${PROJECT_SOURCE_DIR}/lib_lightgbm.so")
|
||||
set(LGBM_SWIG_LIB_SOURCE_PATH "${PROJECT_SOURCE_DIR}/lib_lightgbm_swig.so")
|
||||
set(LGBM_SWIG_LIB_DESTINATION_PATH "${LGBM_SWIG_DESTINATION_DIR}/lib_lightgbm_swig.so")
|
||||
endif()
|
||||
add_custom_command(
|
||||
TARGET _lightgbm_swig
|
||||
POST_BUILD
|
||||
COMMAND "${Java_JAVAC_EXECUTABLE}" -d . java/*.java
|
||||
COMMAND
|
||||
"${CMAKE_COMMAND}"
|
||||
-E
|
||||
copy_if_different
|
||||
"${LGBM_LIB_SOURCE_PATH}"
|
||||
"${LGBM_SWIG_DESTINATION_DIR}"
|
||||
COMMAND
|
||||
"${CMAKE_COMMAND}"
|
||||
-E
|
||||
copy_if_different
|
||||
"${LGBM_SWIG_LIB_SOURCE_PATH}"
|
||||
"${LGBM_SWIG_LIB_DESTINATION_PATH}"
|
||||
COMMAND "${Java_JAR_EXECUTABLE}" -cf lightgbmlib.jar com
|
||||
)
|
||||
endif()
|
||||
|
||||
if(USE_MPI)
|
||||
target_link_libraries(lightgbm_objs PUBLIC ${MPI_CXX_LIBRARIES})
|
||||
endif()
|
||||
|
||||
if(USE_OPENMP)
|
||||
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
||||
target_link_libraries(lightgbm_objs PUBLIC OpenMP::OpenMP_CXX)
|
||||
# c_api headers also includes OpenMP headers, thus compiling
|
||||
# lightgbm_capi_objs needs include directory for OpenMP.
|
||||
# Specifying OpenMP in target_link_libraries will get include directory
|
||||
# requirements for compilation.
|
||||
# This uses CMake's Transitive Usage Requirements. Refer to CMake doc:
|
||||
# https://cmake.org/cmake/help/v3.16/manual/cmake-buildsystem.7.html#transitive-usage-requirements
|
||||
target_link_libraries(lightgbm_capi_objs PUBLIC OpenMP::OpenMP_CXX)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(USE_GPU)
|
||||
target_link_libraries(lightgbm_objs PUBLIC ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
|
||||
endif()
|
||||
|
||||
if(USE_CUDA)
|
||||
target_link_libraries(lightgbm_objs PUBLIC NCCL::NCCL)
|
||||
endif()
|
||||
|
||||
if(USE_ROCM)
|
||||
find_package(rccl)
|
||||
target_link_libraries(lightgbm_objs PUBLIC ${RCCL_LIBRARY})
|
||||
endif()
|
||||
|
||||
if(__INTEGRATE_OPENCL)
|
||||
# targets OpenCL and Boost are added in IntegratedOpenCL.cmake
|
||||
add_dependencies(lightgbm_objs OpenCL Boost)
|
||||
# variables INTEGRATED_OPENCL_* are set in IntegratedOpenCL.cmake
|
||||
target_include_directories(lightgbm_objs PRIVATE ${INTEGRATED_OPENCL_INCLUDES})
|
||||
target_compile_definitions(lightgbm_objs PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS})
|
||||
target_link_libraries(lightgbm_objs PUBLIC ${INTEGRATED_OPENCL_LIBRARIES} ${CMAKE_DL_LIBS})
|
||||
endif()
|
||||
|
||||
if(USE_CUDA)
|
||||
|
||||
set_target_properties(
|
||||
lightgbm_objs
|
||||
PROPERTIES
|
||||
CUDA_SEPARABLE_COMPILATION ON
|
||||
)
|
||||
|
||||
set_target_properties(
|
||||
_lightgbm
|
||||
PROPERTIES
|
||||
CUDA_RESOLVE_DEVICE_SYMBOLS ON
|
||||
)
|
||||
|
||||
if(BUILD_CLI)
|
||||
set_target_properties(
|
||||
lightgbm
|
||||
PROPERTIES
|
||||
CUDA_RESOLVE_DEVICE_SYMBOLS ON
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(USE_ROCM)
|
||||
target_link_libraries(lightgbm_objs PUBLIC hip::host)
|
||||
endif()
|
||||
|
||||
if(WIN32)
|
||||
if(MINGW OR CYGWIN)
|
||||
target_link_libraries(lightgbm_objs PUBLIC ws2_32 iphlpapi)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(__BUILD_FOR_R)
|
||||
# utils/log.h and capi uses R headers, thus both object libraries need to link
|
||||
# with R lib.
|
||||
if(MSVC)
|
||||
set(R_LIB ${LIBR_MSVC_CORE_LIBRARY})
|
||||
else()
|
||||
set(R_LIB ${LIBR_CORE_LIBRARY})
|
||||
endif()
|
||||
target_link_libraries(lightgbm_objs PUBLIC ${R_LIB})
|
||||
target_link_libraries(lightgbm_capi_objs PUBLIC ${R_LIB})
|
||||
endif()
|
||||
|
||||
#-- Google C++ tests
|
||||
if(BUILD_CPP_TEST)
|
||||
find_package(GTest CONFIG)
|
||||
if(NOT GTEST_FOUND)
|
||||
message(STATUS "Did not find Google Test in the system root. Fetching Google Test now...")
|
||||
include(FetchContent)
|
||||
# lint_cmake: -readability/wonkycase
|
||||
FetchContent_Declare(
|
||||
# lint_cmake: +readability/wonkycase
|
||||
googletest
|
||||
GIT_REPOSITORY https://github.com/google/googletest.git
|
||||
GIT_TAG v1.14.0
|
||||
)
|
||||
# lint_cmake: -readability/wonkycase
|
||||
FetchContent_MakeAvailable(googletest)
|
||||
# lint_cmake: +readability/wonkycase
|
||||
add_library(GTest::GTest ALIAS gtest)
|
||||
endif()
|
||||
|
||||
set(LightGBM_TEST_HEADER_DIR ${PROJECT_SOURCE_DIR}/tests/cpp_tests)
|
||||
include_directories(${LightGBM_TEST_HEADER_DIR})
|
||||
|
||||
set(
|
||||
CPP_TEST_SOURCES
|
||||
tests/cpp_tests/test_array_args.cpp
|
||||
tests/cpp_tests/test_arrow.cpp
|
||||
tests/cpp_tests/test_arrow_deprecated.cpp
|
||||
tests/cpp_tests/test_byte_buffer.cpp
|
||||
tests/cpp_tests/test_chunked_array.cpp
|
||||
tests/cpp_tests/test_common.cpp
|
||||
tests/cpp_tests/test_main.cpp
|
||||
tests/cpp_tests/test_serialize.cpp
|
||||
tests/cpp_tests/test_single_row.cpp
|
||||
tests/cpp_tests/test_stream.cpp
|
||||
tests/cpp_tests/testutils.cpp
|
||||
)
|
||||
if(MSVC)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /permissive-")
|
||||
endif()
|
||||
add_executable(testlightgbm ${CPP_TEST_SOURCES})
|
||||
target_link_libraries(testlightgbm PRIVATE lightgbm_objs lightgbm_capi_objs nanoarrow_static GTest::GTest)
|
||||
endif()
|
||||
|
||||
if(BUILD_CLI)
|
||||
install(
|
||||
TARGETS lightgbm
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
|
||||
)
|
||||
endif()
|
||||
|
||||
if(__BUILD_FOR_PYTHON)
|
||||
set(CMAKE_INSTALL_PREFIX "lightgbm")
|
||||
endif()
|
||||
|
||||
# The macOS linker puts an absolute path to linked libraries in lib_lightgbm.dylib.
|
||||
# This block overrides that information for LightGBM's OpenMP dependency, to allow
|
||||
# finding that library in more places.
|
||||
#
|
||||
# This reduces the risk of runtime issues resulting from multiple {libgomp,libiomp,libomp}.dylib being loaded.
|
||||
#
|
||||
if(APPLE AND USE_OPENMP AND NOT BUILD_STATIC_LIB)
|
||||
# store path to {libgomp,libiomp,libomp}.dylib found at build time in a variable
|
||||
get_target_property(
|
||||
OpenMP_LIBRARY_LOCATION
|
||||
OpenMP::OpenMP_CXX
|
||||
INTERFACE_LINK_LIBRARIES
|
||||
)
|
||||
# get just the filename of that path
|
||||
# (to deal with the possibility that it might be 'libomp.dylib' or 'libgomp.dylib' or 'libiomp.dylib')
|
||||
get_filename_component(
|
||||
OpenMP_LIBRARY_NAME
|
||||
${OpenMP_LIBRARY_LOCATION}
|
||||
NAME
|
||||
)
|
||||
# get directory of that path
|
||||
get_filename_component(
|
||||
OpenMP_LIBRARY_DIR
|
||||
${OpenMP_LIBRARY_LOCATION}
|
||||
DIRECTORY
|
||||
)
|
||||
# get exact name of the library in a variable
|
||||
get_target_property(
|
||||
__LIB_LIGHTGBM_OUTPUT_NAME
|
||||
_lightgbm
|
||||
OUTPUT_NAME
|
||||
)
|
||||
if(NOT __LIB_LIGHTGBM_OUTPUT_NAME)
|
||||
set(__LIB_LIGHTGBM_OUTPUT_NAME "lib_lightgbm")
|
||||
endif()
|
||||
|
||||
if(CMAKE_SHARED_LIBRARY_SUFFIX_CXX)
|
||||
set(
|
||||
__LIB_LIGHTGBM_FILENAME "${__LIB_LIGHTGBM_OUTPUT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX_CXX}"
|
||||
CACHE INTERNAL "lightgbm shared library filename"
|
||||
)
|
||||
else()
|
||||
set(
|
||||
__LIB_LIGHTGBM_FILENAME "${__LIB_LIGHTGBM_OUTPUT_NAME}.dylib"
|
||||
CACHE INTERNAL "lightgbm shared library filename"
|
||||
)
|
||||
endif()
|
||||
|
||||
# Override the absolute path to OpenMP with a relative one using @rpath.
|
||||
#
|
||||
# This also ensures that if a {libgomp,libiomp,libomp}.dylib has already been loaded, it'll just use that.
|
||||
add_custom_command(
|
||||
TARGET _lightgbm
|
||||
POST_BUILD
|
||||
COMMAND
|
||||
install_name_tool
|
||||
-change
|
||||
${OpenMP_LIBRARY_LOCATION}
|
||||
"@rpath/${OpenMP_LIBRARY_NAME}"
|
||||
"${__LIB_LIGHTGBM_FILENAME}"
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
COMMENT "Replacing hard-coded OpenMP install_name with '@rpath/${OpenMP_LIBRARY_NAME}'..."
|
||||
)
|
||||
# add RPATH entries to ensure the loader looks in the following, in the following order:
|
||||
#
|
||||
# - (R-only) ${LIBR_LIBS_DIR} (wherever R for macOS stores vendored third-party libraries)
|
||||
# - ${OpenMP_LIBRARY_DIR} (wherever find_package(OpenMP) found OpenMP at build time)
|
||||
# - /opt/homebrew/opt/libomp/lib (where 'brew install' / 'brew link' puts libomp.dylib)
|
||||
# - /opt/local/lib/libomp (where 'port install' puts libomp.dylib)
|
||||
#
|
||||
|
||||
# with some compilers, OpenMP ships with the compiler (e.g. libgomp with gcc)
|
||||
list(APPEND __omp_install_rpaths "${OpenMP_LIBRARY_DIR}")
|
||||
|
||||
# with clang, libomp doesn't ship with the compiler and might be supplied separately
|
||||
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
||||
list(
|
||||
APPEND __omp_install_rpaths
|
||||
"/opt/homebrew/opt/libomp/lib"
|
||||
"/opt/local/lib/libomp"
|
||||
)
|
||||
# It appears that CRAN's macOS binaries compiled with -fopenmp have install names
|
||||
# of the form:
|
||||
#
|
||||
# /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/lib/libomp.dylib
|
||||
#
|
||||
# That corresponds to the libomp.dylib that ships with the R framework for macOS, available
|
||||
# from https://cran.r-project.org/bin/macosx/.
|
||||
#
|
||||
# That absolute-path install name leads to that library being loaded unconditionally.
|
||||
#
|
||||
# That can result in e.g. 'library(data.table)' loading R's libomp.dylib and 'library(lightgbm)' loading
|
||||
# Homebrew's. Having 2 loaded in the same process can lead to segfaults and unpredictable behavior.
|
||||
#
|
||||
# This can't be easily avoided by forcing R-package builds in LightGBM to use R's libomp.dylib
|
||||
# at build time... LightGBM's CMake uses find_package(OpenMP), and R for macOS only provides the
|
||||
# library, not CMake config files for it.
|
||||
#
|
||||
# Best we can do, to allow CMake-based builds of the R-package here to continue to work
|
||||
# alongside CRAN-prepared binaries of other packages with OpenMP dependencies, is to
|
||||
# ensure that R's library directory is the first place the loader searches for
|
||||
# libomp.dylib when clang is used.
|
||||
#
|
||||
# ref: https://github.com/lightgbm-org/LightGBM/issues/6628
|
||||
#
|
||||
if(__BUILD_FOR_R)
|
||||
list(PREPEND __omp_install_rpaths "${LIBR_LIBS_DIR}")
|
||||
endif()
|
||||
endif()
|
||||
set_target_properties(
|
||||
_lightgbm
|
||||
PROPERTIES
|
||||
BUILD_WITH_INSTALL_RPATH TRUE
|
||||
INSTALL_RPATH "${__omp_install_rpaths}"
|
||||
INSTALL_RPATH_USE_LINK_PATH FALSE
|
||||
)
|
||||
endif()
|
||||
|
||||
install(
|
||||
TARGETS _lightgbm
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
|
||||
LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib
|
||||
ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib
|
||||
)
|
||||
|
||||
if(INSTALL_HEADERS)
|
||||
install(
|
||||
DIRECTORY ${LightGBM_HEADER_DIR}/LightGBM
|
||||
DESTINATION ${CMAKE_INSTALL_PREFIX}/include
|
||||
)
|
||||
install(
|
||||
FILES ${FAST_DOUBLE_PARSER_INCLUDE_DIR}/fast_double_parser.h
|
||||
DESTINATION ${CMAKE_INSTALL_PREFIX}/include/LightGBM/utils
|
||||
)
|
||||
install(
|
||||
DIRECTORY ${FMT_INCLUDE_DIR}/
|
||||
DESTINATION ${CMAKE_INSTALL_PREFIX}/include/LightGBM/utils
|
||||
FILES_MATCHING PATTERN "*.h"
|
||||
)
|
||||
endif()
|
||||
@@ -0,0 +1,41 @@
|
||||
# Microsoft Open Source Code of Conduct
|
||||
|
||||
This code of conduct outlines expectations for participation in Microsoft-managed open source communities, as well as steps for reporting unacceptable behavior. We are committed to providing a welcoming and inspiring community for all. People violating this code of conduct may be banned from the community.
|
||||
|
||||
## Our open source communities strive to:
|
||||
|
||||
* Be friendly and patient: Remember you might not be communicating in someone else's primary spoken or programming language, and others may not have your level of understanding.
|
||||
* Be welcoming: Our communities welcome and support people of all backgrounds and identities. This includes, but is not limited to members of any race, ethnicity, culture, national origin, color, immigration status, social and economic class, educational level, sex, sexual orientation, gender identity and expression, age, size, family status, political belief, religion, and mental and physical ability.
|
||||
* Be respectful: We are a world-wide community of professionals, and we conduct ourselves professionally. Disagreement is no excuse for poor behavior and poor manners. Disrespectful and unacceptable behavior includes, but is not limited to:
|
||||
1. Violent threats or language.
|
||||
2. Discriminatory or derogatory jokes and language.
|
||||
3. Posting sexually explicit or violent material.
|
||||
4. Posting, or threatening to post, people's personally identifying information ("doxing").
|
||||
5. Insults, especially those using discriminatory terms or slurs.
|
||||
Behavior that could be perceived as sexual attention.
|
||||
Advocating for or encouraging any of the above behaviors.
|
||||
* Understand disagreements: Disagreements, both social and technical, are useful learning opportunities. Seek to understand the other viewpoints and resolve differences constructively.
|
||||
* This code is not exhaustive or complete. It serves to capture our common understanding of a productive, collaborative environment. We expect the code to be followed in spirit as much as in the letter.
|
||||
|
||||
## Scope
|
||||
|
||||
This code of conduct applies to all repos and communities for Microsoft-managed open source projects regardless of whether or not the repo explicitly calls out its use of this code. The code also applies in public spaces when an individual is representing a project or its community. Examples include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
|
||||
|
||||
Note: Some Microsoft-managed communities have codes of conduct that pre-date this document and issue resolution process. While communities are not required to change their code, they are expected to use the resolution process outlined here. The review team will coordinate with the communities involved to address your concerns.
|
||||
|
||||
## Reporting Code of Conduct Issues
|
||||
|
||||
We encourage all communities to resolve issues on their own whenever possible. This builds a broader and deeper understanding and ultimately a healthier interaction. In the event that an issue cannot be resolved locally, please feel free to report your concerns by contacting opencode@microsoft.com. Your report will be handled in accordance with the issue resolution process described in the Code of Conduct FAQ.
|
||||
|
||||
In your report please include:
|
||||
|
||||
* Your contact information.
|
||||
* Names (real, usernames or pseudonyms) of any individuals involved. If there are additional witnesses, please include them as well.
|
||||
* Your account of what occurred, and if you believe the incident is ongoing. If there is a publicly available record (e.g. a mailing list archive or a public chat log), please include a link or attachment.
|
||||
* Any additional information that may be helpful.
|
||||
|
||||
All reports will be reviewed by a multi-person team and will result in a response that is deemed necessary and appropriate to the circumstances. Where additional perspectives are needed, the team may seek insight from others with relevant expertise or experience. The confidentiality of the person reporting the incident will be kept at all times. Involved parties are never part of the review team.
|
||||
|
||||
Anyone asked to stop unacceptable behavior is expected to comply immediately. If an individual engages in unacceptable behavior, the review team may take any action they deem appropriate, including a permanent ban from the community.
|
||||
|
||||
This code of conduct is based on the template established by the TODO Group and used by numerous other large communities (e.g., Facebook, Yahoo, Twitter, GitHub) and the Scope section from the Contributor Covenant version 1.4.
|
||||
@@ -0,0 +1,28 @@
|
||||
# contributing
|
||||
|
||||
LightGBM has been developed and used by many active community members.
|
||||
|
||||
Your help is very valuable to make it better for everyone.
|
||||
|
||||
## How to Contribute
|
||||
|
||||
- Check the [Feature Requests Hub](https://github.com/lightgbm-org/LightGBM/issues/2302), and submit pull requests to address chosen issue. If you need development guideline, you can check the [Development Guide](https://github.com/lightgbm-org/LightGBM/blob/main/docs/Development-Guide.rst) or directly ask us in Issues/Pull Requests.
|
||||
- Contribute to the [tests](https://github.com/lightgbm-org/LightGBM/tree/main/tests) to make it more reliable.
|
||||
- Contribute to the [documentation](https://github.com/lightgbm-org/LightGBM/tree/main/docs) to make it clearer for everyone.
|
||||
- Contribute to the [examples](https://github.com/lightgbm-org/LightGBM/tree/main/examples) to share your experience with other users.
|
||||
- Add your stories and experience to [Awesome LightGBM](https://github.com/lightgbm-org/LightGBM/blob/main/examples/README.md). If LightGBM helped you in a machine learning competition or some research application, we want to hear about it!
|
||||
- [Open an issue](https://github.com/lightgbm-org/LightGBM/issues) to report problems or recommend new features.
|
||||
|
||||
## Development Guide
|
||||
|
||||
### Linting
|
||||
|
||||
Every commit in the repository is tested with multiple static analyzers.
|
||||
|
||||
When developing locally, run some of them using `pre-commit` ([pre-commit docs](https://pre-commit.com/)).
|
||||
|
||||
```shell
|
||||
pre-commit run --all-files
|
||||
```
|
||||
|
||||
That command will check for some issues and automatically reformat the code.
|
||||
@@ -0,0 +1,22 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) Microsoft Corporation
|
||||
Copyright (c) The LightGBM developers
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
# maintaining
|
||||
|
||||
This document is for LightGBM maintainers.
|
||||
|
||||
## Managing Dependencies
|
||||
|
||||
### Locked Environments with `pixi`
|
||||
|
||||
This project uses `pixi` for tasks and CI jobs that run with a locked set of dependencies.
|
||||
|
||||
In general, updating these environments looks like:
|
||||
|
||||
1. manually modify `pixi.toml`
|
||||
2. run `pixi install`
|
||||
|
||||
And running inside one looks like:
|
||||
|
||||
```shell
|
||||
# interactive shell
|
||||
pixi shell -e py310
|
||||
|
||||
# run a task
|
||||
pixi run -e py310 python -c "import pandas; print(pandas.__version__)"
|
||||
```
|
||||
|
||||
See https://pixi.prefix.dev/latest/ for more details.
|
||||
|
||||
|
||||
## Releasing
|
||||
|
||||
### Step 1: Put up a Release PR
|
||||
|
||||
Create a pull request into `main` which prepares the source code for release.
|
||||
|
||||
Copy the description and checklist from the previous release PR (for example: https://github.com/lightgbm-org/LightGBM/pull/6796).
|
||||
|
||||
This should usually also include a checklist of other issues and PRs that should be completed for the release,
|
||||
and the PR should be used to discuss what makes it into the release.
|
||||
|
||||
### Step 2: Merge the Release PR
|
||||
|
||||
Once the PR is approved, merge it.
|
||||
|
||||
Do not merge any other PRs into `main` until the rest of the release is complete.
|
||||
|
||||
### Step 3: Wait for a New CI Run on `main`
|
||||
|
||||
Wait for all CI runs triggered by the merge to `main` to complete successfully.
|
||||
|
||||
These runs build and test the official artifacts that will be attached to the GitHub release and published to package managers.
|
||||
|
||||
### Step 4: Create a Release
|
||||
|
||||
Navigate to https://github.com/lightgbm-org/LightGBM/releases.
|
||||
|
||||
Click "edit" on the draft release that `release-drafter` has created there.
|
||||
|
||||
* update the tag and release title to match the version of LightGBM, in the format `v{major}.{minor}.{patch}`
|
||||
* ensure that tag points at the commit on `main` created by merging the release PR
|
||||
|
||||
When you're satisfied with the state of the release, click "Publish release".
|
||||
|
||||
### Step 5: Upload Artifacts
|
||||
|
||||
After creating a release, run the following from the root of the repo to populate it with artifacts.
|
||||
|
||||
```shell
|
||||
# download all artifacts to a local directory
|
||||
./.ci/download-artifacts.sh ${COMMIT_ID}
|
||||
|
||||
# attach them to the GitHub release
|
||||
gh release upload \
|
||||
--repo lightgbm-org/LightGBM \
|
||||
"${TAG}" \
|
||||
./release-artifacts/*
|
||||
```
|
||||
|
||||
Where:
|
||||
|
||||
* `COMMIT_ID` = full commit hash of the commit on `main` corresponding to the release
|
||||
* `TAG` = the tag for the release (e.g. `v4.6.0`)
|
||||
|
||||
### Step 6: Complete All Other Post-merge Release Steps
|
||||
|
||||
These include things like publishing to package managers, updating build configs for repackagers like ``conda-forge``, and many other steps.
|
||||
|
||||
See the release checklist on the PR for details.
|
||||
|
||||
## Nightly Packages
|
||||
|
||||
Nightly packages for the `lightgbm` Python package are uploaded to https://anaconda.org/lightgbm-packages on every merge to `main`.
|
||||
|
||||
That's done using an upload token stored in a secret in CI.
|
||||
Those tokens expire after 1 year.
|
||||
|
||||
To generate a new one, run the following.
|
||||
|
||||
```shell
|
||||
# install Anaconda CLI
|
||||
conda install -y -c conda-forge \
|
||||
anaconda-auth \
|
||||
anaconda-client
|
||||
|
||||
# authenticate locally
|
||||
anaconda auth login
|
||||
|
||||
# create a token (this expires after 1 year)
|
||||
TOKEN=$(
|
||||
anaconda org auth \
|
||||
--create \
|
||||
--name nightly-uploads \
|
||||
--org lightgbm-packages \
|
||||
--scopes 'api:read api:write pypi:upload'
|
||||
)
|
||||
```
|
||||
|
||||
That token can be used by maintainers to manually upload packages as well.
|
||||
|
||||
For example:
|
||||
|
||||
```shell
|
||||
./.ci/download-artifacts.sh $(git rev-parse HEAD)
|
||||
|
||||
# NOTE: set upload token in environment variable 'ANACONDA_API_TOKEN'
|
||||
anaconda upload \
|
||||
--package lightgbm \
|
||||
--force-metadata-update \
|
||||
-t pypi \
|
||||
./release-artifacts/*.whl
|
||||
```
|
||||
@@ -0,0 +1,52 @@
|
||||
\.appveyor\.yml
|
||||
AUTOCONF_UBUNTU_VERSION
|
||||
^autom4te.cache/.*$
|
||||
^.*\.bin
|
||||
^build_r.R$
|
||||
\.clang-format
|
||||
^.*\.clusterfuzzlite$
|
||||
^cran-comments\.md$
|
||||
^docs$
|
||||
^.*\.dll
|
||||
\.drone\.yml
|
||||
^.*\.dylib
|
||||
\.git
|
||||
\.gitkeep$
|
||||
^.*\.history
|
||||
^Makefile$
|
||||
^.*\.o
|
||||
^.*\.out
|
||||
^pkgdown$
|
||||
^recreate-configure\.sh$
|
||||
^.*\.so
|
||||
^src/build/.*$
|
||||
^src/CMakeLists.txt$
|
||||
^src/external_libs/compute/.appveyor.yml$
|
||||
^src/external_libs/compute/.coveralls.yml$
|
||||
^src/external_libs/compute/.travis.yml$
|
||||
^src/external_libs/compute/test/.*$
|
||||
^src/external_libs/compute/index.html$
|
||||
^src/external_libs/compute/.git$
|
||||
^src/external_libs/compute/.gitignore$
|
||||
^src/external_libs/compute/CONTRIBUTING.md$
|
||||
^src/external_libs/compute/README.md$
|
||||
src/external_libs/fast_double_parser/benchmarks
|
||||
src/external_libs/fast_double_parser/Makefile
|
||||
src/external_libs/fast_double_parser/.*\.md
|
||||
src/external_libs/fast_double_parser/tests
|
||||
src/external_libs/fast_double_parser/.*\.yaml
|
||||
src/external_libs/fast_double_parser/.*\.yml
|
||||
src/external_libs/fmt/.*\.md
|
||||
src/external_libs/fmt/.travis.yml
|
||||
src/external_libs/fmt/doc
|
||||
src/external_libs/fmt/support/Android\.mk
|
||||
src/external_libs/fmt/support/bazel/.bazel.*
|
||||
src/external_libs/fmt/support/.*\.gradle
|
||||
src/external_libs/fmt/support/.*\.pro
|
||||
src/external_libs/fmt/support/.*\.py
|
||||
src/external_libs/fmt/support/rtd
|
||||
src/external_libs/fmt/support/.*sublime-syntax
|
||||
src/external_libs/fmt/support/Vagrantfile
|
||||
src/external_libs/fmt/support/.*\.xml
|
||||
src/external_libs/fmt/support/.*\.yml
|
||||
src/external_libs/fmt/test
|
||||
@@ -0,0 +1 @@
|
||||
2.71-2
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
Package: lightgbm
|
||||
Type: Package
|
||||
Title: Light Gradient Boosting Machine
|
||||
Version: ~~VERSION~~
|
||||
Date: ~~DATE~~
|
||||
Authors@R: c(
|
||||
person("Yu", "Shi", email = "yushi2@microsoft.com", role = c("aut")),
|
||||
person("Guolin", "Ke", email = "guolin.ke@outlook.com", role = c("aut")),
|
||||
person("Damien", "Soukhavong", email = "damien.soukhavong@skema.edu", role = c("aut")),
|
||||
person("James", "Lamb", email="jaylamb20@gmail.com", role = c("aut", "cre")),
|
||||
person("Qi", "Meng", role = c("aut")),
|
||||
person("Thomas", "Finley", role = c("aut")),
|
||||
person("Taifeng", "Wang", role = c("aut")),
|
||||
person("Wei", "Chen", role = c("aut")),
|
||||
person("Weidong", "Ma", role = c("aut")),
|
||||
person("Qiwei", "Ye", role = c("aut")),
|
||||
person("Tie-Yan", "Liu", role = c("aut")),
|
||||
person("Nikita", "Titov", role = c("aut")),
|
||||
person("Yachen", "Yan", role = c("ctb")),
|
||||
person("Microsoft Corporation", role = c("cph")),
|
||||
person("Dropbox, Inc.", role = c("cph")),
|
||||
person("Alberto", "Ferreira", role = c("ctb")),
|
||||
person("Daniel", "Lemire", role = c("ctb")),
|
||||
person("Victor", "Zverovich", role = c("cph")),
|
||||
person("IBM Corporation", role = c("ctb")),
|
||||
person("David", "Cortes", role = c("aut")),
|
||||
person("Michael", "Mayer", role = c("ctb"))
|
||||
)
|
||||
Description: Tree based algorithms can be improved by introducing boosting frameworks.
|
||||
'LightGBM' is one such framework, based on Ke, Guolin et al. (2017) <https://proceedings.neurips.cc/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html>.
|
||||
This package offers an R interface to work with it.
|
||||
It is designed to be distributed and efficient with the following advantages:
|
||||
1. Faster training speed and higher efficiency.
|
||||
2. Lower memory usage.
|
||||
3. Better accuracy.
|
||||
4. Parallel learning supported.
|
||||
5. Capable of handling large-scale data.
|
||||
In recognition of these advantages, 'LightGBM' has been widely-used in many winning solutions of machine learning competitions.
|
||||
Comparison experiments on public datasets suggest that 'LightGBM' can outperform existing boosting frameworks on both efficiency and accuracy, with significantly lower memory consumption. In addition, parallel experiments suggest that in certain circumstances, 'LightGBM' can achieve a linear speed-up in training time by using multiple machines.
|
||||
Encoding: UTF-8
|
||||
License: MIT + file LICENSE
|
||||
URL: https://github.com/lightgbm-org/LightGBM
|
||||
BugReports: https://github.com/lightgbm-org/LightGBM/issues
|
||||
NeedsCompilation: yes
|
||||
Biarch: true
|
||||
VignetteBuilder: knitr
|
||||
Suggests:
|
||||
knitr,
|
||||
markdown,
|
||||
processx,
|
||||
RhpcBLASctl,
|
||||
testthat
|
||||
Depends:
|
||||
R (>= 4.0)
|
||||
Imports:
|
||||
R6 (>= 2.4.0),
|
||||
data.table (>= 1.9.6),
|
||||
graphics,
|
||||
jsonlite (>= 1.0),
|
||||
Matrix (>= 1.1-0),
|
||||
methods,
|
||||
parallel,
|
||||
utils
|
||||
SystemRequirements:
|
||||
C++17
|
||||
Config/roxygen2/version: 8.0.0
|
||||
@@ -0,0 +1,2 @@
|
||||
YEAR: 2016
|
||||
COPYRIGHT HOLDER: Microsoft Corporation
|
||||
@@ -0,0 +1,66 @@
|
||||
# Generated by roxygen2: do not edit by hand
|
||||
|
||||
S3method("dimnames<-",lgb.Dataset)
|
||||
S3method(dim,lgb.Dataset)
|
||||
S3method(dimnames,lgb.Dataset)
|
||||
S3method(get_field,lgb.Dataset)
|
||||
S3method(predict,lgb.Booster)
|
||||
S3method(print,lgb.Booster)
|
||||
S3method(set_field,lgb.Dataset)
|
||||
S3method(summary,lgb.Booster)
|
||||
export(getLGBMthreads)
|
||||
export(get_field)
|
||||
export(lgb.Dataset)
|
||||
export(lgb.Dataset.construct)
|
||||
export(lgb.Dataset.create.valid)
|
||||
export(lgb.Dataset.save)
|
||||
export(lgb.Dataset.set.categorical)
|
||||
export(lgb.Dataset.set.reference)
|
||||
export(lgb.configure_fast_predict)
|
||||
export(lgb.convert_with_rules)
|
||||
export(lgb.cv)
|
||||
export(lgb.drop_serialized)
|
||||
export(lgb.dump)
|
||||
export(lgb.get.eval.result)
|
||||
export(lgb.importance)
|
||||
export(lgb.interpret)
|
||||
export(lgb.interprete)
|
||||
export(lgb.load)
|
||||
export(lgb.make_serializable)
|
||||
export(lgb.model.dt.tree)
|
||||
export(lgb.plot.importance)
|
||||
export(lgb.plot.interpretation)
|
||||
export(lgb.restore_handle)
|
||||
export(lgb.save)
|
||||
export(lgb.slice.Dataset)
|
||||
export(lgb.train)
|
||||
export(lightgbm)
|
||||
export(setLGBMthreads)
|
||||
export(set_field)
|
||||
import(methods)
|
||||
importClassesFrom(Matrix,CsparseMatrix)
|
||||
importClassesFrom(Matrix,RsparseMatrix)
|
||||
importClassesFrom(Matrix,dgCMatrix)
|
||||
importClassesFrom(Matrix,dgRMatrix)
|
||||
importClassesFrom(Matrix,dsparseMatrix)
|
||||
importClassesFrom(Matrix,dsparseVector)
|
||||
importFrom(Matrix,Matrix)
|
||||
importFrom(R6,R6Class)
|
||||
importFrom(data.table,":=")
|
||||
importFrom(data.table,as.data.table)
|
||||
importFrom(data.table,data.table)
|
||||
importFrom(data.table,rbindlist)
|
||||
importFrom(data.table,set)
|
||||
importFrom(data.table,setnames)
|
||||
importFrom(data.table,setorder)
|
||||
importFrom(data.table,setorderv)
|
||||
importFrom(graphics,barplot)
|
||||
importFrom(graphics,par)
|
||||
importFrom(jsonlite,fromJSON)
|
||||
importFrom(methods,is)
|
||||
importFrom(methods,new)
|
||||
importFrom(parallel,detectCores)
|
||||
importFrom(stats,quantile)
|
||||
importFrom(utils,modifyList)
|
||||
importFrom(utils,read.delim)
|
||||
useDynLib(lightgbm , .registration = TRUE)
|
||||
@@ -0,0 +1,104 @@
|
||||
# Central location for parameter aliases.
|
||||
# See https://lightgbm.readthedocs.io/en/latest/Parameters.html#core-parameters
|
||||
|
||||
# [description] List of respected parameter aliases specific to lgb.Dataset. Wrapped in a function to
|
||||
# take advantage of lazy evaluation (so it doesn't matter what order
|
||||
# R sources files during installation).
|
||||
# [return] A named list, where each key is a parameter relevant to lgb.Dataset and each value is a character
|
||||
# vector of corresponding aliases.
|
||||
.DATASET_PARAMETERS <- function() {
|
||||
all_aliases <- .PARAMETER_ALIASES()
|
||||
return(all_aliases[c(
|
||||
"bin_construct_sample_cnt"
|
||||
, "categorical_feature"
|
||||
, "data_random_seed"
|
||||
, "enable_bundle"
|
||||
, "feature_pre_filter"
|
||||
, "forcedbins_filename"
|
||||
, "group_column"
|
||||
, "header"
|
||||
, "ignore_column"
|
||||
, "is_enable_sparse"
|
||||
, "label_column"
|
||||
, "linear_tree"
|
||||
, "max_bin"
|
||||
, "max_bin_by_feature"
|
||||
, "min_data_in_bin"
|
||||
, "pre_partition"
|
||||
, "precise_float_parser"
|
||||
, "two_round"
|
||||
, "use_missing"
|
||||
, "weight_column"
|
||||
, "zero_as_missing"
|
||||
)])
|
||||
}
|
||||
|
||||
# [description] Non-exported environment, used for caching details that only need to be
|
||||
# computed once per R session.
|
||||
.lgb_session_cache_env <- new.env()
|
||||
|
||||
# [description] List of respected parameter aliases. Wrapped in a function to take advantage of
|
||||
# lazy evaluation (so it doesn't matter what order R sources files during installation).
|
||||
# [return] A named list, where each key is a main LightGBM parameter and each value is a character
|
||||
# vector of corresponding aliases.
|
||||
.PARAMETER_ALIASES <- function() {
|
||||
if (exists("PARAMETER_ALIASES", where = .lgb_session_cache_env)) {
|
||||
return(get("PARAMETER_ALIASES", envir = .lgb_session_cache_env))
|
||||
}
|
||||
params_to_aliases <- jsonlite::fromJSON(
|
||||
.Call(
|
||||
LGBM_DumpParamAliases_R
|
||||
)
|
||||
)
|
||||
for (main_name in names(params_to_aliases)) {
|
||||
aliases_with_main_name <- c(main_name, unlist(params_to_aliases[[main_name]]))
|
||||
params_to_aliases[[main_name]] <- aliases_with_main_name
|
||||
}
|
||||
# store in cache so the next call to `.PARAMETER_ALIASES()` doesn't need to recompute this
|
||||
assign(
|
||||
x = "PARAMETER_ALIASES"
|
||||
, value = params_to_aliases
|
||||
, envir = .lgb_session_cache_env
|
||||
)
|
||||
return(params_to_aliases)
|
||||
}
|
||||
|
||||
# [description]
|
||||
# Per https://github.com/lightgbm-org/LightGBM/blob/main/docs/Parameters.rst#metric,
|
||||
# a few different strings can be used to indicate "no metrics".
|
||||
# [returns]
|
||||
# A character vector
|
||||
.NO_METRIC_STRINGS <- function() {
|
||||
return(
|
||||
c(
|
||||
"na"
|
||||
, "None"
|
||||
, "null"
|
||||
, "custom"
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
.MULTICLASS_OBJECTIVES <- function() {
|
||||
return(
|
||||
c(
|
||||
"multi_logloss"
|
||||
, "multiclass"
|
||||
, "softmax"
|
||||
, "multiclassova"
|
||||
, "multiclass_ova"
|
||||
, "ova"
|
||||
, "ovr"
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
.BINARY_OBJECTIVES <- function() {
|
||||
return(
|
||||
c(
|
||||
"binary_logloss"
|
||||
, "binary"
|
||||
, "binary_error"
|
||||
)
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
# constants that control naming in lists
|
||||
.EVAL_KEY <- function() {
|
||||
return("eval")
|
||||
}
|
||||
.EVAL_ERR_KEY <- function() {
|
||||
return("eval_err")
|
||||
}
|
||||
|
||||
#' @importFrom R6 R6Class
|
||||
CB_ENV <- R6::R6Class(
|
||||
"lgb.cb_env",
|
||||
cloneable = FALSE,
|
||||
public = list(
|
||||
model = NULL,
|
||||
iteration = NULL,
|
||||
begin_iteration = NULL,
|
||||
end_iteration = NULL,
|
||||
eval_list = list(),
|
||||
eval_err_list = list(),
|
||||
best_iter = -1L,
|
||||
best_score = NA,
|
||||
met_early_stop = FALSE
|
||||
)
|
||||
)
|
||||
|
||||
# Format the evaluation metric string
|
||||
.format_eval_string <- function(eval_res, eval_err) {
|
||||
|
||||
# Check for empty evaluation string
|
||||
if (is.null(eval_res) || length(eval_res) == 0L) {
|
||||
stop("no evaluation results")
|
||||
}
|
||||
|
||||
# Check for empty evaluation error
|
||||
if (!is.null(eval_err)) {
|
||||
return(sprintf("%s\'s %s:%g+%g", eval_res$data_name, eval_res$name, eval_res$value, eval_err))
|
||||
} else {
|
||||
return(sprintf("%s\'s %s:%g", eval_res$data_name, eval_res$name, eval_res$value))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
.merge_eval_string <- function(env) {
|
||||
|
||||
# Check length of evaluation list
|
||||
if (length(env$eval_list) <= 0L) {
|
||||
return("")
|
||||
}
|
||||
|
||||
# Get evaluation
|
||||
msg <- list(sprintf("[%d]:", env$iteration))
|
||||
|
||||
# Set if evaluation error
|
||||
is_eval_err <- length(env$eval_err_list) > 0L
|
||||
|
||||
# Loop through evaluation list
|
||||
for (j in seq_along(env$eval_list)) {
|
||||
|
||||
# Store evaluation error
|
||||
eval_err <- NULL
|
||||
if (isTRUE(is_eval_err)) {
|
||||
eval_err <- env$eval_err_list[[j]]
|
||||
}
|
||||
|
||||
# Set error message
|
||||
msg <- c(msg, .format_eval_string(eval_res = env$eval_list[[j]], eval_err = eval_err))
|
||||
|
||||
}
|
||||
|
||||
return(paste(msg, collapse = " "))
|
||||
|
||||
}
|
||||
|
||||
cb_print_evaluation <- function(period) {
|
||||
|
||||
# Create callback
|
||||
callback <- function(env) {
|
||||
|
||||
# Check if period is at least 1 or more
|
||||
if (period > 0L) {
|
||||
|
||||
# Store iteration
|
||||
i <- env$iteration
|
||||
|
||||
# Check if iteration matches moduo
|
||||
if ((i - 1L) %% period == 0L || is.element(i, c(env$begin_iteration, env$end_iteration))) {
|
||||
|
||||
# Merge evaluation string
|
||||
msg <- .merge_eval_string(env = env)
|
||||
|
||||
# Check if message is existing
|
||||
if (nchar(msg) > 0L) {
|
||||
cat(.merge_eval_string(env = env), "\n")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return(invisible(NULL))
|
||||
|
||||
}
|
||||
|
||||
# Store attributes
|
||||
attr(callback, "call") <- match.call()
|
||||
attr(callback, "name") <- "cb_print_evaluation"
|
||||
|
||||
return(callback)
|
||||
|
||||
}
|
||||
|
||||
cb_record_evaluation <- function() {
|
||||
|
||||
# Create callback
|
||||
callback <- function(env) {
|
||||
|
||||
if (length(env$eval_list) <= 0L) {
|
||||
return()
|
||||
}
|
||||
|
||||
# Set if evaluation error
|
||||
is_eval_err <- length(env$eval_err_list) > 0L
|
||||
|
||||
# Check length of recorded evaluation
|
||||
if (length(env$model$record_evals) == 0L) {
|
||||
|
||||
# Loop through each evaluation list element
|
||||
for (j in seq_along(env$eval_list)) {
|
||||
|
||||
# Store names
|
||||
data_name <- env$eval_list[[j]]$data_name
|
||||
name <- env$eval_list[[j]]$name
|
||||
env$model$record_evals$start_iter <- env$begin_iteration
|
||||
|
||||
# Check if evaluation record exists
|
||||
if (is.null(env$model$record_evals[[data_name]])) {
|
||||
env$model$record_evals[[data_name]] <- list()
|
||||
}
|
||||
|
||||
# Create dummy lists
|
||||
env$model$record_evals[[data_name]][[name]] <- list()
|
||||
env$model$record_evals[[data_name]][[name]][[.EVAL_KEY()]] <- list()
|
||||
env$model$record_evals[[data_name]][[name]][[.EVAL_ERR_KEY()]] <- list()
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
# Loop through each evaluation list element
|
||||
for (j in seq_along(env$eval_list)) {
|
||||
|
||||
# Get evaluation data
|
||||
eval_res <- env$eval_list[[j]]
|
||||
eval_err <- NULL
|
||||
if (isTRUE(is_eval_err)) {
|
||||
eval_err <- env$eval_err_list[[j]]
|
||||
}
|
||||
|
||||
# Store names
|
||||
data_name <- eval_res$data_name
|
||||
name <- eval_res$name
|
||||
|
||||
# Store evaluation data
|
||||
env$model$record_evals[[data_name]][[name]][[.EVAL_KEY()]] <- c(
|
||||
env$model$record_evals[[data_name]][[name]][[.EVAL_KEY()]]
|
||||
, eval_res$value
|
||||
)
|
||||
env$model$record_evals[[data_name]][[name]][[.EVAL_ERR_KEY()]] <- c(
|
||||
env$model$record_evals[[data_name]][[name]][[.EVAL_ERR_KEY()]]
|
||||
, eval_err
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
return(invisible(NULL))
|
||||
|
||||
}
|
||||
|
||||
# Store attributes
|
||||
attr(callback, "call") <- match.call()
|
||||
attr(callback, "name") <- "cb_record_evaluation"
|
||||
|
||||
return(callback)
|
||||
|
||||
}
|
||||
|
||||
cb_early_stop <- function(stopping_rounds, first_metric_only, verbose) {
|
||||
|
||||
factor_to_bigger_better <- NULL
|
||||
best_iter <- NULL
|
||||
best_score <- NULL
|
||||
best_msg <- NULL
|
||||
eval_len <- NULL
|
||||
|
||||
# Initialization function
|
||||
init <- function(env) {
|
||||
|
||||
# Early stopping cannot work without metrics
|
||||
if (length(env$eval_list) == 0L) {
|
||||
stop("For early stopping, valids must have at least one element")
|
||||
}
|
||||
|
||||
# Store evaluation length
|
||||
eval_len <<- length(env$eval_list)
|
||||
|
||||
# Check if verbose or not
|
||||
if (isTRUE(verbose)) {
|
||||
msg <- paste0(
|
||||
"Will train until there is no improvement in "
|
||||
, stopping_rounds
|
||||
, " rounds.\n"
|
||||
)
|
||||
cat(msg)
|
||||
}
|
||||
|
||||
# Internally treat everything as a maximization task
|
||||
factor_to_bigger_better <<- rep.int(1.0, eval_len)
|
||||
best_iter <<- rep.int(-1L, eval_len)
|
||||
best_score <<- rep.int(-Inf, eval_len)
|
||||
best_msg <<- list()
|
||||
|
||||
# Loop through evaluation elements
|
||||
for (i in seq_len(eval_len)) {
|
||||
|
||||
# Prepend message
|
||||
best_msg <<- c(best_msg, "")
|
||||
|
||||
# Internally treat everything as a maximization task
|
||||
if (!isTRUE(env$eval_list[[i]]$higher_better)) {
|
||||
factor_to_bigger_better[i] <<- -1.0
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return(invisible(NULL))
|
||||
|
||||
}
|
||||
|
||||
# Create callback
|
||||
callback <- function(env) {
|
||||
|
||||
# Check for empty evaluation
|
||||
if (is.null(eval_len)) {
|
||||
init(env = env)
|
||||
}
|
||||
|
||||
# Store iteration
|
||||
cur_iter <- env$iteration
|
||||
|
||||
# By default, any metric can trigger early stopping. This can be disabled
|
||||
# with 'first_metric_only = TRUE'
|
||||
if (isTRUE(first_metric_only)) {
|
||||
evals_to_check <- 1L
|
||||
} else {
|
||||
evals_to_check <- seq_len(eval_len)
|
||||
}
|
||||
|
||||
# Loop through evaluation
|
||||
for (i in evals_to_check) {
|
||||
|
||||
# Store score
|
||||
score <- env$eval_list[[i]]$value * factor_to_bigger_better[i]
|
||||
|
||||
# Check if score is better
|
||||
if (score > best_score[i]) {
|
||||
|
||||
# Store new scores
|
||||
best_score[i] <<- score
|
||||
best_iter[i] <<- cur_iter
|
||||
|
||||
# Prepare to print if verbose
|
||||
if (verbose) {
|
||||
best_msg[[i]] <<- as.character(.merge_eval_string(env = env))
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
# Check if early stopping is required
|
||||
if (cur_iter - best_iter[i] >= stopping_rounds) {
|
||||
|
||||
if (!is.null(env$model)) {
|
||||
env$model$best_score <- best_score[i]
|
||||
env$model$best_iter <- best_iter[i]
|
||||
}
|
||||
|
||||
if (isTRUE(verbose)) {
|
||||
cat(paste0("Early stopping, best iteration is: ", best_msg[[i]], "\n"))
|
||||
}
|
||||
|
||||
# Store best iteration and stop
|
||||
env$best_iter <- best_iter[i]
|
||||
env$met_early_stop <- TRUE
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (!isTRUE(env$met_early_stop) && cur_iter == env$end_iteration) {
|
||||
|
||||
if (!is.null(env$model)) {
|
||||
env$model$best_score <- best_score[i]
|
||||
env$model$best_iter <- best_iter[i]
|
||||
}
|
||||
|
||||
if (isTRUE(verbose)) {
|
||||
cat(paste0("Did not meet early stopping, best iteration is: ", best_msg[[i]], "\n"))
|
||||
}
|
||||
|
||||
# Store best iteration and stop
|
||||
env$best_iter <- best_iter[i]
|
||||
env$met_early_stop <- TRUE
|
||||
}
|
||||
}
|
||||
|
||||
return(invisible(NULL))
|
||||
|
||||
}
|
||||
|
||||
attr(callback, "call") <- match.call()
|
||||
attr(callback, "name") <- "cb_early_stop"
|
||||
|
||||
return(callback)
|
||||
|
||||
}
|
||||
|
||||
# Extract callback names from the list of callbacks
|
||||
.callback_names <- function(cb_list) {
|
||||
return(unlist(lapply(cb_list, attr, "name")))
|
||||
}
|
||||
|
||||
.add_cb <- function(cb_list, cb) {
|
||||
|
||||
# Combine two elements
|
||||
cb_list <- c(cb_list, cb)
|
||||
|
||||
# Set names of elements
|
||||
names(cb_list) <- .callback_names(cb_list = cb_list)
|
||||
|
||||
if ("cb_early_stop" %in% names(cb_list)) {
|
||||
|
||||
# Concatenate existing elements
|
||||
cb_list <- c(cb_list, cb_list["cb_early_stop"])
|
||||
|
||||
# Remove only the first one
|
||||
cb_list["cb_early_stop"] <- NULL
|
||||
|
||||
}
|
||||
|
||||
return(cb_list)
|
||||
|
||||
}
|
||||
|
||||
.categorize_callbacks <- function(cb_list) {
|
||||
|
||||
# Check for pre-iteration or post-iteration
|
||||
return(
|
||||
list(
|
||||
pre_iter = Filter(function(x) {
|
||||
pre <- attr(x, "is_pre_iteration")
|
||||
!is.null(pre) && pre
|
||||
}, cb_list),
|
||||
post_iter = Filter(function(x) {
|
||||
pre <- attr(x, "is_pre_iteration")
|
||||
is.null(pre) || !pre
|
||||
}, cb_list)
|
||||
)
|
||||
)
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,94 @@
|
||||
DataProcessor <- R6::R6Class(
|
||||
classname = "lgb.DataProcessor",
|
||||
public = list(
|
||||
factor_levels = NULL,
|
||||
|
||||
process_label = function(label, objective, params) {
|
||||
|
||||
if (is.character(label)) {
|
||||
label <- factor(label)
|
||||
}
|
||||
|
||||
if (is.factor(label)) {
|
||||
|
||||
self$factor_levels <- levels(label)
|
||||
if (length(self$factor_levels) <= 1L) {
|
||||
stop("Labels to predict is a factor with <2 possible values.")
|
||||
}
|
||||
|
||||
label <- as.numeric(label) - 1.0
|
||||
out <- list(label = label)
|
||||
if (length(self$factor_levels) == 2L) {
|
||||
if (objective == "auto") {
|
||||
objective <- "binary"
|
||||
}
|
||||
if (!(objective %in% .BINARY_OBJECTIVES())) {
|
||||
stop("Two-level factors as labels only allowed for objective='binary' or objective='auto'.")
|
||||
}
|
||||
} else {
|
||||
if (objective == "auto") {
|
||||
objective <- "multiclass"
|
||||
}
|
||||
if (!(objective %in% .MULTICLASS_OBJECTIVES())) {
|
||||
stop(
|
||||
sprintf(
|
||||
"Factors with >2 levels as labels only allowed for multi-class objectives. Got: %s (allowed: %s)"
|
||||
, objective
|
||||
, toString(.MULTICLASS_OBJECTIVES())
|
||||
)
|
||||
)
|
||||
}
|
||||
data_num_class <- length(self$factor_levels)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "num_class"
|
||||
, params = params
|
||||
, alternative_kwarg_value = data_num_class
|
||||
)
|
||||
if (params[["num_class"]] != data_num_class) {
|
||||
warning(
|
||||
sprintf(
|
||||
"Found num_class=%d in params, but 'label' is a factor with %d levels. 'num_class' will be ignored."
|
||||
, params[["num_class"]]
|
||||
, data_num_class
|
||||
)
|
||||
)
|
||||
params$num_class <- data_num_class
|
||||
}
|
||||
}
|
||||
out$objective <- objective
|
||||
out$params <- params
|
||||
return(out)
|
||||
|
||||
} else {
|
||||
|
||||
label <- as.numeric(label)
|
||||
if (objective == "auto") {
|
||||
objective <- "regression"
|
||||
}
|
||||
out <- list(
|
||||
label = label
|
||||
, objective = objective
|
||||
, params = params
|
||||
)
|
||||
return(out)
|
||||
|
||||
}
|
||||
},
|
||||
|
||||
process_predictions = function(pred, type) {
|
||||
if (NROW(self$factor_levels)) {
|
||||
if (type == "class") {
|
||||
pred <- as.integer(pred) + 1L
|
||||
attributes(pred)$levels <- self$factor_levels
|
||||
attributes(pred)$class <- "factor"
|
||||
} else if (type %in% c("response", "raw")) {
|
||||
if (is.matrix(pred) && ncol(pred) == length(self$factor_levels)) {
|
||||
colnames(pred) <- self$factor_levels
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return(pred)
|
||||
}
|
||||
)
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,529 @@
|
||||
#' @importFrom methods is new
|
||||
#' @importFrom R6 R6Class
|
||||
#' @importFrom utils read.delim
|
||||
#' @importClassesFrom Matrix dsparseMatrix dsparseVector dgCMatrix dgRMatrix CsparseMatrix RsparseMatrix
|
||||
Predictor <- R6::R6Class(
|
||||
|
||||
classname = "lgb.Predictor",
|
||||
cloneable = FALSE,
|
||||
public = list(
|
||||
|
||||
# Initialize will create a starter model
|
||||
initialize = function(modelfile, params = list(), fast_predict_config = list()) {
|
||||
private$params <- .params2str(params = params)
|
||||
handle <- NULL
|
||||
|
||||
if (is.character(modelfile)) {
|
||||
|
||||
# Create handle on it
|
||||
handle <- .Call(
|
||||
LGBM_BoosterCreateFromModelfile_R
|
||||
, path.expand(modelfile)
|
||||
)
|
||||
private$need_free_handle <- TRUE
|
||||
|
||||
} else if (methods::is(modelfile, "lgb.Booster.handle") || inherits(modelfile, "externalptr")) {
|
||||
|
||||
# Check if model file is a booster handle already
|
||||
handle <- modelfile
|
||||
private$need_free_handle <- FALSE
|
||||
|
||||
} else if (.is_Booster(modelfile)) {
|
||||
|
||||
handle <- modelfile$get_handle()
|
||||
private$need_free_handle <- FALSE
|
||||
|
||||
} else {
|
||||
|
||||
stop("lgb.Predictor: modelfile must be either a character filename or an lgb.Booster.handle")
|
||||
|
||||
}
|
||||
|
||||
private$fast_predict_config <- fast_predict_config
|
||||
|
||||
# Override class and store it
|
||||
class(handle) <- "lgb.Booster.handle"
|
||||
private$handle <- handle
|
||||
|
||||
return(invisible(NULL))
|
||||
|
||||
},
|
||||
|
||||
# Get current iteration
|
||||
current_iter = function() {
|
||||
|
||||
cur_iter <- 0L
|
||||
.Call(
|
||||
LGBM_BoosterGetCurrentIteration_R
|
||||
, private$handle
|
||||
, cur_iter
|
||||
)
|
||||
return(cur_iter)
|
||||
|
||||
},
|
||||
|
||||
# Predict from data
|
||||
predict = function(data,
|
||||
start_iteration = NULL,
|
||||
num_iteration = NULL,
|
||||
rawscore = FALSE,
|
||||
predleaf = FALSE,
|
||||
predcontrib = FALSE,
|
||||
header = FALSE) {
|
||||
|
||||
# Check if number of iterations is existing - if not, then set it to -1 (use all)
|
||||
if (is.null(num_iteration)) {
|
||||
num_iteration <- -1L
|
||||
}
|
||||
# Check if start iterations is existing - if not, then set it to 0 (start from the first iteration)
|
||||
if (is.null(start_iteration)) {
|
||||
start_iteration <- 0L
|
||||
}
|
||||
|
||||
# Check if data is a file name and not a matrix
|
||||
if (identical(class(data), "character") && length(data) == 1L) {
|
||||
|
||||
data <- path.expand(data)
|
||||
|
||||
# Data is a filename, create a temporary file with a "lightgbm_" pattern in it
|
||||
tmp_filename <- tempfile(pattern = "lightgbm_")
|
||||
on.exit(unlink(tmp_filename), add = TRUE)
|
||||
|
||||
# Predict from temporary file
|
||||
.Call(
|
||||
LGBM_BoosterPredictForFile_R
|
||||
, private$handle
|
||||
, data
|
||||
, as.integer(header)
|
||||
, as.integer(rawscore)
|
||||
, as.integer(predleaf)
|
||||
, as.integer(predcontrib)
|
||||
, as.integer(start_iteration)
|
||||
, as.integer(num_iteration)
|
||||
, private$params
|
||||
, tmp_filename
|
||||
)
|
||||
|
||||
# Get predictions from file
|
||||
preds <- utils::read.delim(tmp_filename, header = FALSE, sep = "\t")
|
||||
num_row <- nrow(preds)
|
||||
preds <- as.vector(t(preds))
|
||||
|
||||
} else if (predcontrib && inherits(data, c("dsparseMatrix", "dsparseVector"))) {
|
||||
|
||||
ncols <- .Call(LGBM_BoosterGetNumFeature_R, private$handle)
|
||||
ncols_out <- integer(1L)
|
||||
.Call(LGBM_BoosterGetNumClasses_R, private$handle, ncols_out)
|
||||
ncols_out <- (ncols + 1L) * max(ncols_out, 1L)
|
||||
if (is.na(ncols_out)) {
|
||||
ncols_out <- as.numeric(ncols + 1L) * as.numeric(max(ncols_out, 1L))
|
||||
}
|
||||
if (!inherits(data, "dsparseVector") && ncols_out > .Machine$integer.max) {
|
||||
stop("Resulting matrix of feature contributions is too large for R to handle.")
|
||||
}
|
||||
|
||||
if (inherits(data, "dsparseVector")) {
|
||||
|
||||
if (length(data) > ncols) {
|
||||
stop(sprintf("Model was fitted to data with %d columns, input data has %.0f columns."
|
||||
, ncols
|
||||
, length(data)))
|
||||
}
|
||||
res <- .Call(
|
||||
LGBM_BoosterPredictSparseOutput_R
|
||||
, private$handle
|
||||
, c(0L, as.integer(length(data@x)))
|
||||
, data@i - 1L
|
||||
, data@x
|
||||
, TRUE
|
||||
, 1L
|
||||
, ncols
|
||||
, start_iteration
|
||||
, num_iteration
|
||||
, private$params
|
||||
)
|
||||
out <- methods::new("dsparseVector")
|
||||
out@i <- res$indices + 1L
|
||||
out@x <- res$data
|
||||
out@length <- ncols_out
|
||||
return(out)
|
||||
|
||||
} else if (inherits(data, "dgRMatrix")) {
|
||||
|
||||
if (ncol(data) > ncols) {
|
||||
stop(sprintf("Model was fitted to data with %d columns, input data has %.0f columns."
|
||||
, ncols
|
||||
, ncol(data)))
|
||||
}
|
||||
res <- .Call(
|
||||
LGBM_BoosterPredictSparseOutput_R
|
||||
, private$handle
|
||||
, data@p
|
||||
, data@j
|
||||
, data@x
|
||||
, TRUE
|
||||
, nrow(data)
|
||||
, ncols
|
||||
, start_iteration
|
||||
, num_iteration
|
||||
, private$params
|
||||
)
|
||||
out <- methods::new("dgRMatrix")
|
||||
out@p <- res$indptr
|
||||
out@j <- res$indices
|
||||
out@x <- res$data
|
||||
out@Dim <- as.integer(c(nrow(data), ncols_out))
|
||||
|
||||
} else if (inherits(data, "dgCMatrix")) {
|
||||
|
||||
if (ncol(data) != ncols) {
|
||||
stop(sprintf("Model was fitted to data with %d columns, input data has %.0f columns."
|
||||
, ncols
|
||||
, ncol(data)))
|
||||
}
|
||||
res <- .Call(
|
||||
LGBM_BoosterPredictSparseOutput_R
|
||||
, private$handle
|
||||
, data@p
|
||||
, data@i
|
||||
, data@x
|
||||
, FALSE
|
||||
, nrow(data)
|
||||
, ncols
|
||||
, start_iteration
|
||||
, num_iteration
|
||||
, private$params
|
||||
)
|
||||
out <- methods::new("dgCMatrix")
|
||||
out@p <- res$indptr
|
||||
out@i <- res$indices
|
||||
out@x <- res$data
|
||||
out@Dim <- as.integer(c(nrow(data), length(res$indptr) - 1L))
|
||||
|
||||
} else {
|
||||
|
||||
stop(sprintf("Predictions on sparse inputs are only allowed for '%s', '%s', '%s' - got: %s"
|
||||
, "dsparseVector"
|
||||
, "dgRMatrix"
|
||||
, "dgCMatrix"
|
||||
, toString(class(data))))
|
||||
}
|
||||
|
||||
if (NROW(row.names(data))) {
|
||||
out@Dimnames[[1L]] <- row.names(data)
|
||||
}
|
||||
return(out)
|
||||
|
||||
} else {
|
||||
|
||||
# Not a file, we need to predict from R object
|
||||
num_row <- nrow(data)
|
||||
if (is.null(num_row)) {
|
||||
num_row <- 1L
|
||||
}
|
||||
|
||||
npred <- 0L
|
||||
|
||||
# Check number of predictions to do
|
||||
.Call(
|
||||
LGBM_BoosterCalcNumPredict_R
|
||||
, private$handle
|
||||
, as.integer(num_row)
|
||||
, as.integer(rawscore)
|
||||
, as.integer(predleaf)
|
||||
, as.integer(predcontrib)
|
||||
, as.integer(start_iteration)
|
||||
, as.integer(num_iteration)
|
||||
, npred
|
||||
)
|
||||
|
||||
# Pre-allocate empty vector
|
||||
preds <- numeric(npred)
|
||||
|
||||
# Check if data is a matrix
|
||||
if (is.matrix(data)) {
|
||||
# this if() prevents the memory and computational costs
|
||||
# of converting something that is already "double" to "double"
|
||||
if (storage.mode(data) != "double") {
|
||||
storage.mode(data) <- "double"
|
||||
}
|
||||
|
||||
if (nrow(data) == 1L) {
|
||||
|
||||
use_fast_config <- private$check_can_use_fast_predict_config(
|
||||
csr = FALSE
|
||||
, rawscore = rawscore
|
||||
, predleaf = predleaf
|
||||
, predcontrib = predcontrib
|
||||
, start_iteration = start_iteration
|
||||
, num_iteration = num_iteration
|
||||
)
|
||||
|
||||
if (use_fast_config) {
|
||||
.Call(
|
||||
LGBM_BoosterPredictForMatSingleRowFast_R
|
||||
, private$fast_predict_config$handle
|
||||
, data
|
||||
, preds
|
||||
)
|
||||
} else {
|
||||
.Call(
|
||||
LGBM_BoosterPredictForMatSingleRow_R
|
||||
, private$handle
|
||||
, data
|
||||
, rawscore
|
||||
, predleaf
|
||||
, predcontrib
|
||||
, start_iteration
|
||||
, num_iteration
|
||||
, private$params
|
||||
, preds
|
||||
)
|
||||
}
|
||||
|
||||
} else {
|
||||
.Call(
|
||||
LGBM_BoosterPredictForMat_R
|
||||
, private$handle
|
||||
, data
|
||||
, as.integer(nrow(data))
|
||||
, as.integer(ncol(data))
|
||||
, as.integer(rawscore)
|
||||
, as.integer(predleaf)
|
||||
, as.integer(predcontrib)
|
||||
, as.integer(start_iteration)
|
||||
, as.integer(num_iteration)
|
||||
, private$params
|
||||
, preds
|
||||
)
|
||||
}
|
||||
|
||||
} else if (inherits(data, "dsparseVector")) {
|
||||
|
||||
if (length(self$fast_predict_config)) {
|
||||
ncols <- self$fast_predict_config$ncols
|
||||
use_fast_config <- private$check_can_use_fast_predict_config(
|
||||
csr = TRUE
|
||||
, rawscore = rawscore
|
||||
, predleaf = predleaf
|
||||
, predcontrib = predcontrib
|
||||
, start_iteration = start_iteration
|
||||
, num_iteration = num_iteration
|
||||
)
|
||||
} else {
|
||||
ncols <- .Call(LGBM_BoosterGetNumFeature_R, private$handle)
|
||||
use_fast_config <- FALSE
|
||||
}
|
||||
|
||||
if (length(data) > ncols) {
|
||||
stop(sprintf("Model was fitted to data with %d columns, input data has %.0f columns."
|
||||
, ncols
|
||||
, length(data)))
|
||||
}
|
||||
|
||||
if (use_fast_config) {
|
||||
.Call(
|
||||
LGBM_BoosterPredictForCSRSingleRowFast_R
|
||||
, self$fast_predict_config$handle
|
||||
, data@i - 1L
|
||||
, data@x
|
||||
, preds
|
||||
)
|
||||
} else {
|
||||
.Call(
|
||||
LGBM_BoosterPredictForCSRSingleRow_R
|
||||
, private$handle
|
||||
, data@i - 1L
|
||||
, data@x
|
||||
, ncols
|
||||
, as.integer(rawscore)
|
||||
, as.integer(predleaf)
|
||||
, as.integer(predcontrib)
|
||||
, start_iteration
|
||||
, num_iteration
|
||||
, private$params
|
||||
, preds
|
||||
)
|
||||
}
|
||||
|
||||
} else if (inherits(data, "dgRMatrix")) {
|
||||
|
||||
ncols <- .Call(LGBM_BoosterGetNumFeature_R, private$handle)
|
||||
if (ncol(data) > ncols) {
|
||||
stop(sprintf("Model was fitted to data with %d columns, input data has %.0f columns."
|
||||
, ncols
|
||||
, ncol(data)))
|
||||
}
|
||||
|
||||
if (nrow(data) == 1L) {
|
||||
|
||||
if (length(self$fast_predict_config)) {
|
||||
ncols <- self$fast_predict_config$ncols
|
||||
use_fast_config <- private$check_can_use_fast_predict_config(
|
||||
csr = TRUE
|
||||
, rawscore = rawscore
|
||||
, predleaf = predleaf
|
||||
, predcontrib = predcontrib
|
||||
, start_iteration = start_iteration
|
||||
, num_iteration = num_iteration
|
||||
)
|
||||
} else {
|
||||
ncols <- .Call(LGBM_BoosterGetNumFeature_R, private$handle)
|
||||
use_fast_config <- FALSE
|
||||
}
|
||||
|
||||
if (use_fast_config) {
|
||||
.Call(
|
||||
LGBM_BoosterPredictForCSRSingleRowFast_R
|
||||
, self$fast_predict_config$handle
|
||||
, data@j
|
||||
, data@x
|
||||
, preds
|
||||
)
|
||||
} else {
|
||||
.Call(
|
||||
LGBM_BoosterPredictForCSRSingleRow_R
|
||||
, private$handle
|
||||
, data@j
|
||||
, data@x
|
||||
, ncols
|
||||
, as.integer(rawscore)
|
||||
, as.integer(predleaf)
|
||||
, as.integer(predcontrib)
|
||||
, start_iteration
|
||||
, num_iteration
|
||||
, private$params
|
||||
, preds
|
||||
)
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
.Call(
|
||||
LGBM_BoosterPredictForCSR_R
|
||||
, private$handle
|
||||
, data@p
|
||||
, data@j
|
||||
, data@x
|
||||
, ncols
|
||||
, as.integer(rawscore)
|
||||
, as.integer(predleaf)
|
||||
, as.integer(predcontrib)
|
||||
, start_iteration
|
||||
, num_iteration
|
||||
, private$params
|
||||
, preds
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
} else if (methods::is(data, "dgCMatrix")) {
|
||||
if (length(data@p) > 2147483647L) {
|
||||
stop("Cannot support large CSC matrix")
|
||||
}
|
||||
# Check if data is a dgCMatrix (sparse matrix, column compressed format)
|
||||
.Call(
|
||||
LGBM_BoosterPredictForCSC_R
|
||||
, private$handle
|
||||
, data@p
|
||||
, data@i
|
||||
, data@x
|
||||
, length(data@p)
|
||||
, length(data@x)
|
||||
, nrow(data)
|
||||
, as.integer(rawscore)
|
||||
, as.integer(predleaf)
|
||||
, as.integer(predcontrib)
|
||||
, as.integer(start_iteration)
|
||||
, as.integer(num_iteration)
|
||||
, private$params
|
||||
, preds
|
||||
)
|
||||
|
||||
} else {
|
||||
|
||||
stop("predict: cannot predict on data of class ", sQuote(class(data), q = FALSE))
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
# Check if number of rows is strange (not a multiple of the dataset rows)
|
||||
if (length(preds) %% num_row != 0L) {
|
||||
stop(
|
||||
"predict: prediction length "
|
||||
, sQuote(length(preds), q = FALSE)
|
||||
, " is not a multiple of nrows(data): "
|
||||
, sQuote(num_row, q = FALSE)
|
||||
)
|
||||
}
|
||||
|
||||
# Get number of cases per row
|
||||
npred_per_case <- length(preds) / num_row
|
||||
|
||||
# Data reshaping
|
||||
if (npred_per_case > 1L || predleaf || predcontrib) {
|
||||
preds <- matrix(preds, ncol = npred_per_case, byrow = TRUE)
|
||||
}
|
||||
|
||||
# Keep row names if possible
|
||||
if (NROW(row.names(data)) && NROW(data) == NROW(preds)) {
|
||||
if (is.null(dim(preds))) {
|
||||
names(preds) <- row.names(data)
|
||||
} else {
|
||||
row.names(preds) <- row.names(data)
|
||||
}
|
||||
}
|
||||
|
||||
return(preds)
|
||||
}
|
||||
|
||||
),
|
||||
private = list(
|
||||
handle = NULL
|
||||
, need_free_handle = FALSE
|
||||
, params = ""
|
||||
, fast_predict_config = list()
|
||||
, check_can_use_fast_predict_config = function(csr,
|
||||
rawscore,
|
||||
predleaf,
|
||||
predcontrib,
|
||||
start_iteration,
|
||||
num_iteration) {
|
||||
|
||||
if (!NROW(private$fast_predict_config)) {
|
||||
return(FALSE)
|
||||
}
|
||||
|
||||
if (.is_null_handle(private$fast_predict_config$handle)) {
|
||||
warning(paste0("Model had fast CSR predict configuration, but it is inactive."
|
||||
, " Try re-generating it through 'lgb.configure_fast_predict'."))
|
||||
return(FALSE)
|
||||
}
|
||||
|
||||
if (isTRUE(csr) != private$fast_predict_config$csr) {
|
||||
return(FALSE)
|
||||
}
|
||||
|
||||
return(
|
||||
private$params == "" &&
|
||||
private$fast_predict_config$rawscore == rawscore &&
|
||||
private$fast_predict_config$predleaf == predleaf &&
|
||||
private$fast_predict_config$predcontrib == predcontrib &&
|
||||
.equal_or_both_null(private$fast_predict_config$start_iteration, start_iteration) &&
|
||||
.equal_or_both_null(private$fast_predict_config$num_iteration, num_iteration)
|
||||
)
|
||||
}
|
||||
|
||||
# finalize() will free up the handles
|
||||
, finalize = function() {
|
||||
if (private$need_free_handle) {
|
||||
.Call(
|
||||
LGBM_BoosterFree_R
|
||||
, private$handle
|
||||
)
|
||||
private$handle <- NULL
|
||||
}
|
||||
return(invisible(NULL))
|
||||
}
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,207 @@
|
||||
# [description] get all column classes of a data.table or data.frame.
|
||||
# This function collapses the result of class() into a single string
|
||||
.get_column_classes <- function(df) {
|
||||
return(
|
||||
vapply(
|
||||
X = df
|
||||
, FUN = function(x) {
|
||||
paste(class(x), collapse = ",")
|
||||
}
|
||||
, FUN.VALUE = character(1L)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
# [description] check a data frame or data table for columns that are any
|
||||
# type other than numeric and integer. This is used by lgb.convert_with_rules()
|
||||
# to warn if more action is needed by users
|
||||
# before a dataset can be converted to a lgb.Dataset.
|
||||
.warn_for_unconverted_columns <- function(df, function_name) {
|
||||
column_classes <- .get_column_classes(df = df)
|
||||
unconverted_columns <- column_classes[!(column_classes %in% c("numeric", "integer"))]
|
||||
if (length(unconverted_columns) > 0L) {
|
||||
col_detail_string <- toString(
|
||||
paste0(
|
||||
names(unconverted_columns)
|
||||
, " ("
|
||||
, unconverted_columns
|
||||
, ")"
|
||||
)
|
||||
)
|
||||
msg <- paste0(
|
||||
function_name
|
||||
, ": "
|
||||
, length(unconverted_columns)
|
||||
, " columns are not numeric or integer. These need to be dropped or converted to "
|
||||
, "be used in an lgb.Dataset object. "
|
||||
, col_detail_string
|
||||
)
|
||||
warning(msg)
|
||||
}
|
||||
return(invisible(NULL))
|
||||
}
|
||||
|
||||
.LGB_CONVERT_DEFAULT_FOR_LOGICAL_NA <- function() {
|
||||
return(-1L)
|
||||
}
|
||||
.LGB_CONVERT_DEFAULT_FOR_NON_LOGICAL_NA <- function() {
|
||||
return(0L)
|
||||
}
|
||||
|
||||
|
||||
#' @name lgb.convert_with_rules
|
||||
#' @title Data preparator for LightGBM datasets with rules (integer)
|
||||
#' @description Attempts to prepare a clean dataset to prepare to put in a \code{lgb.Dataset}.
|
||||
#' Factor, character, and logical columns are converted to integer. Missing values
|
||||
#' in factors and characters will be filled with 0L. Missing values in logicals
|
||||
#' will be filled with -1L.
|
||||
#'
|
||||
#' This function returns and optionally takes in "rules" the describe exactly
|
||||
#' how to convert values in columns.
|
||||
#'
|
||||
#' Columns that contain only NA values will be converted by this function but will
|
||||
#' not show up in the returned \code{rules}.
|
||||
#'
|
||||
#' NOTE: In previous releases of LightGBM, this function was called \code{lgb.prepare_rules2}.
|
||||
#' @param data A data.frame or data.table to prepare.
|
||||
#' @param rules A set of rules from the data preparator, if already used. This should be an R list,
|
||||
#' where names are column names in \code{data} and values are named character
|
||||
#' vectors whose names are column values and whose values are new values to
|
||||
#' replace them with.
|
||||
#' @return A list with the cleaned dataset (\code{data}) and the rules (\code{rules}).
|
||||
#' Note that the data must be converted to a matrix format (\code{as.matrix}) for input in
|
||||
#' \code{lgb.Dataset}.
|
||||
#'
|
||||
#' @examples
|
||||
#' \donttest{
|
||||
#' data(iris)
|
||||
#'
|
||||
#' str(iris)
|
||||
#'
|
||||
#' new_iris <- lgb.convert_with_rules(data = iris)
|
||||
#' str(new_iris$data)
|
||||
#'
|
||||
#' data(iris) # Erase iris dataset
|
||||
#' iris$Species[1L] <- "NEW FACTOR" # Introduce junk factor (NA)
|
||||
#'
|
||||
#' # Use conversion using known rules
|
||||
#' # Unknown factors become 0, excellent for sparse datasets
|
||||
#' newer_iris <- lgb.convert_with_rules(data = iris, rules = new_iris$rules)
|
||||
#'
|
||||
#' # Unknown factor is now zero, perfect for sparse datasets
|
||||
#' newer_iris$data[1L, ] # Species became 0 as it is an unknown factor
|
||||
#'
|
||||
#' newer_iris$data[1L, 5L] <- 1.0 # Put back real initial value
|
||||
#'
|
||||
#' # Is the newly created dataset equal? YES!
|
||||
#' all.equal(new_iris$data, newer_iris$data)
|
||||
#'
|
||||
#' # Can we test our own rules?
|
||||
#' data(iris) # Erase iris dataset
|
||||
#'
|
||||
#' # We remapped values differently
|
||||
#' personal_rules <- list(
|
||||
#' Species = c(
|
||||
#' "setosa" = 3L
|
||||
#' , "versicolor" = 2L
|
||||
#' , "virginica" = 1L
|
||||
#' )
|
||||
#' )
|
||||
#' newest_iris <- lgb.convert_with_rules(data = iris, rules = personal_rules)
|
||||
#' str(newest_iris$data) # SUCCESS!
|
||||
#' }
|
||||
#' @importFrom data.table set
|
||||
#' @export
|
||||
lgb.convert_with_rules <- function(data, rules = NULL) {
|
||||
|
||||
column_classes <- .get_column_classes(df = data)
|
||||
|
||||
is_data_table <- data.table::is.data.table(x = data)
|
||||
is_data_frame <- is.data.frame(data)
|
||||
|
||||
if (!(is_data_table || is_data_frame)) {
|
||||
stop(
|
||||
"lgb.convert_with_rules: you provided "
|
||||
, paste(class(data), collapse = " & ")
|
||||
, " but data should have class data.frame or data.table"
|
||||
)
|
||||
}
|
||||
|
||||
# if user didn't provide rules, create them
|
||||
if (is.null(rules)) {
|
||||
rules <- list()
|
||||
columns_to_fix <- which(column_classes %in% c("character", "factor", "logical"))
|
||||
|
||||
for (i in columns_to_fix) {
|
||||
|
||||
col_values <- data[[i]]
|
||||
|
||||
# Get unique values
|
||||
if (is.factor(col_values)) {
|
||||
unique_vals <- levels(col_values)
|
||||
unique_vals <- unique_vals[!is.na(unique_vals)]
|
||||
mini_numeric <- seq_along(unique_vals) # respect ordinal
|
||||
} else if (is.character(col_values)) {
|
||||
unique_vals <- as.factor(unique(col_values))
|
||||
unique_vals <- unique_vals[!is.na(unique_vals)]
|
||||
mini_numeric <- as.integer(unique_vals) # no respect for ordinal
|
||||
} else if (is.logical(col_values)) {
|
||||
unique_vals <- c(FALSE, TRUE)
|
||||
mini_numeric <- c(0L, 1L)
|
||||
}
|
||||
|
||||
# don't add rules for all-NA columns
|
||||
if (length(unique_vals) > 0L) {
|
||||
col_name <- names(data)[i]
|
||||
rules[[col_name]] <- mini_numeric
|
||||
names(rules[[col_name]]) <- unique_vals
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (col_name in names(rules)) {
|
||||
if (column_classes[[col_name]] == "logical") {
|
||||
default_value_for_na <- .LGB_CONVERT_DEFAULT_FOR_LOGICAL_NA()
|
||||
} else {
|
||||
default_value_for_na <- .LGB_CONVERT_DEFAULT_FOR_NON_LOGICAL_NA()
|
||||
}
|
||||
if (is_data_table) {
|
||||
data.table::set(
|
||||
x = data
|
||||
, j = col_name
|
||||
, value = unname(rules[[col_name]][data[[col_name]]])
|
||||
)
|
||||
data[is.na(get(col_name)), (col_name) := default_value_for_na]
|
||||
} else {
|
||||
data[[col_name]] <- unname(rules[[col_name]][data[[col_name]]])
|
||||
data[is.na(data[col_name]), col_name] <- default_value_for_na
|
||||
}
|
||||
}
|
||||
|
||||
# if any all-NA columns exist, they won't be in rules. Convert them
|
||||
all_na_cols <- which(
|
||||
sapply(
|
||||
X = data
|
||||
, FUN = function(x) {
|
||||
(is.factor(x) || is.character(x) || is.logical(x)) && all(is.na(unique(x)))
|
||||
}
|
||||
)
|
||||
)
|
||||
for (col_name in all_na_cols) {
|
||||
if (column_classes[[col_name]] == "logical") {
|
||||
default_value_for_na <- .LGB_CONVERT_DEFAULT_FOR_LOGICAL_NA()
|
||||
} else {
|
||||
default_value_for_na <- .LGB_CONVERT_DEFAULT_FOR_NON_LOGICAL_NA()
|
||||
}
|
||||
if (is_data_table) {
|
||||
data[, (col_name) := rep(default_value_for_na, .N)]
|
||||
} else {
|
||||
data[[col_name]] <- default_value_for_na
|
||||
}
|
||||
}
|
||||
|
||||
.warn_for_unconverted_columns(df = data, function_name = "lgb.convert_with_rules")
|
||||
|
||||
return(list(data = data, rules = rules))
|
||||
|
||||
}
|
||||
@@ -0,0 +1,617 @@
|
||||
#' @importFrom R6 R6Class
|
||||
CVBooster <- R6::R6Class(
|
||||
classname = "lgb.CVBooster",
|
||||
cloneable = FALSE,
|
||||
public = list(
|
||||
best_iter = -1L,
|
||||
best_score = NA,
|
||||
record_evals = list(),
|
||||
boosters = list(),
|
||||
initialize = function(x) {
|
||||
self$boosters <- x
|
||||
return(invisible(NULL))
|
||||
},
|
||||
reset_parameter = function(new_params) {
|
||||
for (x in self$boosters) {
|
||||
x[["booster"]]$reset_parameter(params = new_params)
|
||||
}
|
||||
return(invisible(self))
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
#' @name lgb.cv
|
||||
#' @title Main CV logic for LightGBM
|
||||
#' @description Cross validation logic used by LightGBM
|
||||
#' @inheritParams lgb_shared_params
|
||||
#' @param nfold the original dataset is randomly partitioned into \code{nfold} equal size subsamples.
|
||||
#' @param record Boolean, TRUE will record iteration message to \code{booster$record_evals}
|
||||
#' @param showsd \code{boolean}, whether to show standard deviation of cross validation.
|
||||
#' This parameter defaults to \code{TRUE}. Setting it to \code{FALSE} can lead to a
|
||||
#' slight speedup by avoiding unnecessary computation.
|
||||
#' @param stratified a \code{boolean} indicating whether sampling of folds should be stratified
|
||||
#' by the values of outcome labels.
|
||||
#' @param folds \code{list} provides a possibility to use a list of pre-defined CV folds
|
||||
#' (each element must be a vector of test fold's indices). When folds are supplied,
|
||||
#' the \code{nfold} and \code{stratified} parameters are ignored.
|
||||
#' @param callbacks List of callback functions that are applied at each iteration.
|
||||
#' @param reset_data Boolean, setting it to TRUE (not the default value) will transform the booster model
|
||||
#' into a predictor model which frees up memory and the original datasets
|
||||
#' @param eval_train_metric \code{boolean}, whether to add the cross validation results on the
|
||||
#' training data. This parameter defaults to \code{FALSE}. Setting it to \code{TRUE}
|
||||
#' will increase run time.
|
||||
#' @inheritSection lgb_shared_params Early Stopping
|
||||
#' @return a trained model \code{lgb.CVBooster}.
|
||||
#'
|
||||
#' @examples
|
||||
#' \donttest{
|
||||
#' \dontshow{setLGBMthreads(2L)}
|
||||
#' \dontshow{data.table::setDTthreads(1L)}
|
||||
#' data(agaricus.train, package = "lightgbm")
|
||||
#' train <- agaricus.train
|
||||
#' dtrain <- lgb.Dataset(train$data, label = train$label)
|
||||
#' params <- list(
|
||||
#' objective = "regression"
|
||||
#' , metric = "l2"
|
||||
#' , min_data = 1L
|
||||
#' , learning_rate = 1.0
|
||||
#' , num_threads = 2L
|
||||
#' )
|
||||
#' model <- lgb.cv(
|
||||
#' params = params
|
||||
#' , data = dtrain
|
||||
#' , nrounds = 5L
|
||||
#' , nfold = 3L
|
||||
#' )
|
||||
#' }
|
||||
#'
|
||||
#' @importFrom data.table data.table setorderv
|
||||
#' @export
|
||||
lgb.cv <- function(params = list()
|
||||
, data
|
||||
, nrounds = 100L
|
||||
, nfold = 3L
|
||||
, obj = NULL
|
||||
, eval = NULL
|
||||
, verbose = 1L
|
||||
, record = TRUE
|
||||
, eval_freq = 1L
|
||||
, showsd = TRUE
|
||||
, stratified = TRUE
|
||||
, folds = NULL
|
||||
, init_model = NULL
|
||||
, early_stopping_rounds = NULL
|
||||
, callbacks = list()
|
||||
, reset_data = FALSE
|
||||
, serializable = TRUE
|
||||
, eval_train_metric = FALSE
|
||||
) {
|
||||
|
||||
if (nrounds <= 0L) {
|
||||
stop("nrounds should be greater than zero")
|
||||
}
|
||||
if (!.is_Dataset(x = data)) {
|
||||
stop("lgb.cv: data must be an lgb.Dataset instance")
|
||||
}
|
||||
|
||||
# set some parameters, resolving the way they were passed in with other parameters
|
||||
# in `params`.
|
||||
# this ensures that the model stored with Booster$save() correctly represents
|
||||
# what was passed in
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "verbosity"
|
||||
, params = params
|
||||
, alternative_kwarg_value = verbose
|
||||
)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "num_iterations"
|
||||
, params = params
|
||||
, alternative_kwarg_value = nrounds
|
||||
)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "metric"
|
||||
, params = params
|
||||
, alternative_kwarg_value = NULL
|
||||
)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "objective"
|
||||
, params = params
|
||||
, alternative_kwarg_value = obj
|
||||
)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "early_stopping_round"
|
||||
, params = params
|
||||
, alternative_kwarg_value = early_stopping_rounds
|
||||
)
|
||||
early_stopping_rounds <- params[["early_stopping_round"]]
|
||||
|
||||
# extract any function objects passed for objective or metric
|
||||
fobj <- NULL
|
||||
if (is.function(params$objective)) {
|
||||
fobj <- params$objective
|
||||
params$objective <- "none"
|
||||
}
|
||||
|
||||
# If eval is a single function, store it as a 1-element list
|
||||
# (for backwards compatibility). If it is a list of functions, store
|
||||
# all of them. This makes it possible to pass any mix of strings like "auc"
|
||||
# and custom functions to eval
|
||||
params <- .check_eval(params = params, eval = eval)
|
||||
eval_functions <- list(NULL)
|
||||
if (is.function(eval)) {
|
||||
eval_functions <- list(eval)
|
||||
}
|
||||
if (methods::is(eval, "list")) {
|
||||
eval_functions <- Filter(
|
||||
f = is.function
|
||||
, x = eval
|
||||
)
|
||||
}
|
||||
|
||||
# Init predictor to empty
|
||||
predictor <- NULL
|
||||
|
||||
# Check for boosting from a trained model
|
||||
if (is.character(init_model)) {
|
||||
predictor <- Predictor$new(modelfile = init_model)
|
||||
} else if (.is_Booster(x = init_model)) {
|
||||
predictor <- init_model$to_predictor()
|
||||
}
|
||||
|
||||
# Set the iteration to start from / end to (and check for boosting from a trained model, again)
|
||||
begin_iteration <- 1L
|
||||
if (!is.null(predictor)) {
|
||||
begin_iteration <- predictor$current_iter() + 1L
|
||||
}
|
||||
end_iteration <- begin_iteration + params[["num_iterations"]] - 1L
|
||||
|
||||
# pop interaction_constraints off of params. It needs some preprocessing on the
|
||||
# R side before being passed into the Dataset object
|
||||
interaction_constraints <- params[["interaction_constraints"]]
|
||||
params["interaction_constraints"] <- NULL
|
||||
|
||||
# Construct datasets, if needed
|
||||
data$update_params(params = params)
|
||||
data$construct()
|
||||
|
||||
# Check interaction constraints
|
||||
params[["interaction_constraints"]] <- .check_interaction_constraints(
|
||||
interaction_constraints = interaction_constraints
|
||||
, column_names = data$get_colnames()
|
||||
)
|
||||
|
||||
# Update parameters with parsed parameters
|
||||
data$update_params(params = params)
|
||||
|
||||
# Create the predictor set
|
||||
data$.__enclos_env__$private$set_predictor(predictor = predictor)
|
||||
|
||||
if (!is.null(folds)) {
|
||||
|
||||
# Check for list of folds or for single value
|
||||
if (!identical(class(folds), "list") || length(folds) < 2L) {
|
||||
stop(sQuote("folds"), " must be a list with 2 or more elements that are vectors of indices for each CV-fold")
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
if (nfold <= 1L) {
|
||||
stop(sQuote("nfold"), " must be > 1")
|
||||
}
|
||||
|
||||
# Create folds
|
||||
folds <- .generate_cv_folds(
|
||||
nfold = nfold
|
||||
, nrows = nrow(data)
|
||||
, stratified = stratified
|
||||
, label = get_field(dataset = data, field_name = "label")
|
||||
, group = get_field(dataset = data, field_name = "group")
|
||||
, params = params
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
# Add printing log callback
|
||||
if (params[["verbosity"]] > 0L && eval_freq > 0L) {
|
||||
callbacks <- .add_cb(cb_list = callbacks, cb = cb_print_evaluation(period = eval_freq))
|
||||
}
|
||||
|
||||
# Add evaluation log callback
|
||||
if (record) {
|
||||
callbacks <- .add_cb(cb_list = callbacks, cb = cb_record_evaluation())
|
||||
}
|
||||
|
||||
# Did user pass parameters that indicate they want to use early stopping?
|
||||
using_early_stopping <- !is.null(early_stopping_rounds) && early_stopping_rounds > 0L
|
||||
|
||||
boosting_param_names <- .PARAMETER_ALIASES()[["boosting"]]
|
||||
using_dart <- any(
|
||||
sapply(
|
||||
X = boosting_param_names
|
||||
, FUN = function(param) {
|
||||
identical(params[[param]], "dart")
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# Cannot use early stopping with 'dart' boosting
|
||||
if (using_dart) {
|
||||
if (using_early_stopping) {
|
||||
warning("Early stopping is not available in 'dart' mode.")
|
||||
}
|
||||
using_early_stopping <- FALSE
|
||||
|
||||
# Remove the cb_early_stop() function if it was passed in to callbacks
|
||||
callbacks <- Filter(
|
||||
f = function(cb_func) {
|
||||
!identical(attr(cb_func, "name"), "cb_early_stop")
|
||||
}
|
||||
, x = callbacks
|
||||
)
|
||||
}
|
||||
|
||||
# If user supplied early_stopping_rounds, add the early stopping callback
|
||||
if (using_early_stopping) {
|
||||
callbacks <- .add_cb(
|
||||
cb_list = callbacks
|
||||
, cb = cb_early_stop(
|
||||
stopping_rounds = early_stopping_rounds
|
||||
, first_metric_only = isTRUE(params[["first_metric_only"]])
|
||||
, verbose = params[["verbosity"]] > 0L
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
cb <- .categorize_callbacks(cb_list = callbacks)
|
||||
|
||||
# Construct booster for each fold. The data.table() code below is used to
|
||||
# guarantee that indices are sorted while keeping init_score and weight together
|
||||
# with the correct indices. Note that it takes advantage of the fact that
|
||||
# someDT$some_column returns NULL is 'some_column' does not exist in the data.table
|
||||
bst_folds <- lapply(
|
||||
X = seq_along(folds)
|
||||
, FUN = function(k) {
|
||||
|
||||
# For learning-to-rank, each fold is a named list with two elements:
|
||||
# * `fold` = an integer vector of row indices
|
||||
# * `group` = an integer vector describing which groups are in the fold
|
||||
# For classification or regression tasks, it will just be an integer
|
||||
# vector of row indices
|
||||
folds_have_group <- "group" %in% names(folds[[k]])
|
||||
if (folds_have_group) {
|
||||
test_indices <- folds[[k]]$fold
|
||||
test_group_indices <- folds[[k]]$group
|
||||
test_groups <- get_field(dataset = data, field_name = "group")[test_group_indices]
|
||||
train_groups <- get_field(dataset = data, field_name = "group")[-test_group_indices]
|
||||
} else {
|
||||
test_indices <- folds[[k]]
|
||||
}
|
||||
train_indices <- seq_len(nrow(data))[-test_indices]
|
||||
|
||||
# set up test set
|
||||
indexDT <- data.table::data.table(
|
||||
indices = test_indices
|
||||
, weight = get_field(dataset = data, field_name = "weight")[test_indices]
|
||||
, init_score = get_field(dataset = data, field_name = "init_score")[test_indices]
|
||||
)
|
||||
data.table::setorderv(x = indexDT, cols = "indices", order = 1L)
|
||||
dtest <- lgb.slice.Dataset(data, indexDT$indices)
|
||||
set_field(dataset = dtest, field_name = "weight", data = indexDT$weight)
|
||||
set_field(dataset = dtest, field_name = "init_score", data = indexDT$init_score)
|
||||
|
||||
# set up training set
|
||||
indexDT <- data.table::data.table(
|
||||
indices = train_indices
|
||||
, weight = get_field(dataset = data, field_name = "weight")[train_indices]
|
||||
, init_score = get_field(dataset = data, field_name = "init_score")[train_indices]
|
||||
)
|
||||
data.table::setorderv(x = indexDT, cols = "indices", order = 1L)
|
||||
dtrain <- lgb.slice.Dataset(data, indexDT$indices)
|
||||
set_field(dataset = dtrain, field_name = "weight", data = indexDT$weight)
|
||||
set_field(dataset = dtrain, field_name = "init_score", data = indexDT$init_score)
|
||||
|
||||
if (folds_have_group) {
|
||||
set_field(dataset = dtest, field_name = "group", data = test_groups)
|
||||
set_field(dataset = dtrain, field_name = "group", data = train_groups)
|
||||
}
|
||||
|
||||
booster <- Booster$new(params = params, train_set = dtrain)
|
||||
if (isTRUE(eval_train_metric)) {
|
||||
booster$add_valid(data = dtrain, name = "train")
|
||||
}
|
||||
booster$add_valid(data = dtest, name = "valid")
|
||||
return(
|
||||
list(booster = booster)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
# Create new booster
|
||||
cv_booster <- CVBooster$new(x = bst_folds)
|
||||
|
||||
# Callback env
|
||||
env <- CB_ENV$new()
|
||||
env$model <- cv_booster
|
||||
env$begin_iteration <- begin_iteration
|
||||
env$end_iteration <- end_iteration
|
||||
|
||||
# Start training model using number of iterations to start and end with
|
||||
for (i in seq.int(from = begin_iteration, to = end_iteration)) {
|
||||
|
||||
# Overwrite iteration in environment
|
||||
env$iteration <- i
|
||||
env$eval_list <- list()
|
||||
|
||||
for (f in cb$pre_iter) {
|
||||
f(env)
|
||||
}
|
||||
|
||||
# Update one boosting iteration
|
||||
msg <- lapply(cv_booster$boosters, function(fd) {
|
||||
fd$booster$update(fobj = fobj)
|
||||
out <- list()
|
||||
for (eval_function in eval_functions) {
|
||||
out <- append(out, fd$booster$eval_valid(feval = eval_function))
|
||||
}
|
||||
return(out)
|
||||
})
|
||||
|
||||
# Prepare collection of evaluation results
|
||||
merged_msg <- .merge_cv_result(
|
||||
msg = msg
|
||||
, showsd = showsd
|
||||
)
|
||||
|
||||
# Write evaluation result in environment
|
||||
env$eval_list <- merged_msg$eval_list
|
||||
|
||||
# Check for standard deviation requirement
|
||||
if (showsd) {
|
||||
env$eval_err_list <- merged_msg$eval_err_list
|
||||
}
|
||||
|
||||
# Loop through env
|
||||
for (f in cb$post_iter) {
|
||||
f(env)
|
||||
}
|
||||
|
||||
# Check for early stopping and break if needed
|
||||
if (env$met_early_stop) break
|
||||
|
||||
}
|
||||
|
||||
# When early stopping is not activated, we compute the best iteration / score ourselves
|
||||
# based on the first first metric
|
||||
if (record && is.na(env$best_score)) {
|
||||
# when using a custom eval function, the metric name is returned from the
|
||||
# function, so figure it out from record_evals
|
||||
if (!is.null(eval_functions[1L])) {
|
||||
first_metric <- names(cv_booster$record_evals[["valid"]])[1L]
|
||||
} else {
|
||||
first_metric <- cv_booster$.__enclos_env__$private$eval_names[1L]
|
||||
}
|
||||
.find_best <- which.min
|
||||
if (isTRUE(env$eval_list[[1L]]$higher_better[1L])) {
|
||||
.find_best <- which.max
|
||||
}
|
||||
cv_booster$best_iter <- unname(
|
||||
.find_best(
|
||||
unlist(
|
||||
cv_booster$record_evals[["valid"]][[first_metric]][[.EVAL_KEY()]]
|
||||
)
|
||||
)
|
||||
)
|
||||
cv_booster$best_score <- cv_booster$record_evals[["valid"]][[first_metric]][[.EVAL_KEY()]][[cv_booster$best_iter]]
|
||||
}
|
||||
# Propagate the best_iter attribute from the cv_booster to the individual boosters
|
||||
for (bst in cv_booster$boosters) {
|
||||
bst$booster$best_iter <- cv_booster$best_iter
|
||||
}
|
||||
|
||||
if (reset_data) {
|
||||
lapply(cv_booster$boosters, function(fd) {
|
||||
# Store temporarily model data elsewhere
|
||||
booster_old <- list(
|
||||
best_iter = fd$booster$best_iter
|
||||
, best_score = fd$booster$best_score
|
||||
, record_evals = fd$booster$record_evals
|
||||
)
|
||||
# Reload model
|
||||
fd$booster <- lgb.load(model_str = fd$booster$save_model_to_string())
|
||||
fd$booster$best_iter <- booster_old$best_iter
|
||||
fd$booster$best_score <- booster_old$best_score
|
||||
fd$booster$record_evals <- booster_old$record_evals
|
||||
})
|
||||
}
|
||||
|
||||
if (serializable) {
|
||||
lapply(cv_booster$boosters, function(model) model$booster$save_raw())
|
||||
}
|
||||
|
||||
return(cv_booster)
|
||||
|
||||
}
|
||||
|
||||
# Generates random (stratified if needed) CV folds
|
||||
.generate_cv_folds <- function(nfold, nrows, stratified, label, group, params) {
|
||||
|
||||
# Check for group existence
|
||||
if (is.null(group)) {
|
||||
|
||||
# Shuffle
|
||||
rnd_idx <- sample.int(nrows)
|
||||
|
||||
# Request stratified folds
|
||||
if (isTRUE(stratified) && params$objective %in% c("binary", "multiclass") && length(label) == length(rnd_idx)) {
|
||||
|
||||
y <- label[rnd_idx]
|
||||
y <- as.factor(y)
|
||||
folds <- .stratified_folds(y = y, k = nfold)
|
||||
|
||||
} else {
|
||||
|
||||
# Make simple non-stratified folds
|
||||
folds <- list()
|
||||
|
||||
# Loop through each fold
|
||||
for (i in seq_len(nfold)) {
|
||||
kstep <- length(rnd_idx) %/% (nfold - i + 1L)
|
||||
folds[[i]] <- rnd_idx[seq_len(kstep)]
|
||||
rnd_idx <- rnd_idx[-seq_len(kstep)]
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
# When doing group, stratified is not possible (only random selection)
|
||||
if (nfold > length(group)) {
|
||||
stop("\nYou requested too many folds for the number of available groups.\n")
|
||||
}
|
||||
|
||||
# Degroup the groups
|
||||
ungrouped <- inverse.rle(list(lengths = group, values = seq_along(group)))
|
||||
|
||||
# Can't stratify, shuffle
|
||||
rnd_idx <- sample.int(length(group))
|
||||
|
||||
# Make simple non-stratified folds
|
||||
folds <- list()
|
||||
|
||||
# Loop through each fold
|
||||
for (i in seq_len(nfold)) {
|
||||
kstep <- length(rnd_idx) %/% (nfold - i + 1L)
|
||||
folds[[i]] <- list(
|
||||
fold = which(ungrouped %in% rnd_idx[seq_len(kstep)])
|
||||
, group = rnd_idx[seq_len(kstep)]
|
||||
)
|
||||
rnd_idx <- rnd_idx[-seq_len(kstep)]
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return(folds)
|
||||
|
||||
}
|
||||
|
||||
# Creates CV folds stratified by the values of y.
|
||||
# It was borrowed from caret::createFolds and simplified
|
||||
# by always returning an unnamed list of fold indices.
|
||||
#' @importFrom stats quantile
|
||||
.stratified_folds <- function(y, k) {
|
||||
|
||||
# Group the numeric data based on their magnitudes
|
||||
# and sample within those groups.
|
||||
# When the number of samples is low, we may have
|
||||
# issues further slicing the numeric data into
|
||||
# groups. The number of groups will depend on the
|
||||
# ratio of the number of folds to the sample size.
|
||||
# At most, we will use quantiles. If the sample
|
||||
# is too small, we just do regular unstratified CV
|
||||
if (is.numeric(y)) {
|
||||
|
||||
cuts <- length(y) %/% k
|
||||
if (cuts < 2L) {
|
||||
cuts <- 2L
|
||||
}
|
||||
if (cuts > 5L) {
|
||||
cuts <- 5L
|
||||
}
|
||||
y <- cut(
|
||||
y
|
||||
, unique(stats::quantile(y, probs = seq.int(0.0, 1.0, length.out = cuts)))
|
||||
, include.lowest = TRUE
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
if (k < length(y)) {
|
||||
|
||||
# Reset levels so that the possible levels and
|
||||
# the levels in the vector are the same
|
||||
y <- as.factor(as.character(y))
|
||||
numInClass <- table(y)
|
||||
foldVector <- vector(mode = "integer", length(y))
|
||||
|
||||
# For each class, balance the fold allocation as far
|
||||
# as possible, then resample the remainder.
|
||||
# The final assignment of folds is also randomized.
|
||||
for (i in seq_along(numInClass)) {
|
||||
|
||||
# Create a vector of integers from 1:k as many times as possible without
|
||||
# going over the number of samples in the class. Note that if the number
|
||||
# of samples in a class is less than k, nothing is produced here.
|
||||
seqVector <- rep(seq_len(k), numInClass[i] %/% k)
|
||||
|
||||
# Add enough random integers to get length(seqVector) == numInClass[i]
|
||||
if (numInClass[i] %% k > 0L) {
|
||||
seqVector <- c(seqVector, sample.int(k, numInClass[i] %% k))
|
||||
}
|
||||
|
||||
# Shuffle the integers for fold assignment and assign to this classes's data
|
||||
foldVector[y == dimnames(numInClass)$y[i]] <- sample(seqVector)
|
||||
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
foldVector <- seq(along = y)
|
||||
|
||||
}
|
||||
|
||||
out <- split(seq(along = y), foldVector)
|
||||
names(out) <- NULL
|
||||
return(out)
|
||||
}
|
||||
|
||||
.merge_cv_result <- function(msg, showsd) {
|
||||
|
||||
if (length(msg) == 0L) {
|
||||
stop("lgb.cv: size of cv result error")
|
||||
}
|
||||
|
||||
eval_len <- length(msg[[1L]])
|
||||
|
||||
if (eval_len == 0L) {
|
||||
stop("lgb.cv: should provide at least one metric for CV")
|
||||
}
|
||||
|
||||
# Get evaluation results using a list apply
|
||||
eval_result <- lapply(seq_len(eval_len), function(j) {
|
||||
as.numeric(lapply(seq_along(msg), function(i) {
|
||||
msg[[i]][[j]]$value }))
|
||||
})
|
||||
|
||||
# Get evaluation. Just taking the first element here to
|
||||
# get structure (name, higher_better, data_name)
|
||||
ret_eval <- msg[[1L]]
|
||||
|
||||
for (j in seq_len(eval_len)) {
|
||||
ret_eval[[j]]$value <- mean(eval_result[[j]])
|
||||
}
|
||||
|
||||
ret_eval_err <- NULL
|
||||
|
||||
# Check for standard deviation
|
||||
if (showsd) {
|
||||
|
||||
# Parse standard deviation
|
||||
for (j in seq_len(eval_len)) {
|
||||
ret_eval_err <- c(
|
||||
ret_eval_err
|
||||
, sqrt(mean(eval_result[[j]] ^ 2L) - mean(eval_result[[j]]) ^ 2L)
|
||||
)
|
||||
}
|
||||
|
||||
ret_eval_err <- as.list(ret_eval_err)
|
||||
|
||||
}
|
||||
|
||||
return(
|
||||
list(
|
||||
eval_list = ret_eval
|
||||
, eval_err_list = ret_eval_err
|
||||
)
|
||||
)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
#' @name lgb.drop_serialized
|
||||
#' @title Drop serialized raw bytes in a LightGBM model object
|
||||
#' @description If a LightGBM model object was produced with argument `serializable=TRUE`, the R object will keep
|
||||
#' a copy of the underlying C++ object as raw bytes, which can be used to reconstruct such object after getting
|
||||
#' serialized and de-serialized, but at the cost of extra memory usage. If these raw bytes are not needed anymore,
|
||||
#' they can be dropped through this function in order to save memory. Note that the object will be modified in-place.
|
||||
#'
|
||||
#' \emph{New in version 4.0.0}
|
||||
#'
|
||||
#' @param model \code{lgb.Booster} object which was produced with `serializable=TRUE`.
|
||||
#'
|
||||
#' @return \code{lgb.Booster} (the same `model` object that was passed as input, as invisible).
|
||||
#' @seealso \link{lgb.restore_handle}, \link{lgb.make_serializable}.
|
||||
#' @export
|
||||
lgb.drop_serialized <- function(model) {
|
||||
if (!.is_Booster(x = model)) {
|
||||
stop("lgb.drop_serialized: model should be an ", sQuote("lgb.Booster", q = FALSE))
|
||||
}
|
||||
model$drop_raw()
|
||||
return(invisible(model))
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
#' @name lgb.importance
|
||||
#' @title Compute feature importance in a model
|
||||
#' @description Creates a \code{data.table} of feature importances in a model.
|
||||
#' @param model object of class \code{lgb.Booster}.
|
||||
#' @param percentage whether to show importance in relative percentage.
|
||||
#'
|
||||
#' @return For a tree model, a \code{data.table} with the following columns:
|
||||
#' \itemize{
|
||||
#' \item{\code{Feature}: Feature names in the model.}
|
||||
#' \item{\code{Gain}: The total gain of this feature's splits.}
|
||||
#' \item{\code{Cover}: The number of observation related to this feature.}
|
||||
#' \item{\code{Frequency}: The number of times a feature split in trees.}
|
||||
#' }
|
||||
#'
|
||||
#' @examples
|
||||
#' \donttest{
|
||||
#' \dontshow{setLGBMthreads(2L)}
|
||||
#' \dontshow{data.table::setDTthreads(1L)}
|
||||
#' data(agaricus.train, package = "lightgbm")
|
||||
#' train <- agaricus.train
|
||||
#' dtrain <- lgb.Dataset(train$data, label = train$label)
|
||||
#'
|
||||
#' params <- list(
|
||||
#' objective = "binary"
|
||||
#' , learning_rate = 0.1
|
||||
#' , max_depth = -1L
|
||||
#' , min_data_in_leaf = 1L
|
||||
#' , min_sum_hessian_in_leaf = 1.0
|
||||
#' , num_threads = 2L
|
||||
#' )
|
||||
#' model <- lgb.train(
|
||||
#' params = params
|
||||
#' , data = dtrain
|
||||
#' , nrounds = 5L
|
||||
#' )
|
||||
#'
|
||||
#' tree_imp1 <- lgb.importance(model, percentage = TRUE)
|
||||
#' tree_imp2 <- lgb.importance(model, percentage = FALSE)
|
||||
#' }
|
||||
#' @importFrom data.table := setnames setorderv
|
||||
#' @export
|
||||
lgb.importance <- function(model, percentage = TRUE) {
|
||||
|
||||
if (!.is_Booster(x = model)) {
|
||||
stop("'model' has to be an object of class lgb.Booster")
|
||||
}
|
||||
|
||||
# Setup importance
|
||||
tree_dt <- lgb.model.dt.tree(model = model)
|
||||
|
||||
# Extract elements
|
||||
tree_imp_dt <- tree_dt[
|
||||
!is.na(split_index)
|
||||
, .(Gain = sum(split_gain), Cover = sum(internal_count), Frequency = .N)
|
||||
, by = "split_feature"
|
||||
]
|
||||
|
||||
data.table::setnames(
|
||||
x = tree_imp_dt
|
||||
, old = "split_feature"
|
||||
, new = "Feature"
|
||||
)
|
||||
|
||||
# Sort features by Gain
|
||||
data.table::setorderv(
|
||||
x = tree_imp_dt
|
||||
, cols = "Gain"
|
||||
, order = -1L
|
||||
)
|
||||
|
||||
# Check if relative values are requested
|
||||
if (percentage) {
|
||||
tree_imp_dt[, `:=`(
|
||||
Gain = Gain / sum(Gain)
|
||||
, Cover = Cover / sum(Cover)
|
||||
, Frequency = Frequency / sum(Frequency)
|
||||
)]
|
||||
}
|
||||
|
||||
# adding an empty [] to ensure the table is printed the first time print.data.table() is called
|
||||
return(tree_imp_dt[])
|
||||
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
#' @name lgb.interpret
|
||||
#' @title Compute feature contribution of prediction
|
||||
#' @description Computes feature contribution components of rawscore prediction.
|
||||
#' @param model object of class \code{lgb.Booster}.
|
||||
#' @param data a matrix object or a dgCMatrix object.
|
||||
#' @param idxset an integer vector of indices of rows needed.
|
||||
#' @param num_iteration number of iteration want to predict with, NULL or <= 0 means use best iteration.
|
||||
#'
|
||||
#' @return For regression, binary classification and lambdarank model, a \code{list} of \code{data.table}
|
||||
#' with the following columns:
|
||||
#' \itemize{
|
||||
#' \item{\code{Feature}: Feature names in the model.}
|
||||
#' \item{\code{Contribution}: The total contribution of this feature's splits.}
|
||||
#' }
|
||||
#' For multiclass classification, a \code{list} of \code{data.table} with the Feature column and
|
||||
#' Contribution columns to each class.
|
||||
#'
|
||||
#' @examples
|
||||
#' \donttest{
|
||||
#' \dontshow{setLGBMthreads(2L)}
|
||||
#' \dontshow{data.table::setDTthreads(1L)}
|
||||
#' Logit <- function(x) log(x / (1.0 - x))
|
||||
#' data(agaricus.train, package = "lightgbm")
|
||||
#' train <- agaricus.train
|
||||
#' dtrain <- lgb.Dataset(train$data, label = train$label)
|
||||
#' set_field(
|
||||
#' dataset = dtrain
|
||||
#' , field_name = "init_score"
|
||||
#' , data = rep(Logit(mean(train$label)), length(train$label))
|
||||
#' )
|
||||
#' data(agaricus.test, package = "lightgbm")
|
||||
#' test <- agaricus.test
|
||||
#'
|
||||
#' params <- list(
|
||||
#' objective = "binary"
|
||||
#' , learning_rate = 0.1
|
||||
#' , max_depth = -1L
|
||||
#' , min_data_in_leaf = 1L
|
||||
#' , min_sum_hessian_in_leaf = 1.0
|
||||
#' , num_threads = 2L
|
||||
#' )
|
||||
#' model <- lgb.train(
|
||||
#' params = params
|
||||
#' , data = dtrain
|
||||
#' , nrounds = 3L
|
||||
#' )
|
||||
#'
|
||||
#' tree_interpretation <- lgb.interpret(model, test$data, 1L:5L)
|
||||
#' }
|
||||
#' @importFrom data.table as.data.table
|
||||
#' @export
|
||||
lgb.interpret <- function(model,
|
||||
data,
|
||||
idxset,
|
||||
num_iteration = NULL) {
|
||||
|
||||
# Get tree model
|
||||
tree_dt <- lgb.model.dt.tree(model = model, num_iteration = num_iteration)
|
||||
|
||||
# Check number of classes
|
||||
num_class <- model$.__enclos_env__$private$num_class
|
||||
|
||||
# Get vector list
|
||||
tree_interpretation_dt_list <- vector(mode = "list", length = length(idxset))
|
||||
|
||||
# Get parsed predictions of data
|
||||
pred_mat <- t(
|
||||
model$predict(
|
||||
data = data[idxset, , drop = FALSE]
|
||||
, num_iteration = num_iteration
|
||||
, predleaf = TRUE
|
||||
)
|
||||
)
|
||||
leaf_index_dt <- data.table::as.data.table(x = pred_mat)
|
||||
leaf_index_mat_list <- lapply(
|
||||
X = leaf_index_dt
|
||||
, FUN = matrix
|
||||
, ncol = num_class
|
||||
, byrow = TRUE
|
||||
)
|
||||
|
||||
# Get list of trees
|
||||
tree_index_mat_list <- lapply(
|
||||
X = leaf_index_mat_list
|
||||
, FUN = function(x) {
|
||||
matrix(seq_along(x) - 1L, ncol = num_class, byrow = TRUE)
|
||||
}
|
||||
)
|
||||
|
||||
for (i in seq_along(idxset)) {
|
||||
tree_interpretation_dt_list[[i]] <- .single_row_interpret(
|
||||
tree_dt = tree_dt
|
||||
, num_class = num_class
|
||||
, tree_index_mat = tree_index_mat_list[[i]]
|
||||
, leaf_index_mat = leaf_index_mat_list[[i]]
|
||||
)
|
||||
}
|
||||
|
||||
return(tree_interpretation_dt_list)
|
||||
|
||||
}
|
||||
|
||||
#' @name lgb.interprete
|
||||
#' @title DEPRECATED - use lgb.interpret() instead
|
||||
#' @description Alias for \code{lgb.interpret}.
|
||||
#' @param ... Arguments passed through to \code{lgb.interpret}
|
||||
#' @export
|
||||
lgb.interprete <- function(...) {
|
||||
warning("lgb.interprete() is deprecated and will be removed in a future release. Use lgb.interpret() instead.")
|
||||
return(lgb.interpret(...))
|
||||
}
|
||||
|
||||
#' @importFrom data.table data.table
|
||||
single.tree.interpret <- function(tree_dt,
|
||||
tree_id,
|
||||
leaf_id) {
|
||||
|
||||
# Match tree id
|
||||
single_tree_dt <- tree_dt[tree_index == tree_id, ]
|
||||
|
||||
# Get leaves
|
||||
leaf_dt <- single_tree_dt[leaf_index == leaf_id, .(leaf_index, leaf_parent, leaf_value)]
|
||||
|
||||
# Get nodes
|
||||
node_dt <- single_tree_dt[!is.na(split_index), .(split_index, split_feature, node_parent, internal_value)]
|
||||
|
||||
# Prepare sequences
|
||||
feature_seq <- character(0L)
|
||||
value_seq <- numeric(0L)
|
||||
|
||||
# Get to root from leaf
|
||||
leaf_to_root <- function(parent_id, current_value) {
|
||||
|
||||
value_seq <<- c(current_value, value_seq)
|
||||
|
||||
if (!is.na(parent_id)) {
|
||||
|
||||
# Not null means existing node
|
||||
this_node <- node_dt[split_index == parent_id, ]
|
||||
feature_seq <<- c(this_node[["split_feature"]], feature_seq)
|
||||
leaf_to_root(
|
||||
parent_id = this_node[["node_parent"]]
|
||||
, current_value = this_node[["internal_value"]]
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
# Perform leaf to root conversion
|
||||
leaf_to_root(
|
||||
parent_id = leaf_dt[["leaf_parent"]]
|
||||
, current_value = leaf_dt[["leaf_value"]]
|
||||
)
|
||||
|
||||
return(
|
||||
data.table::data.table(
|
||||
Feature = feature_seq
|
||||
, Contribution = diff.default(value_seq)
|
||||
)
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
#' @importFrom data.table := rbindlist setorder
|
||||
.multiple_tree_interpret <- function(tree_dt,
|
||||
tree_index,
|
||||
leaf_index) {
|
||||
|
||||
interp_dt <- data.table::rbindlist(
|
||||
l = mapply(
|
||||
FUN = single.tree.interpret
|
||||
, tree_id = tree_index
|
||||
, leaf_id = leaf_index
|
||||
, MoreArgs = list(
|
||||
tree_dt = tree_dt
|
||||
)
|
||||
, SIMPLIFY = FALSE
|
||||
, USE.NAMES = TRUE
|
||||
)
|
||||
, use.names = TRUE
|
||||
)
|
||||
|
||||
interp_dt <- interp_dt[, .(Contribution = sum(Contribution)), by = "Feature"]
|
||||
|
||||
# Sort features in descending order by contribution
|
||||
interp_dt[, abs_contribution := abs(Contribution)]
|
||||
data.table::setorder(
|
||||
x = interp_dt
|
||||
, -abs_contribution
|
||||
)
|
||||
|
||||
# Drop absolute value of contribution (only needed for sorting)
|
||||
interp_dt[, abs_contribution := NULL]
|
||||
|
||||
return(interp_dt)
|
||||
|
||||
}
|
||||
|
||||
#' @importFrom data.table set setnames
|
||||
.single_row_interpret <- function(tree_dt, num_class, tree_index_mat, leaf_index_mat) {
|
||||
|
||||
# Prepare vector list
|
||||
tree_interpretation <- vector(mode = "list", length = num_class)
|
||||
|
||||
# Loop throughout each class
|
||||
for (i in seq_len(num_class)) {
|
||||
|
||||
next_interp_dt <- .multiple_tree_interpret(
|
||||
tree_dt = tree_dt
|
||||
, tree_index = tree_index_mat[, i]
|
||||
, leaf_index = leaf_index_mat[, i]
|
||||
)
|
||||
|
||||
if (num_class > 1L) {
|
||||
data.table::setnames(
|
||||
x = next_interp_dt
|
||||
, old = "Contribution"
|
||||
, new = paste("Class", i - 1L)
|
||||
)
|
||||
}
|
||||
|
||||
tree_interpretation[[i]] <- next_interp_dt
|
||||
|
||||
}
|
||||
|
||||
if (num_class == 1L) {
|
||||
|
||||
tree_interpretation_dt <- tree_interpretation[[1L]]
|
||||
|
||||
} else {
|
||||
|
||||
# Full interpretation elements
|
||||
tree_interpretation_dt <- Reduce(
|
||||
f = function(x, y) {
|
||||
merge(x, y, by = "Feature", all = TRUE)
|
||||
}
|
||||
, x = tree_interpretation
|
||||
)
|
||||
|
||||
# Loop throughout each tree
|
||||
for (j in 2L:ncol(tree_interpretation_dt)) {
|
||||
|
||||
data.table::set(
|
||||
x = tree_interpretation_dt
|
||||
, i = which(is.na(tree_interpretation_dt[[j]]))
|
||||
, j = j
|
||||
, value = 0.0
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return(tree_interpretation_dt)
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
#' @name lgb.make_serializable
|
||||
#' @title Make a LightGBM object serializable by keeping raw bytes
|
||||
#' @description If a LightGBM model object was produced with argument `serializable=FALSE`, the R object will not
|
||||
#' be serializable (e.g. cannot save and load with \code{saveRDS} and \code{readRDS}) as it will lack the raw bytes
|
||||
#' needed to reconstruct its underlying C++ object. This function can be used to forcibly produce those serialized
|
||||
#' raw bytes and make the object serializable. Note that the object will be modified in-place.
|
||||
#'
|
||||
#' \emph{New in version 4.0.0}
|
||||
#'
|
||||
#' @param model \code{lgb.Booster} object which was produced with `serializable=FALSE`.
|
||||
#'
|
||||
#' @return \code{lgb.Booster} (the same `model` object that was passed as input, as invisible).
|
||||
#' @seealso \link{lgb.restore_handle}, \link{lgb.drop_serialized}.
|
||||
#' @export
|
||||
lgb.make_serializable <- function(model) {
|
||||
if (!.is_Booster(x = model)) {
|
||||
stop("lgb.make_serializable: model should be an ", sQuote("lgb.Booster", q = FALSE))
|
||||
}
|
||||
model$save_raw()
|
||||
return(invisible(model))
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
#' @name lgb.model.dt.tree
|
||||
#' @title Parse a LightGBM model json dump
|
||||
#' @description Parse a LightGBM model json dump into a \code{data.table} structure.
|
||||
#' @param model object of class \code{lgb.Booster}.
|
||||
#' @param num_iteration Number of iterations to include. NULL or <= 0 means use best iteration.
|
||||
#' @param start_iteration Index (1-based) of the first boosting round to include in the output.
|
||||
#' For example, passing \code{start_iteration=5, num_iteration=3} for a regression model
|
||||
#' means "return information about the fifth, sixth, and seventh trees".
|
||||
#'
|
||||
#' \emph{New in version 4.4.0}
|
||||
#'
|
||||
#' @return
|
||||
#' A \code{data.table} with detailed information about model trees' nodes and leaves.
|
||||
#'
|
||||
#' The columns of the \code{data.table} are:
|
||||
#'
|
||||
#' \itemize{
|
||||
#' \item{\code{tree_index}: ID of a tree in a model (integer)}
|
||||
#' \item{\code{split_index}: ID of a node in a tree (integer)}
|
||||
#' \item{\code{split_feature}: for a node, it's a feature name (character);
|
||||
#' for a leaf, it simply labels it as \code{"NA"}}
|
||||
#' \item{\code{node_parent}: ID of the parent node for current node (integer)}
|
||||
#' \item{\code{leaf_index}: ID of a leaf in a tree (integer)}
|
||||
#' \item{\code{leaf_parent}: ID of the parent node for current leaf (integer)}
|
||||
#' \item{\code{split_gain}: Split gain of a node}
|
||||
#' \item{\code{threshold}: Splitting threshold value of a node}
|
||||
#' \item{\code{decision_type}: Decision type of a node}
|
||||
#' \item{\code{default_left}: Determine how to handle NA value, TRUE -> Left, FALSE -> Right}
|
||||
#' \item{\code{internal_value}: Node value}
|
||||
#' \item{\code{internal_count}: The number of observation collected by a node}
|
||||
#' \item{\code{leaf_value}: Leaf value}
|
||||
#' \item{\code{leaf_count}: The number of observation collected by a leaf}
|
||||
#' }
|
||||
#'
|
||||
#' @examples
|
||||
#' \donttest{
|
||||
#' \dontshow{setLGBMthreads(2L)}
|
||||
#' \dontshow{data.table::setDTthreads(1L)}
|
||||
#' data(agaricus.train, package = "lightgbm")
|
||||
#' train <- agaricus.train
|
||||
#' dtrain <- lgb.Dataset(train$data, label = train$label)
|
||||
#'
|
||||
#' params <- list(
|
||||
#' objective = "binary"
|
||||
#' , learning_rate = 0.01
|
||||
#' , num_leaves = 63L
|
||||
#' , max_depth = -1L
|
||||
#' , min_data_in_leaf = 1L
|
||||
#' , min_sum_hessian_in_leaf = 1.0
|
||||
#' , num_threads = 2L
|
||||
#' )
|
||||
#' model <- lgb.train(params, dtrain, 10L)
|
||||
#'
|
||||
#' tree_dt <- lgb.model.dt.tree(model)
|
||||
#' }
|
||||
#' @importFrom data.table := rbindlist
|
||||
#' @importFrom jsonlite fromJSON
|
||||
#' @export
|
||||
lgb.model.dt.tree <- function(
|
||||
model, num_iteration = NULL, start_iteration = 1L
|
||||
) {
|
||||
|
||||
json_model <- lgb.dump(
|
||||
booster = model
|
||||
, num_iteration = num_iteration
|
||||
, start_iteration = start_iteration
|
||||
)
|
||||
|
||||
parsed_json_model <- jsonlite::fromJSON(
|
||||
txt = json_model
|
||||
, simplifyVector = TRUE
|
||||
, simplifyDataFrame = FALSE
|
||||
, simplifyMatrix = FALSE
|
||||
, flatten = FALSE
|
||||
)
|
||||
|
||||
# Parse tree model
|
||||
tree_list <- lapply(
|
||||
X = parsed_json_model$tree_info
|
||||
, FUN = .single_tree_parse
|
||||
)
|
||||
|
||||
# Combine into single data.table
|
||||
tree_dt <- data.table::rbindlist(l = tree_list, use.names = TRUE)
|
||||
|
||||
# Substitute feature index with the actual feature name
|
||||
|
||||
# Since the index comes from C++ (which is 0-indexed), be sure
|
||||
# to add 1 (e.g. index 28 means the 29th feature in feature_names)
|
||||
split_feature_indx <- tree_dt[, split_feature] + 1L
|
||||
|
||||
# Get corresponding feature names. Positions in split_feature_indx
|
||||
# which are NA will result in an NA feature name
|
||||
feature_names <- parsed_json_model$feature_names[split_feature_indx]
|
||||
tree_dt[, split_feature := feature_names]
|
||||
|
||||
return(tree_dt)
|
||||
}
|
||||
|
||||
|
||||
#' @importFrom data.table := data.table rbindlist
|
||||
.single_tree_parse <- function(lgb_tree) {
|
||||
tree_info_cols <- c(
|
||||
"split_index"
|
||||
, "split_feature"
|
||||
, "split_gain"
|
||||
, "threshold"
|
||||
, "decision_type"
|
||||
, "default_left"
|
||||
, "internal_value"
|
||||
, "internal_count"
|
||||
)
|
||||
|
||||
# Traverse tree function
|
||||
pre_order_traversal <- function(env = NULL, tree_node_leaf, current_depth = 0L, parent_index = NA_integer_) {
|
||||
|
||||
if (is.null(env)) {
|
||||
# Setup initial default data.table with default types
|
||||
env <- new.env(parent = emptyenv())
|
||||
env$single_tree_dt <- list()
|
||||
env$single_tree_dt[[1L]] <- data.table::data.table(
|
||||
tree_index = integer(0L)
|
||||
, depth = integer(0L)
|
||||
, split_index = integer(0L)
|
||||
, split_feature = integer(0L)
|
||||
, node_parent = integer(0L)
|
||||
, leaf_index = integer(0L)
|
||||
, leaf_parent = integer(0L)
|
||||
, split_gain = numeric(0L)
|
||||
, threshold = numeric(0L)
|
||||
, decision_type = character(0L)
|
||||
, default_left = character(0L)
|
||||
, internal_value = integer(0L)
|
||||
, internal_count = integer(0L)
|
||||
, leaf_value = integer(0L)
|
||||
, leaf_count = integer(0L)
|
||||
)
|
||||
# start tree traversal
|
||||
pre_order_traversal(
|
||||
env = env
|
||||
, tree_node_leaf = tree_node_leaf
|
||||
, current_depth = current_depth
|
||||
, parent_index = parent_index
|
||||
)
|
||||
} else {
|
||||
|
||||
# Check if split index is not null in leaf
|
||||
if (!is.null(tree_node_leaf$split_index)) {
|
||||
|
||||
# update data.table
|
||||
env$single_tree_dt[[length(env$single_tree_dt) + 1L]] <- c(
|
||||
tree_node_leaf[tree_info_cols]
|
||||
, list("depth" = current_depth, "node_parent" = parent_index)
|
||||
)
|
||||
|
||||
# Traverse tree again both left and right
|
||||
pre_order_traversal(
|
||||
env = env
|
||||
, tree_node_leaf = tree_node_leaf$left_child
|
||||
, current_depth = current_depth + 1L
|
||||
, parent_index = tree_node_leaf$split_index
|
||||
)
|
||||
pre_order_traversal(
|
||||
env = env
|
||||
, tree_node_leaf = tree_node_leaf$right_child
|
||||
, current_depth = current_depth + 1L
|
||||
, parent_index = tree_node_leaf$split_index
|
||||
)
|
||||
} else if (!is.null(tree_node_leaf$leaf_index)) {
|
||||
|
||||
# update list
|
||||
env$single_tree_dt[[length(env$single_tree_dt) + 1L]] <- c(
|
||||
tree_node_leaf[c("leaf_index", "leaf_value", "leaf_count")]
|
||||
, list("depth" = current_depth, "leaf_parent" = parent_index)
|
||||
)
|
||||
}
|
||||
}
|
||||
return(env$single_tree_dt)
|
||||
}
|
||||
|
||||
# Traverse structure and rowbind everything
|
||||
single_tree_dt <- data.table::rbindlist(
|
||||
pre_order_traversal(tree_node_leaf = lgb_tree$tree_structure)
|
||||
, use.names = TRUE
|
||||
, fill = TRUE
|
||||
)
|
||||
|
||||
# Store index
|
||||
single_tree_dt[, tree_index := lgb_tree$tree_index]
|
||||
|
||||
return(single_tree_dt)
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
#' @name lgb.plot.importance
|
||||
#' @title Plot feature importance as a bar graph
|
||||
#' @description Plot previously calculated feature importance: Gain, Cover and Frequency, as a bar graph.
|
||||
#' @param tree_imp a \code{data.table} returned by \code{\link{lgb.importance}}.
|
||||
#' @param top_n maximal number of top features to include into the plot.
|
||||
#' @param measure the name of importance measure to plot, can be "Gain", "Cover" or "Frequency".
|
||||
#' @param left_margin (base R barplot) allows to adjust the left margin size to fit feature names.
|
||||
#' @param cex (base R barplot) passed as \code{cex.names} parameter to \code{\link[graphics]{barplot}}.
|
||||
#' Set a number smaller than 1.0 to make the bar labels smaller than R's default and values
|
||||
#' greater than 1.0 to make them larger.
|
||||
#'
|
||||
#' @details
|
||||
#' The graph represents each feature as a horizontal bar of length proportional to the defined importance of a feature.
|
||||
#' Features are shown ranked in a decreasing importance order.
|
||||
#'
|
||||
#' @return
|
||||
#' The \code{lgb.plot.importance} function creates a \code{barplot}
|
||||
#' and silently returns a processed data.table with \code{top_n} features sorted by defined importance.
|
||||
#'
|
||||
#' @examples
|
||||
#' \donttest{
|
||||
#' \dontshow{setLGBMthreads(2L)}
|
||||
#' \dontshow{data.table::setDTthreads(1L)}
|
||||
#' data(agaricus.train, package = "lightgbm")
|
||||
#' train <- agaricus.train
|
||||
#' dtrain <- lgb.Dataset(train$data, label = train$label)
|
||||
#'
|
||||
#' params <- list(
|
||||
#' objective = "binary"
|
||||
#' , learning_rate = 0.1
|
||||
#' , min_data_in_leaf = 1L
|
||||
#' , min_sum_hessian_in_leaf = 1.0
|
||||
#' , num_threads = 2L
|
||||
#' )
|
||||
#'
|
||||
#' model <- lgb.train(
|
||||
#' params = params
|
||||
#' , data = dtrain
|
||||
#' , nrounds = 5L
|
||||
#' )
|
||||
#'
|
||||
#' tree_imp <- lgb.importance(model, percentage = TRUE)
|
||||
#' lgb.plot.importance(tree_imp, top_n = 5L, measure = "Gain")
|
||||
#' }
|
||||
#' @importFrom graphics barplot par
|
||||
#' @export
|
||||
lgb.plot.importance <- function(tree_imp,
|
||||
top_n = 10L,
|
||||
measure = "Gain",
|
||||
left_margin = 10L,
|
||||
cex = NULL
|
||||
) {
|
||||
|
||||
# Check for measurement (column names) correctness
|
||||
measure <- match.arg(
|
||||
measure
|
||||
, choices = c("Gain", "Cover", "Frequency")
|
||||
, several.ok = FALSE
|
||||
)
|
||||
|
||||
# Get top N importance (defaults to 10)
|
||||
top_n <- min(top_n, nrow(tree_imp))
|
||||
|
||||
# Parse importance
|
||||
tree_imp <- tree_imp[order(abs(get(measure)), decreasing = TRUE), ][seq_len(top_n), ]
|
||||
|
||||
# Attempt to setup a correct cex
|
||||
if (is.null(cex)) {
|
||||
cex <- 2.5 / log2(1.0 + top_n)
|
||||
}
|
||||
|
||||
# Refresh plot
|
||||
op <- graphics::par(no.readonly = TRUE)
|
||||
on.exit(graphics::par(op))
|
||||
|
||||
graphics::par(
|
||||
mar = c(
|
||||
op$mar[1L]
|
||||
, left_margin
|
||||
, op$mar[3L]
|
||||
, op$mar[4L]
|
||||
)
|
||||
)
|
||||
|
||||
tree_imp[rev(seq_len(.N)),
|
||||
graphics::barplot(
|
||||
height = get(measure)
|
||||
, names.arg = Feature
|
||||
, horiz = TRUE
|
||||
, border = NA
|
||||
, main = "Feature Importance"
|
||||
, xlab = measure
|
||||
, cex.names = cex
|
||||
, las = 1L
|
||||
)]
|
||||
|
||||
return(invisible(tree_imp))
|
||||
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
#' @name lgb.plot.interpretation
|
||||
#' @title Plot feature contribution as a bar graph
|
||||
#' @description Plot previously calculated feature contribution as a bar graph.
|
||||
#' @param tree_interpretation_dt a \code{data.table} returned by \code{\link{lgb.interpret}}.
|
||||
#' @param top_n maximal number of top features to include into the plot.
|
||||
#' @param cols the column numbers of layout, will be used only for multiclass classification feature contribution.
|
||||
#' @param left_margin (base R barplot) allows to adjust the left margin size to fit feature names.
|
||||
#' @param cex (base R barplot) passed as \code{cex.names} parameter to \code{barplot}.
|
||||
#'
|
||||
#' @details
|
||||
#' The graph represents each feature as a horizontal bar of length proportional to the defined
|
||||
#' contribution of a feature. Features are shown ranked in a decreasing contribution order.
|
||||
#'
|
||||
#' @return
|
||||
#' The \code{lgb.plot.interpretation} function creates a \code{barplot}.
|
||||
#'
|
||||
#' @examples
|
||||
#' \donttest{
|
||||
#' \dontshow{setLGBMthreads(2L)}
|
||||
#' \dontshow{data.table::setDTthreads(1L)}
|
||||
#' Logit <- function(x) {
|
||||
#' log(x / (1.0 - x))
|
||||
#' }
|
||||
#' data(agaricus.train, package = "lightgbm")
|
||||
#' labels <- agaricus.train$label
|
||||
#' dtrain <- lgb.Dataset(
|
||||
#' agaricus.train$data
|
||||
#' , label = labels
|
||||
#' )
|
||||
#' set_field(
|
||||
#' dataset = dtrain
|
||||
#' , field_name = "init_score"
|
||||
#' , data = rep(Logit(mean(labels)), length(labels))
|
||||
#' )
|
||||
#'
|
||||
#' data(agaricus.test, package = "lightgbm")
|
||||
#'
|
||||
#' params <- list(
|
||||
#' objective = "binary"
|
||||
#' , learning_rate = 0.1
|
||||
#' , max_depth = -1L
|
||||
#' , min_data_in_leaf = 1L
|
||||
#' , min_sum_hessian_in_leaf = 1.0
|
||||
#' , num_threads = 2L
|
||||
#' )
|
||||
#' model <- lgb.train(
|
||||
#' params = params
|
||||
#' , data = dtrain
|
||||
#' , nrounds = 5L
|
||||
#' )
|
||||
#'
|
||||
#' tree_interpretation <- lgb.interpret(
|
||||
#' model = model
|
||||
#' , data = agaricus.test$data
|
||||
#' , idxset = 1L:5L
|
||||
#' )
|
||||
#' lgb.plot.interpretation(
|
||||
#' tree_interpretation_dt = tree_interpretation[[1L]]
|
||||
#' , top_n = 3L
|
||||
#' )
|
||||
#' }
|
||||
#' @importFrom data.table setnames
|
||||
#' @importFrom graphics barplot par
|
||||
#' @export
|
||||
lgb.plot.interpretation <- function(tree_interpretation_dt,
|
||||
top_n = 10L,
|
||||
cols = 1L,
|
||||
left_margin = 10L,
|
||||
cex = NULL) {
|
||||
|
||||
num_class <- ncol(tree_interpretation_dt) - 1L
|
||||
|
||||
# Refresh plot
|
||||
op <- graphics::par(no.readonly = TRUE)
|
||||
on.exit(graphics::par(op))
|
||||
|
||||
# Do some magic plotting
|
||||
bottom_margin <- 3.0
|
||||
top_margin <- 2.0
|
||||
right_margin <- op$mar[4L]
|
||||
|
||||
graphics::par(
|
||||
mar = c(
|
||||
bottom_margin
|
||||
, left_margin
|
||||
, top_margin
|
||||
, right_margin
|
||||
)
|
||||
)
|
||||
|
||||
if (num_class == 1L) {
|
||||
|
||||
# Only one class, plot straight away
|
||||
.multiple_tree_plot_interpretation(
|
||||
tree_interpretation = tree_interpretation_dt
|
||||
, top_n = top_n
|
||||
, title = NULL
|
||||
, cex = cex
|
||||
)
|
||||
|
||||
} else {
|
||||
|
||||
# More than one class, shape data first
|
||||
layout_mat <- matrix(
|
||||
seq.int(to = cols * ceiling(num_class / cols))
|
||||
, ncol = cols
|
||||
, nrow = ceiling(num_class / cols)
|
||||
)
|
||||
|
||||
# Shape output
|
||||
graphics::par(mfcol = c(nrow(layout_mat), ncol(layout_mat)))
|
||||
|
||||
# Loop throughout all classes
|
||||
for (i in seq_len(num_class)) {
|
||||
|
||||
# Prepare interpretation, perform T, get the names, and plot straight away
|
||||
plot_dt <- tree_interpretation_dt[, c(1L, i + 1L), with = FALSE]
|
||||
data.table::setnames(
|
||||
x = plot_dt
|
||||
, old = names(plot_dt)
|
||||
, new = c("Feature", "Contribution")
|
||||
)
|
||||
.multiple_tree_plot_interpretation(
|
||||
tree_interpretation = plot_dt
|
||||
, top_n = top_n
|
||||
, title = paste("Class", i - 1L)
|
||||
, cex = cex
|
||||
)
|
||||
|
||||
}
|
||||
}
|
||||
return(invisible(NULL))
|
||||
}
|
||||
|
||||
#' @importFrom graphics barplot
|
||||
.multiple_tree_plot_interpretation <- function(tree_interpretation,
|
||||
top_n,
|
||||
title,
|
||||
cex) {
|
||||
|
||||
# Parse tree
|
||||
tree_interpretation <- tree_interpretation[order(abs(Contribution), decreasing = TRUE), ][seq_len(min(top_n, .N)), ]
|
||||
|
||||
# Attempt to setup a correct cex
|
||||
if (is.null(cex)) {
|
||||
cex <- 2.5 / log2(1.0 + top_n)
|
||||
}
|
||||
|
||||
# create plot
|
||||
tree_interpretation[abs(Contribution) > 0.0, bar_color := "firebrick"]
|
||||
tree_interpretation[Contribution == 0.0, bar_color := "steelblue"]
|
||||
tree_interpretation[rev(seq_len(.N)),
|
||||
graphics::barplot(
|
||||
height = Contribution
|
||||
, names.arg = Feature
|
||||
, horiz = TRUE
|
||||
, col = bar_color
|
||||
, border = NA
|
||||
, main = title
|
||||
, cex.names = cex
|
||||
, las = 1L
|
||||
)]
|
||||
|
||||
return(invisible(NULL))
|
||||
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
#' @name lgb.restore_handle
|
||||
#' @title Restore the C++ component of a de-serialized LightGBM model
|
||||
#' @description After a LightGBM model object is de-serialized through functions such as \code{save} or
|
||||
#' \code{saveRDS}, its underlying C++ object will be blank and needs to be restored to able to use it. Such
|
||||
#' object is restored automatically when calling functions such as \code{predict}, but this function can be
|
||||
#' used to forcibly restore it beforehand. Note that the object will be modified in-place.
|
||||
#'
|
||||
#' \emph{New in version 4.0.0}
|
||||
#'
|
||||
#' @details Be aware that fast single-row prediction configurations are not restored through this
|
||||
#' function. If you wish to make fast single-row predictions using a \code{lgb.Booster} loaded this way,
|
||||
#' call \link{lgb.configure_fast_predict} on the loaded \code{lgb.Booster} object.
|
||||
#' @param model \code{lgb.Booster} object which was de-serialized and whose underlying C++ object and R handle
|
||||
#' need to be restored.
|
||||
#'
|
||||
#' @return \code{lgb.Booster} (the same `model` object that was passed as input, invisibly).
|
||||
#' @seealso \link{lgb.make_serializable}, \link{lgb.drop_serialized}.
|
||||
#' @examples
|
||||
#' \donttest{
|
||||
#' library(lightgbm)
|
||||
#' \dontshow{setLGBMthreads(2L)}
|
||||
#' \dontshow{data.table::setDTthreads(1L)}
|
||||
#' data("agaricus.train")
|
||||
#' model <- lightgbm(
|
||||
#' agaricus.train$data
|
||||
#' , agaricus.train$label
|
||||
#' , params = list(objective = "binary")
|
||||
#' , nrounds = 5L
|
||||
#' , verbose = 0
|
||||
#' , num_threads = 2L
|
||||
#' )
|
||||
#' fname <- tempfile(fileext="rds")
|
||||
#' saveRDS(model, fname)
|
||||
#'
|
||||
#' model_new <- readRDS(fname)
|
||||
#' model_new$check_null_handle()
|
||||
#' lgb.restore_handle(model_new)
|
||||
#' model_new$check_null_handle()
|
||||
#' }
|
||||
#' @export
|
||||
lgb.restore_handle <- function(model) {
|
||||
if (!.is_Booster(x = model)) {
|
||||
stop("lgb.restore_handle: model should be an ", sQuote("lgb.Booster", q = FALSE))
|
||||
}
|
||||
model$restore_handle()
|
||||
return(invisible(model))
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
#' @name lgb.train
|
||||
#' @title Main training logic for LightGBM
|
||||
#' @description Low-level R interface to train a LightGBM model. Unlike \code{\link{lightgbm}},
|
||||
#' this function is focused on performance (e.g. speed, memory efficiency). It is also
|
||||
#' less likely to have breaking API changes in new releases than \code{\link{lightgbm}}.
|
||||
#' @inheritParams lgb_shared_params
|
||||
#' @param valids a list of \code{lgb.Dataset} objects, used for validation
|
||||
#' @param record Boolean, TRUE will record iteration message to \code{booster$record_evals}
|
||||
#' @param callbacks List of callback functions that are applied at each iteration.
|
||||
#' @param reset_data Boolean, setting it to TRUE (not the default value) will transform the
|
||||
#' booster model into a predictor model which frees up memory and the
|
||||
#' original datasets
|
||||
#' @inheritSection lgb_shared_params Early Stopping
|
||||
#' @return a trained booster model \code{lgb.Booster}.
|
||||
#'
|
||||
#' @examples
|
||||
#' \donttest{
|
||||
#' \dontshow{setLGBMthreads(2L)}
|
||||
#' \dontshow{data.table::setDTthreads(1L)}
|
||||
#' data(agaricus.train, package = "lightgbm")
|
||||
#' train <- agaricus.train
|
||||
#' dtrain <- lgb.Dataset(train$data, label = train$label)
|
||||
#' data(agaricus.test, package = "lightgbm")
|
||||
#' test <- agaricus.test
|
||||
#' dtest <- lgb.Dataset.create.valid(dtrain, test$data, label = test$label)
|
||||
#' params <- list(
|
||||
#' objective = "regression"
|
||||
#' , metric = "l2"
|
||||
#' , min_data = 1L
|
||||
#' , learning_rate = 1.0
|
||||
#' , num_threads = 2L
|
||||
#' )
|
||||
#' valids <- list(test = dtest)
|
||||
#' model <- lgb.train(
|
||||
#' params = params
|
||||
#' , data = dtrain
|
||||
#' , nrounds = 5L
|
||||
#' , valids = valids
|
||||
#' , early_stopping_rounds = 3L
|
||||
#' )
|
||||
#' }
|
||||
#'
|
||||
#' @export
|
||||
lgb.train <- function(params = list(),
|
||||
data,
|
||||
nrounds = 100L,
|
||||
valids = list(),
|
||||
obj = NULL,
|
||||
eval = NULL,
|
||||
verbose = 1L,
|
||||
record = TRUE,
|
||||
eval_freq = 1L,
|
||||
init_model = NULL,
|
||||
early_stopping_rounds = NULL,
|
||||
callbacks = list(),
|
||||
reset_data = FALSE,
|
||||
serializable = TRUE) {
|
||||
|
||||
# validate inputs early to avoid unnecessary computation
|
||||
if (nrounds <= 0L) {
|
||||
stop("nrounds should be greater than zero")
|
||||
}
|
||||
if (!.is_Dataset(x = data)) {
|
||||
stop("lgb.train: data must be an lgb.Dataset instance")
|
||||
}
|
||||
if (length(valids) > 0L) {
|
||||
if (!identical(class(valids), "list") || !all(vapply(valids, .is_Dataset, logical(1L)))) {
|
||||
stop("lgb.train: valids must be a list of lgb.Dataset elements")
|
||||
}
|
||||
evnames <- names(valids)
|
||||
if (is.null(evnames) || !all(nzchar(evnames))) {
|
||||
stop("lgb.train: each element of valids must have a name")
|
||||
}
|
||||
}
|
||||
|
||||
# set some parameters, resolving the way they were passed in with other parameters
|
||||
# in `params`.
|
||||
# this ensures that the model stored with Booster$save() correctly represents
|
||||
# what was passed in
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "verbosity"
|
||||
, params = params
|
||||
, alternative_kwarg_value = verbose
|
||||
)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "num_iterations"
|
||||
, params = params
|
||||
, alternative_kwarg_value = nrounds
|
||||
)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "metric"
|
||||
, params = params
|
||||
, alternative_kwarg_value = NULL
|
||||
)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "objective"
|
||||
, params = params
|
||||
, alternative_kwarg_value = obj
|
||||
)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "early_stopping_round"
|
||||
, params = params
|
||||
, alternative_kwarg_value = early_stopping_rounds
|
||||
)
|
||||
early_stopping_rounds <- params[["early_stopping_round"]]
|
||||
|
||||
# extract any function objects passed for objective or metric
|
||||
fobj <- NULL
|
||||
if (is.function(params$objective)) {
|
||||
fobj <- params$objective
|
||||
params$objective <- "none"
|
||||
}
|
||||
|
||||
# If eval is a single function, store it as a 1-element list
|
||||
# (for backwards compatibility). If it is a list of functions, store
|
||||
# all of them. This makes it possible to pass any mix of strings like "auc"
|
||||
# and custom functions to eval
|
||||
params <- .check_eval(params = params, eval = eval)
|
||||
eval_functions <- list(NULL)
|
||||
if (is.function(eval)) {
|
||||
eval_functions <- list(eval)
|
||||
}
|
||||
if (methods::is(eval, "list")) {
|
||||
eval_functions <- Filter(
|
||||
f = is.function
|
||||
, x = eval
|
||||
)
|
||||
}
|
||||
|
||||
# Init predictor to empty
|
||||
predictor <- NULL
|
||||
|
||||
# Check for boosting from a trained model
|
||||
if (is.character(init_model)) {
|
||||
predictor <- Predictor$new(modelfile = init_model)
|
||||
} else if (.is_Booster(x = init_model)) {
|
||||
predictor <- init_model$to_predictor()
|
||||
}
|
||||
|
||||
# Set the iteration to start from / end to (and check for boosting from a trained model, again)
|
||||
begin_iteration <- 1L
|
||||
if (!is.null(predictor)) {
|
||||
begin_iteration <- predictor$current_iter() + 1L
|
||||
}
|
||||
end_iteration <- begin_iteration + params[["num_iterations"]] - 1L
|
||||
|
||||
# pop interaction_constraints off of params. It needs some preprocessing on the
|
||||
# R side before being passed into the Dataset object
|
||||
interaction_constraints <- params[["interaction_constraints"]]
|
||||
params["interaction_constraints"] <- NULL
|
||||
|
||||
# Construct datasets, if needed
|
||||
data$update_params(params = params)
|
||||
data$construct()
|
||||
|
||||
# Check interaction constraints
|
||||
params[["interaction_constraints"]] <- .check_interaction_constraints(
|
||||
interaction_constraints = interaction_constraints
|
||||
, column_names = data$get_colnames()
|
||||
)
|
||||
|
||||
# Update parameters with parsed parameters
|
||||
data$update_params(params)
|
||||
|
||||
# Create the predictor set
|
||||
data$.__enclos_env__$private$set_predictor(predictor)
|
||||
|
||||
valid_contain_train <- FALSE
|
||||
train_data_name <- "train"
|
||||
reduced_valid_sets <- list()
|
||||
|
||||
# Parse validation datasets
|
||||
if (length(valids) > 0L) {
|
||||
|
||||
for (key in names(valids)) {
|
||||
|
||||
# Use names to get validation datasets
|
||||
valid_data <- valids[[key]]
|
||||
|
||||
# Check for duplicate train/validation dataset
|
||||
if (identical(data, valid_data)) {
|
||||
valid_contain_train <- TRUE
|
||||
train_data_name <- key
|
||||
next
|
||||
}
|
||||
|
||||
# Update parameters, data
|
||||
valid_data$update_params(params)
|
||||
valid_data$set_reference(data)
|
||||
reduced_valid_sets[[key]] <- valid_data
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
# Add printing log callback
|
||||
if (params[["verbosity"]] > 0L && eval_freq > 0L) {
|
||||
callbacks <- .add_cb(
|
||||
cb_list = callbacks
|
||||
, cb = cb_print_evaluation(period = eval_freq)
|
||||
)
|
||||
}
|
||||
|
||||
# Add evaluation log callback
|
||||
if (record && length(valids) > 0L) {
|
||||
callbacks <- .add_cb(
|
||||
cb_list = callbacks
|
||||
, cb = cb_record_evaluation()
|
||||
)
|
||||
}
|
||||
|
||||
# Did user pass parameters that indicate they want to use early stopping?
|
||||
using_early_stopping <- !is.null(early_stopping_rounds) && early_stopping_rounds > 0L
|
||||
|
||||
boosting_param_names <- .PARAMETER_ALIASES()[["boosting"]]
|
||||
using_dart <- any(
|
||||
sapply(
|
||||
X = boosting_param_names
|
||||
, FUN = function(param) {
|
||||
identical(params[[param]], "dart")
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# Cannot use early stopping with 'dart' boosting
|
||||
if (using_dart) {
|
||||
if (using_early_stopping) {
|
||||
warning("Early stopping is not available in 'dart' mode.")
|
||||
}
|
||||
using_early_stopping <- FALSE
|
||||
|
||||
# Remove the cb_early_stop() function if it was passed in to callbacks
|
||||
callbacks <- Filter(
|
||||
f = function(cb_func) {
|
||||
!identical(attr(cb_func, "name"), "cb_early_stop")
|
||||
}
|
||||
, x = callbacks
|
||||
)
|
||||
}
|
||||
|
||||
# If user supplied early_stopping_rounds, add the early stopping callback
|
||||
if (using_early_stopping) {
|
||||
callbacks <- .add_cb(
|
||||
cb_list = callbacks
|
||||
, cb = cb_early_stop(
|
||||
stopping_rounds = early_stopping_rounds
|
||||
, first_metric_only = isTRUE(params[["first_metric_only"]])
|
||||
, verbose = params[["verbosity"]] > 0L
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
cb <- .categorize_callbacks(cb_list = callbacks)
|
||||
|
||||
# Construct booster with datasets
|
||||
booster <- Booster$new(params = params, train_set = data)
|
||||
if (valid_contain_train) {
|
||||
booster$set_train_data_name(name = train_data_name)
|
||||
}
|
||||
|
||||
for (key in names(reduced_valid_sets)) {
|
||||
booster$add_valid(data = reduced_valid_sets[[key]], name = key)
|
||||
}
|
||||
|
||||
# Callback env
|
||||
env <- CB_ENV$new()
|
||||
env$model <- booster
|
||||
env$begin_iteration <- begin_iteration
|
||||
env$end_iteration <- end_iteration
|
||||
|
||||
# Start training model using number of iterations to start and end with
|
||||
for (i in seq.int(from = begin_iteration, to = end_iteration)) {
|
||||
|
||||
# Overwrite iteration in environment
|
||||
env$iteration <- i
|
||||
env$eval_list <- list()
|
||||
|
||||
# Loop through "pre_iter" element
|
||||
for (f in cb$pre_iter) {
|
||||
f(env)
|
||||
}
|
||||
|
||||
# Update one boosting iteration
|
||||
booster$update(fobj = fobj)
|
||||
|
||||
# Prepare collection of evaluation results
|
||||
eval_list <- list()
|
||||
|
||||
# Collection: Has validation dataset?
|
||||
if (length(valids) > 0L) {
|
||||
|
||||
# Get evaluation results with passed-in functions
|
||||
for (eval_function in eval_functions) {
|
||||
|
||||
# Validation has training dataset?
|
||||
if (valid_contain_train) {
|
||||
eval_list <- append(eval_list, booster$eval_train(feval = eval_function))
|
||||
}
|
||||
|
||||
eval_list <- append(eval_list, booster$eval_valid(feval = eval_function))
|
||||
}
|
||||
|
||||
# Calling booster$eval_valid() will get
|
||||
# evaluation results with the metrics in params$metric by calling LGBM_BoosterGetEval_R",
|
||||
# so need to be sure that gets called, which it wouldn't be above if no functions
|
||||
# were passed in
|
||||
if (length(eval_functions) == 0L) {
|
||||
if (valid_contain_train) {
|
||||
eval_list <- append(eval_list, booster$eval_train(feval = eval_function))
|
||||
}
|
||||
eval_list <- append(eval_list, booster$eval_valid(feval = eval_function))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
# Write evaluation result in environment
|
||||
env$eval_list <- eval_list
|
||||
|
||||
# Loop through env
|
||||
for (f in cb$post_iter) {
|
||||
f(env)
|
||||
}
|
||||
|
||||
# Check for early stopping and break if needed
|
||||
if (env$met_early_stop) break
|
||||
|
||||
}
|
||||
|
||||
# check if any valids were given other than the training data
|
||||
non_train_valid_names <- names(valids)[!(names(valids) == train_data_name)]
|
||||
first_valid_name <- non_train_valid_names[1L]
|
||||
|
||||
# When early stopping is not activated, we compute the best iteration / score ourselves by
|
||||
# selecting the first metric and the first dataset
|
||||
if (record && length(non_train_valid_names) > 0L && is.na(env$best_score)) {
|
||||
|
||||
# when using a custom eval function, the metric name is returned from the
|
||||
# function, so figure it out from record_evals
|
||||
if (!is.null(eval_functions[1L])) {
|
||||
first_metric <- names(booster$record_evals[[first_valid_name]])[1L]
|
||||
} else {
|
||||
first_metric <- booster$.__enclos_env__$private$eval_names[1L]
|
||||
}
|
||||
|
||||
.find_best <- which.min
|
||||
if (isTRUE(env$eval_list[[1L]]$higher_better[1L])) {
|
||||
.find_best <- which.max
|
||||
}
|
||||
booster$best_iter <- unname(
|
||||
.find_best(
|
||||
unlist(
|
||||
booster$record_evals[[first_valid_name]][[first_metric]][[.EVAL_KEY()]]
|
||||
)
|
||||
)
|
||||
)
|
||||
booster$best_score <- booster$record_evals[[first_valid_name]][[first_metric]][[.EVAL_KEY()]][[booster$best_iter]]
|
||||
}
|
||||
|
||||
# Check for booster model conversion to predictor model
|
||||
if (reset_data) {
|
||||
|
||||
# Store temporarily model data elsewhere
|
||||
booster_old <- list(
|
||||
best_iter = booster$best_iter
|
||||
, best_score = booster$best_score
|
||||
, record_evals = booster$record_evals
|
||||
)
|
||||
|
||||
# Reload model
|
||||
booster <- lgb.load(model_str = booster$save_model_to_string())
|
||||
booster$best_iter <- booster_old$best_iter
|
||||
booster$best_score <- booster_old$best_score
|
||||
booster$record_evals <- booster_old$record_evals
|
||||
|
||||
}
|
||||
|
||||
if (serializable) {
|
||||
booster$save_raw()
|
||||
}
|
||||
|
||||
return(booster)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
#' @name lgb_shared_params
|
||||
#' @title Shared parameter docs
|
||||
#' @description Parameter docs shared by \code{lgb.train}, \code{lgb.cv}, and \code{lightgbm}
|
||||
#' @param callbacks List of callback functions that are applied at each iteration.
|
||||
#' @param data a \code{lgb.Dataset} object, used for training. Some functions, such as \code{\link{lgb.cv}},
|
||||
#' may allow you to pass other types of data like \code{matrix} and then separately supply
|
||||
#' \code{label} as a keyword argument.
|
||||
#' @param early_stopping_rounds int. Activates early stopping. When this parameter is non-null,
|
||||
#' training will stop if the evaluation of any metric on any validation set
|
||||
#' fails to improve for \code{early_stopping_rounds} consecutive boosting rounds.
|
||||
#' If training stops early, the returned model will have attribute \code{best_iter}
|
||||
#' set to the iteration number of the best iteration.
|
||||
#' @param eval evaluation function(s). This can be a character vector, function, or list with a mixture of
|
||||
#' strings and functions.
|
||||
#'
|
||||
#' \itemize{
|
||||
#' \item{\bold{a. character vector}:
|
||||
#' If you provide a character vector to this argument, it should contain strings with valid
|
||||
#' evaluation metrics.
|
||||
#' See \href{https://lightgbm.readthedocs.io/en/latest/Parameters.html#metric}{
|
||||
#' The "metric" section of the documentation}
|
||||
#' for a list of valid metrics.
|
||||
#' }
|
||||
#' \item{\bold{b. function}:
|
||||
#' You can provide a custom evaluation function. This
|
||||
#' should accept the keyword arguments \code{preds} and \code{dtrain} and should return a named
|
||||
#' list with three elements:
|
||||
#' \itemize{
|
||||
#' \item{\code{name}: A string with the name of the metric, used for printing
|
||||
#' and storing results.
|
||||
#' }
|
||||
#' \item{\code{value}: A single number indicating the value of the metric for the
|
||||
#' given predictions and true values
|
||||
#' }
|
||||
#' \item{
|
||||
#' \code{higher_better}: A boolean indicating whether higher values indicate a better fit.
|
||||
#' For example, this would be \code{FALSE} for metrics like MAE or RMSE.
|
||||
#' }
|
||||
#' }
|
||||
#' }
|
||||
#' \item{\bold{c. list}:
|
||||
#' If a list is given, it should only contain character vectors and functions.
|
||||
#' These should follow the requirements from the descriptions above.
|
||||
#' }
|
||||
#' }
|
||||
#' @param eval_freq evaluation output frequency, only effective when verbose > 0 and \code{valids} has been provided
|
||||
#' @param init_model path of model file or \code{lgb.Booster} object, will continue training from this model
|
||||
#' @param nrounds number of training rounds
|
||||
#' @param obj objective function, can be character or custom objective function. Examples include
|
||||
#' \code{regression}, \code{regression_l1}, \code{huber},
|
||||
#' \code{binary}, \code{lambdarank}, \code{multiclass}, \code{multiclass}
|
||||
#' @param params a list of parameters. See \href{https://lightgbm.readthedocs.io/en/latest/Parameters.html}{
|
||||
#' the "Parameters" section of the documentation} for a list of parameters and valid values.
|
||||
#' @param verbose verbosity for output, if <= 0 and \code{valids} has been provided, also will disable the
|
||||
#' printing of evaluation during training
|
||||
#' @param serializable whether to make the resulting objects serializable through functions such as
|
||||
#' \code{save} or \code{saveRDS} (see section "Model serialization").
|
||||
#' @section Early Stopping:
|
||||
#'
|
||||
#' "early stopping" refers to stopping the training process if the model's performance on a given
|
||||
#' validation set does not improve for several consecutive iterations.
|
||||
#'
|
||||
#' If multiple arguments are given to \code{eval}, their order will be preserved. If you enable
|
||||
#' early stopping by setting \code{early_stopping_rounds} in \code{params}, by default all
|
||||
#' metrics will be considered for early stopping.
|
||||
#'
|
||||
#' If you want to only consider the first metric for early stopping, pass
|
||||
#' \code{first_metric_only = TRUE} in \code{params}. Note that if you also specify \code{metric}
|
||||
#' in \code{params}, that metric will be considered the "first" one. If you omit \code{metric},
|
||||
#' a default metric will be used based on your choice for the parameter \code{obj} (keyword argument)
|
||||
#' or \code{objective} (passed into \code{params}).
|
||||
#'
|
||||
#' \bold{NOTE:} if using \code{boosting_type="dart"}, any early stopping configuration will be ignored
|
||||
#' and early stopping will not be performed.
|
||||
#' @section Model serialization:
|
||||
#'
|
||||
#' LightGBM model objects can be serialized and de-serialized through functions such as \code{save}
|
||||
#' or \code{saveRDS}, but similarly to libraries such as 'xgboost', serialization works a bit differently
|
||||
#' from typical R objects. In order to make models serializable in R, a copy of the underlying C++ object
|
||||
#' as serialized raw bytes is produced and stored in the R model object, and when this R object is
|
||||
#' de-serialized, the underlying C++ model object gets reconstructed from these raw bytes, but will only
|
||||
#' do so once some function that uses it is called, such as \code{predict}. In order to forcibly
|
||||
#' reconstruct the C++ object after deserialization (e.g. after calling \code{readRDS} or similar), one
|
||||
#' can use the function \link{lgb.restore_handle} (for example, if one makes predictions in parallel or in
|
||||
#' forked processes, it will be faster to restore the handle beforehand).
|
||||
#'
|
||||
#' Producing and keeping these raw bytes however uses extra memory, and if they are not required,
|
||||
#' it is possible to avoid producing them by passing `serializable=FALSE`. In such cases, these raw
|
||||
#' bytes can be added to the model on demand through function \link{lgb.make_serializable}.
|
||||
#'
|
||||
#' \emph{New in version 4.0.0}
|
||||
#'
|
||||
#' @keywords internal
|
||||
NULL
|
||||
|
||||
#' @name lightgbm
|
||||
#' @title Train a LightGBM model
|
||||
#' @description High-level R interface to train a LightGBM model. Unlike \code{\link{lgb.train}}, this function
|
||||
#' is focused on compatibility with other statistics and machine learning interfaces in R.
|
||||
#' This focus on compatibility means that this interface may experience more frequent breaking API changes
|
||||
#' than \code{\link{lgb.train}}.
|
||||
#' For efficiency-sensitive applications, or for applications where breaking API changes across releases
|
||||
#' is very expensive, use \code{\link{lgb.train}}.
|
||||
#' @inheritParams lgb_shared_params
|
||||
#' @param label Vector of labels, used if \code{data} is not an \code{\link{lgb.Dataset}}
|
||||
#' @param weights Sample / observation weights for rows in the input data. If \code{NULL}, will assume that all
|
||||
#' observations / rows have the same importance / weight.
|
||||
#'
|
||||
#' \emph{Changed from 'weight', in version 4.0.0}
|
||||
#'
|
||||
#' @param objective Optimization objective (e.g. `"regression"`, `"binary"`, etc.).
|
||||
#' For a list of accepted objectives, see
|
||||
#' \href{https://lightgbm.readthedocs.io/en/latest/Parameters.html#objective}{
|
||||
#' the "objective" item of the "Parameters" section of the documentation}.
|
||||
#'
|
||||
#' If passing \code{"auto"} and \code{data} is not of type \code{lgb.Dataset}, the objective will
|
||||
#' be determined according to what is passed for \code{label}:\itemize{
|
||||
#' \item If passing a factor with two variables, will use objective \code{"binary"}.
|
||||
#' \item If passing a factor with more than two variables, will use objective \code{"multiclass"}
|
||||
#' (note that parameter \code{num_class} in this case will also be determined automatically from
|
||||
#' \code{label}).
|
||||
#' \item Otherwise (or if passing \code{lgb.Dataset} as input), will use objective \code{"regression"}.
|
||||
#' }
|
||||
#'
|
||||
#' \emph{New in version 4.0.0}
|
||||
#'
|
||||
#' @param init_score initial score is the base prediction lightgbm will boost from
|
||||
#'
|
||||
#' \emph{New in version 4.0.0}
|
||||
#'
|
||||
#' @param num_threads Number of parallel threads to use. For best speed, this should be set to the number of
|
||||
#' physical cores in the CPU - in a typical x86-64 machine, this corresponds to half the
|
||||
#' number of maximum threads.
|
||||
#'
|
||||
#' Be aware that using too many threads can result in speed degradation in smaller datasets
|
||||
#' (see the parameters documentation for more details).
|
||||
#'
|
||||
#' If passing zero, will use the default number of threads configured for OpenMP
|
||||
#' (typically controlled through an environment variable \code{OMP_NUM_THREADS}).
|
||||
#'
|
||||
#' If passing \code{NULL} (the default), will try to use the number of physical cores in the
|
||||
#' system, but be aware that getting the number of cores detected correctly requires package
|
||||
#' \code{RhpcBLASctl} to be installed.
|
||||
#'
|
||||
#' This parameter gets overridden by \code{num_threads} and its aliases under \code{params}
|
||||
#' if passed there.
|
||||
#'
|
||||
#' \emph{New in version 4.0.0}
|
||||
#'
|
||||
#' @param colnames Character vector of features. Only used if \code{data} is not an \code{\link{lgb.Dataset}}.
|
||||
#' @param categorical_feature categorical features. This can either be a character vector of feature
|
||||
#' names or an integer vector with the indices of the features (e.g.
|
||||
#' \code{c(1L, 10L)} to say "the first and tenth columns").
|
||||
#' Only used if \code{data} is not an \code{\link{lgb.Dataset}}.
|
||||
#'
|
||||
#' @param ... Additional arguments passed to \code{\link{lgb.train}}. For example
|
||||
#' \itemize{
|
||||
#' \item{\code{valids}: a list of \code{lgb.Dataset} objects, used for validation}
|
||||
#' \item{\code{obj}: objective function, can be character or custom objective function. Examples include
|
||||
#' \code{regression}, \code{regression_l1}, \code{huber},
|
||||
#' \code{binary}, \code{lambdarank}, \code{multiclass}, \code{multiclass}}
|
||||
#' \item{\code{eval}: evaluation function, can be (a list of) character or custom eval function}
|
||||
#' \item{\code{record}: Boolean, TRUE will record iteration message to \code{booster$record_evals}}
|
||||
#' \item{\code{reset_data}: Boolean, setting it to TRUE (not the default value) will transform the booster model
|
||||
#' into a predictor model which frees up memory and the original datasets}
|
||||
#' }
|
||||
#' @inheritSection lgb_shared_params Early Stopping
|
||||
#' @return a trained \code{lgb.Booster}
|
||||
#' @export
|
||||
lightgbm <- function(data,
|
||||
label = NULL,
|
||||
weights = NULL,
|
||||
params = list(),
|
||||
nrounds = 100L,
|
||||
verbose = 1L,
|
||||
eval_freq = 1L,
|
||||
early_stopping_rounds = NULL,
|
||||
init_model = NULL,
|
||||
callbacks = list(),
|
||||
serializable = TRUE,
|
||||
objective = "auto",
|
||||
init_score = NULL,
|
||||
num_threads = NULL,
|
||||
colnames = NULL,
|
||||
categorical_feature = NULL,
|
||||
...) {
|
||||
|
||||
# validate inputs early to avoid unnecessary computation
|
||||
if (nrounds <= 0L) {
|
||||
stop("nrounds should be greater than zero")
|
||||
}
|
||||
|
||||
if (is.null(num_threads)) {
|
||||
num_threads <- .get_default_num_threads()
|
||||
}
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "num_threads"
|
||||
, params = params
|
||||
, alternative_kwarg_value = num_threads
|
||||
)
|
||||
params <- .check_wrapper_param(
|
||||
main_param_name = "verbosity"
|
||||
, params = params
|
||||
, alternative_kwarg_value = verbose
|
||||
)
|
||||
|
||||
# Process factors as labels and auto-determine objective
|
||||
if (!.is_Dataset(data)) {
|
||||
data_processor <- DataProcessor$new()
|
||||
temp <- data_processor$process_label(
|
||||
label = label
|
||||
, objective = objective
|
||||
, params = params
|
||||
)
|
||||
label <- temp$label
|
||||
objective <- temp$objective
|
||||
params <- temp$params
|
||||
rm(temp)
|
||||
} else {
|
||||
data_processor <- NULL
|
||||
if (objective == "auto") {
|
||||
objective <- "regression"
|
||||
}
|
||||
}
|
||||
|
||||
# Set data to a temporary variable
|
||||
dtrain <- data
|
||||
|
||||
# Check whether data is lgb.Dataset, if not then create lgb.Dataset manually
|
||||
if (!.is_Dataset(x = dtrain)) {
|
||||
dtrain <- lgb.Dataset(
|
||||
data = data
|
||||
, label = label
|
||||
, weight = weights
|
||||
, init_score = init_score
|
||||
, categorical_feature = categorical_feature
|
||||
, colnames = colnames
|
||||
)
|
||||
}
|
||||
|
||||
train_args <- list(
|
||||
"params" = params
|
||||
, "data" = dtrain
|
||||
, "nrounds" = nrounds
|
||||
, "obj" = objective
|
||||
, "verbose" = params[["verbosity"]]
|
||||
, "eval_freq" = eval_freq
|
||||
, "early_stopping_rounds" = early_stopping_rounds
|
||||
, "init_model" = init_model
|
||||
, "callbacks" = callbacks
|
||||
, "serializable" = serializable
|
||||
)
|
||||
train_args <- append(train_args, list(...))
|
||||
|
||||
if (! "valids" %in% names(train_args)) {
|
||||
train_args[["valids"]] <- list()
|
||||
}
|
||||
|
||||
# Train a model using the regular way
|
||||
bst <- do.call(
|
||||
what = lgb.train
|
||||
, args = train_args
|
||||
)
|
||||
bst$data_processor <- data_processor
|
||||
|
||||
return(bst)
|
||||
}
|
||||
|
||||
#' @name agaricus.train
|
||||
#' @title Training part from Mushroom Data Set
|
||||
#' @description This data set is originally from the Mushroom data set,
|
||||
#' UCI Machine Learning Repository.
|
||||
#' This data set includes the following fields:
|
||||
#'
|
||||
#' \itemize{
|
||||
#' \item{\code{label}: the label for each record}
|
||||
#' \item{\code{data}: a sparse Matrix of \code{dgCMatrix} class, with 126 columns.}
|
||||
#' }
|
||||
#'
|
||||
#' @references
|
||||
#' https://archive.ics.uci.edu/ml/datasets/Mushroom
|
||||
#'
|
||||
#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
|
||||
#' [https://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
|
||||
#' School of Information and Computer Science.
|
||||
#'
|
||||
#' @docType data
|
||||
#' @keywords datasets
|
||||
#' @usage data(agaricus.train)
|
||||
#' @format A list containing a label vector, and a dgCMatrix object with 6513
|
||||
#' rows and 127 variables
|
||||
NULL
|
||||
|
||||
#' @name agaricus.test
|
||||
#' @title Test part from Mushroom Data Set
|
||||
#' @description This data set is originally from the Mushroom data set,
|
||||
#' UCI Machine Learning Repository.
|
||||
#' This data set includes the following fields:
|
||||
#'
|
||||
#' \itemize{
|
||||
#' \item{\code{label}: the label for each record}
|
||||
#' \item{\code{data}: a sparse Matrix of \code{dgCMatrix} class, with 126 columns.}
|
||||
#' }
|
||||
#' @references
|
||||
#' https://archive.ics.uci.edu/ml/datasets/Mushroom
|
||||
#'
|
||||
#' Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository
|
||||
#' [https://archive.ics.uci.edu/ml]. Irvine, CA: University of California,
|
||||
#' School of Information and Computer Science.
|
||||
#'
|
||||
#' @docType data
|
||||
#' @keywords datasets
|
||||
#' @usage data(agaricus.test)
|
||||
#' @format A list containing a label vector, and a dgCMatrix object with 1611
|
||||
#' rows and 126 variables
|
||||
NULL
|
||||
|
||||
#' @name bank
|
||||
#' @title Bank Marketing Data Set
|
||||
#' @description This data set is originally from the Bank Marketing data set,
|
||||
#' UCI Machine Learning Repository.
|
||||
#'
|
||||
#' It contains only the following: bank.csv with 10% of the examples and 17 inputs,
|
||||
#' randomly selected from 3 (older version of this dataset with less inputs).
|
||||
#'
|
||||
#' @references
|
||||
#' https://archive.ics.uci.edu/ml/datasets/Bank+Marketing
|
||||
#'
|
||||
#' S. Moro, P. Cortez and P. Rita. (2014)
|
||||
#' A Data-Driven Approach to Predict the Success of Bank Telemarketing. Decision Support Systems
|
||||
#'
|
||||
#' @docType data
|
||||
#' @keywords datasets
|
||||
#' @usage data(bank)
|
||||
#' @format A data.table with 4521 rows and 17 variables
|
||||
NULL
|
||||
|
||||
# Various imports
|
||||
#' @import methods
|
||||
#' @importFrom Matrix Matrix
|
||||
#' @importFrom R6 R6Class
|
||||
#' @useDynLib lightgbm , .registration = TRUE
|
||||
NULL
|
||||
|
||||
# Suppress false positive warnings from R CMD CHECK about
|
||||
# "unrecognized global variable"
|
||||
globalVariables(c(
|
||||
"."
|
||||
, ".N"
|
||||
, ".SD"
|
||||
, "abs_contribution"
|
||||
, "bar_color"
|
||||
, "Contribution"
|
||||
, "Cover"
|
||||
, "Feature"
|
||||
, "Frequency"
|
||||
, "Gain"
|
||||
, "internal_count"
|
||||
, "internal_value"
|
||||
, "leaf_index"
|
||||
, "leaf_parent"
|
||||
, "leaf_value"
|
||||
, "node_parent"
|
||||
, "split_feature"
|
||||
, "split_gain"
|
||||
, "split_index"
|
||||
, "tree_index"
|
||||
))
|
||||
@@ -0,0 +1,38 @@
|
||||
# [description] List of metrics known to LightGBM. The most up to date list can be found
|
||||
# at https://lightgbm.readthedocs.io/en/latest/Parameters.html#metric-parameters
|
||||
#
|
||||
# [return] A named logical vector, where each key is a metric name and each value is a boolean.
|
||||
# TRUE if higher values of the metric are desirable, FALSE if lower values are desirable.
|
||||
# Note that only the 'main' metrics are stored here, not aliases, since only the 'main' metrics
|
||||
# are returned from the C++ side. For example, if you use `metric = "mse"` in your code,
|
||||
# the metric name `"l2"` will be returned.
|
||||
.METRICS_HIGHER_BETTER <- function() {
|
||||
return(
|
||||
c(
|
||||
"l1" = FALSE
|
||||
, "l2" = FALSE
|
||||
, "mape" = FALSE
|
||||
, "rmse" = FALSE
|
||||
, "quantile" = FALSE
|
||||
, "huber" = FALSE
|
||||
, "fair" = FALSE
|
||||
, "poisson" = FALSE
|
||||
, "gamma" = FALSE
|
||||
, "gamma_deviance" = FALSE
|
||||
, "tweedie" = FALSE
|
||||
, "ndcg" = TRUE
|
||||
, "map" = TRUE
|
||||
, "auc" = TRUE
|
||||
, "average_precision" = TRUE
|
||||
, "r2" = TRUE
|
||||
, "binary_logloss" = FALSE
|
||||
, "binary_error" = FALSE
|
||||
, "auc_mu" = TRUE
|
||||
, "multi_logloss" = FALSE
|
||||
, "multi_error" = FALSE
|
||||
, "cross_entropy" = FALSE
|
||||
, "cross_entropy_lambda" = FALSE
|
||||
, "kullback_leibler" = FALSE
|
||||
)
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
#' @name setLGBMThreads
|
||||
#' @title Set maximum number of threads used by LightGBM
|
||||
#' @description LightGBM attempts to speed up many operations by using multi-threading.
|
||||
#' The number of threads used in those operations can be controlled via the
|
||||
#' \code{num_threads} parameter passed through \code{params} to functions like
|
||||
#' \link{lgb.train} and \link{lgb.Dataset}. However, some operations (like materializing
|
||||
#' a model from a text file) are done via code paths that don't explicitly accept thread-control
|
||||
#' configuration.
|
||||
#'
|
||||
#' Use this function to set the maximum number of threads LightGBM will use for such operations.
|
||||
#'
|
||||
#' This function affects all LightGBM operations in the same process.
|
||||
#'
|
||||
#' So, for example, if you call \code{setLGBMthreads(4)}, no other multi-threaded LightGBM
|
||||
#' operation in the same process will use more than 4 threads.
|
||||
#'
|
||||
#' Call \code{setLGBMthreads(-1)} to remove this limitation.
|
||||
#' @param num_threads maximum number of threads to be used by LightGBM in multi-threaded operations
|
||||
#' @return NULL
|
||||
#' @seealso \link{getLGBMthreads}
|
||||
#' @export
|
||||
setLGBMthreads <- function(num_threads) {
|
||||
.Call(
|
||||
LGBM_SetMaxThreads_R,
|
||||
num_threads
|
||||
)
|
||||
return(invisible(NULL))
|
||||
}
|
||||
|
||||
#' @name getLGBMThreads
|
||||
#' @title Get default number of threads used by LightGBM
|
||||
#' @description LightGBM attempts to speed up many operations by using multi-threading.
|
||||
#' The number of threads used in those operations can be controlled via the
|
||||
#' \code{num_threads} parameter passed through \code{params} to functions like
|
||||
#' \link{lgb.train} and \link{lgb.Dataset}. However, some operations (like materializing
|
||||
#' a model from a text file) are done via code paths that don't explicitly accept thread-control
|
||||
#' configuration.
|
||||
#'
|
||||
#' Use this function to see the default number of threads LightGBM will use for such operations.
|
||||
#' @return number of threads as an integer. \code{-1} means that in situations where parameter \code{num_threads} is
|
||||
#' not explicitly supplied, LightGBM will choose a number of threads to use automatically.
|
||||
#' @seealso \link{setLGBMthreads}
|
||||
#' @export
|
||||
getLGBMthreads <- function() {
|
||||
out <- 0L
|
||||
.Call(
|
||||
LGBM_GetMaxThreads_R,
|
||||
out
|
||||
)
|
||||
return(out)
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
.is_Booster <- function(x) {
|
||||
return(all(c("R6", "lgb.Booster") %in% class(x))) # nolint: class_equals.
|
||||
}
|
||||
|
||||
.is_Dataset <- function(x) {
|
||||
return(all(c("R6", "lgb.Dataset") %in% class(x))) # nolint: class_equals.
|
||||
}
|
||||
|
||||
.is_Predictor <- function(x) {
|
||||
return(all(c("R6", "lgb.Predictor") %in% class(x))) # nolint: class_equals.
|
||||
}
|
||||
|
||||
.is_null_handle <- function(x) {
|
||||
if (is.null(x)) {
|
||||
return(TRUE)
|
||||
}
|
||||
return(
|
||||
isTRUE(.Call(LGBM_HandleIsNull_R, x))
|
||||
)
|
||||
}
|
||||
|
||||
.params2str <- function(params) {
|
||||
|
||||
if (!identical(class(params), "list")) {
|
||||
stop("params must be a list")
|
||||
}
|
||||
|
||||
names(params) <- gsub(".", "_", names(params), fixed = TRUE)
|
||||
param_names <- names(params)
|
||||
ret <- list()
|
||||
|
||||
# Perform key value join
|
||||
for (i in seq_along(params)) {
|
||||
|
||||
# If a parameter has multiple values, join those values together with commas.
|
||||
# trimws() is necessary because format() will pad to make strings the same width
|
||||
val <- paste(
|
||||
trimws(
|
||||
format(
|
||||
x = unname(params[[i]])
|
||||
, scientific = FALSE
|
||||
)
|
||||
)
|
||||
, collapse = ","
|
||||
)
|
||||
if (nchar(val) <= 0L) next # Skip join
|
||||
|
||||
# Join key value
|
||||
pair <- paste(c(param_names[[i]], val), collapse = "=")
|
||||
ret <- c(ret, pair)
|
||||
|
||||
}
|
||||
|
||||
if (length(ret) == 0L) {
|
||||
return("")
|
||||
}
|
||||
|
||||
return(paste(ret, collapse = " "))
|
||||
|
||||
}
|
||||
|
||||
# [description]
|
||||
#
|
||||
# Besides applying checks, this function
|
||||
#
|
||||
# 1. turns feature *names* into 1-based integer positions, then
|
||||
# 2. adds an extra list element with skipped features, then
|
||||
# 3. turns 1-based integer positions into 0-based positions, and finally
|
||||
# 4. collapses the values of each list element into a string like "[0, 1]".
|
||||
#
|
||||
.check_interaction_constraints <- function(interaction_constraints, column_names) {
|
||||
if (is.null(interaction_constraints)) {
|
||||
return(list())
|
||||
}
|
||||
if (!identical(class(interaction_constraints), "list")) {
|
||||
stop("interaction_constraints must be a list")
|
||||
}
|
||||
|
||||
column_indices <- seq_along(column_names)
|
||||
|
||||
# Convert feature names to 1-based integer positions and apply checks
|
||||
for (j in seq_along(interaction_constraints)) {
|
||||
constraint <- interaction_constraints[[j]]
|
||||
|
||||
if (is.character(constraint)) {
|
||||
constraint_indices <- match(constraint, column_names)
|
||||
} else if (is.numeric(constraint)) {
|
||||
constraint_indices <- as.integer(constraint)
|
||||
} else {
|
||||
stop("every element in interaction_constraints must be a character vector or numeric vector")
|
||||
}
|
||||
|
||||
# Features outside range?
|
||||
bad <- !(constraint_indices %in% column_indices)
|
||||
if (any(bad)) {
|
||||
stop(
|
||||
"unknown feature(s) in interaction_constraints: "
|
||||
, toString(sQuote(constraint[bad], q = FALSE))
|
||||
)
|
||||
}
|
||||
|
||||
interaction_constraints[[j]] <- constraint_indices
|
||||
}
|
||||
|
||||
# Add missing features as new interaction set
|
||||
remaining_indices <- setdiff(
|
||||
column_indices, sort(unique(unlist(interaction_constraints)))
|
||||
)
|
||||
if (length(remaining_indices) > 0L) {
|
||||
interaction_constraints <- c(
|
||||
interaction_constraints, list(remaining_indices)
|
||||
)
|
||||
}
|
||||
|
||||
# Turn indices 0-based and convert to string
|
||||
for (j in seq_along(interaction_constraints)) {
|
||||
interaction_constraints[[j]] <- paste0(
|
||||
"[", paste(interaction_constraints[[j]] - 1L, collapse = ","), "]"
|
||||
)
|
||||
}
|
||||
return(interaction_constraints)
|
||||
}
|
||||
|
||||
|
||||
# [description]
|
||||
# Take any character values from eval and store them in params$metric.
|
||||
# This has to account for the fact that `eval` could be a character vector,
|
||||
# a function, a list of functions, or a list with a mix of strings and
|
||||
# functions
|
||||
.check_eval <- function(params, eval) {
|
||||
|
||||
if (is.null(params$metric)) {
|
||||
params$metric <- list()
|
||||
} else if (is.character(params$metric)) {
|
||||
params$metric <- as.list(params$metric)
|
||||
}
|
||||
|
||||
# if 'eval' is a character vector or list, find the character
|
||||
# elements and add them to 'metric'
|
||||
if (!is.function(eval)) {
|
||||
for (i in seq_along(eval)) {
|
||||
element <- eval[[i]]
|
||||
if (is.character(element)) {
|
||||
params$metric <- append(params$metric, element)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# If more than one character metric was given, then "None" should
|
||||
# not be included
|
||||
if (length(params$metric) > 1L) {
|
||||
params$metric <- Filter(
|
||||
f = function(metric) {
|
||||
!(metric %in% .NO_METRIC_STRINGS())
|
||||
}
|
||||
, x = params$metric
|
||||
)
|
||||
}
|
||||
|
||||
# duplicate metrics should be filtered out
|
||||
params$metric <- as.list(unique(unlist(params$metric)))
|
||||
|
||||
return(params)
|
||||
}
|
||||
|
||||
|
||||
# [description]
|
||||
#
|
||||
# Resolve differences between passed-in keyword arguments, parameters,
|
||||
# and parameter aliases. This function exists because some functions in the
|
||||
# package take in parameters through their own keyword arguments other than
|
||||
# the `params` list.
|
||||
#
|
||||
# If the same underlying parameter is provided multiple
|
||||
# ways, the first item in this list is used:
|
||||
#
|
||||
# 1. the main (non-alias) parameter found in `params`
|
||||
# 2. the alias with the highest priority found in `params`
|
||||
# 3. the keyword argument passed in
|
||||
#
|
||||
# For example, "num_iterations" can also be provided to lgb.train()
|
||||
# via keyword "nrounds". lgb.train() will choose one value for this parameter
|
||||
# based on the first match in this list:
|
||||
#
|
||||
# 1. params[["num_iterations]]
|
||||
# 2. the highest priority alias of "num_iterations" found in params
|
||||
# 3. the nrounds keyword argument
|
||||
#
|
||||
# If multiple aliases are found in `params` for the same parameter, they are
|
||||
# all removed before returning `params`.
|
||||
#
|
||||
# [return]
|
||||
# params with num_iterations set to the chosen value, and other aliases
|
||||
# of num_iterations removed
|
||||
.check_wrapper_param <- function(main_param_name, params, alternative_kwarg_value) {
|
||||
|
||||
aliases <- .PARAMETER_ALIASES()[[main_param_name]]
|
||||
aliases_provided <- aliases[aliases %in% names(params)]
|
||||
aliases_provided <- aliases_provided[aliases_provided != main_param_name]
|
||||
|
||||
# prefer the main parameter
|
||||
if (!is.null(params[[main_param_name]])) {
|
||||
for (param in aliases_provided) {
|
||||
params[[param]] <- NULL
|
||||
}
|
||||
return(params)
|
||||
}
|
||||
|
||||
# if the main parameter wasn't provided, prefer the first alias
|
||||
if (length(aliases_provided) > 0L) {
|
||||
first_param <- aliases_provided[1L]
|
||||
params[[main_param_name]] <- params[[first_param]]
|
||||
for (param in aliases_provided) {
|
||||
params[[param]] <- NULL
|
||||
}
|
||||
return(params)
|
||||
}
|
||||
|
||||
# if not provided in params at all, use the alternative value provided
|
||||
# through a keyword argument from lgb.train(), lgb.cv(), etc.
|
||||
params[[main_param_name]] <- alternative_kwarg_value
|
||||
return(params)
|
||||
}
|
||||
|
||||
#' @importFrom parallel detectCores
|
||||
.get_default_num_threads <- function() {
|
||||
if (requireNamespace("RhpcBLASctl", quietly = TRUE)) { # nolint: undesirable_function.
|
||||
return(RhpcBLASctl::get_num_cores())
|
||||
} else {
|
||||
msg <- "Optional package 'RhpcBLASctl' not found."
|
||||
cores <- 0L
|
||||
if (Sys.info()["sysname"] != "Linux") {
|
||||
cores <- parallel::detectCores(logical = FALSE)
|
||||
if (is.na(cores) || cores < 0L) {
|
||||
cores <- 0L
|
||||
}
|
||||
}
|
||||
if (cores == 0L) {
|
||||
msg <- paste(msg, "Will use default number of OpenMP threads.", sep = " ")
|
||||
} else {
|
||||
msg <- paste(msg, "Detection of CPU cores might not be accurate.", sep = " ")
|
||||
}
|
||||
warning(msg)
|
||||
return(cores)
|
||||
}
|
||||
}
|
||||
|
||||
.equal_or_both_null <- function(a, b) {
|
||||
if (is.null(a)) {
|
||||
if (!is.null(b)) {
|
||||
return(FALSE)
|
||||
}
|
||||
return(TRUE)
|
||||
} else {
|
||||
if (is.null(b)) {
|
||||
return(FALSE)
|
||||
}
|
||||
return(a == b)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,508 @@
|
||||
# LightGBM R-package
|
||||
|
||||
[](https://cran.r-project.org/package=lightgbm)
|
||||
[](https://cran.r-project.org/package=lightgbm)
|
||||
[](https://lightgbm.readthedocs.io/en/latest/R/reference/)
|
||||
|
||||
<img src="man/figures/logo.svg" align="right" alt="" width="175" />
|
||||
|
||||
### Contents
|
||||
|
||||
* [Installation](#installation)
|
||||
- [Installing the CRAN Package](#installing-the-cran-package)
|
||||
- [Installing from Source with CMake](#install)
|
||||
- [Installing a GPU-enabled Build](#installing-a-gpu-enabled-build)
|
||||
- [Installing Precompiled Binaries](#installing-precompiled-binaries)
|
||||
- [Installing from a Pre-compiled lib_lightgbm](#lib_lightgbm)
|
||||
* [Examples](#examples)
|
||||
* [Testing](#testing)
|
||||
- [Running the Tests](#running-the-tests)
|
||||
- [Code Coverage](#code-coverage)
|
||||
* [Updating Documentation](#updating-documentation)
|
||||
* [Preparing a CRAN Package](#preparing-a-cran-package)
|
||||
* [Known Issues](#known-issues)
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
For the easiest installation, go to ["Installing the CRAN package"](#installing-the-cran-package).
|
||||
|
||||
If you experience any issues with that, try ["Installing from Source with CMake"](#install). This can produce a more efficient version of the library on Windows systems with Visual Studio.
|
||||
|
||||
To build a GPU-enabled version of the package, follow the steps in ["Installing a GPU-enabled Build"](#installing-a-gpu-enabled-build).
|
||||
|
||||
If any of the above options do not work for you or do not meet your needs, please let the maintainers know by [opening an issue](https://github.com/lightgbm-org/LightGBM/issues).
|
||||
|
||||
When your package installation is done, you can check quickly if your LightGBM R-package is working by running the following:
|
||||
|
||||
```r
|
||||
library(lightgbm)
|
||||
data(agaricus.train, package='lightgbm')
|
||||
train <- agaricus.train
|
||||
dtrain <- lgb.Dataset(train$data, label = train$label)
|
||||
model <- lgb.cv(
|
||||
params = list(
|
||||
objective = "regression"
|
||||
, metric = "l2"
|
||||
)
|
||||
, data = dtrain
|
||||
)
|
||||
```
|
||||
|
||||
### Installing the CRAN package
|
||||
|
||||
`{lightgbm}` is [available on CRAN](https://cran.r-project.org/package=lightgbm), and can be installed with the following R code.
|
||||
|
||||
```r
|
||||
install.packages("lightgbm", repos = "https://cran.r-project.org")
|
||||
```
|
||||
|
||||
This is the easiest way to install `{lightgbm}`. It does not require `CMake` or `Visual Studio`, and should work well on many different operating systems and compilers.
|
||||
|
||||
Each CRAN package is also available on [LightGBM releases](https://github.com/lightgbm-org/LightGBM/releases), with a name like `lightgbm-{VERSION}-r-cran.tar.gz`.
|
||||
|
||||
#### Custom Installation (Linux, Mac)
|
||||
|
||||
The steps above should work on most systems, but users with highly-customized environments might want to change how R builds packages from source.
|
||||
|
||||
To change the compiler used when installing the CRAN package, you can create a file `~/.R/Makevars` which overrides `CC` (`C` compiler) and `CXX` (`C++` compiler).
|
||||
|
||||
For example, to use `gcc-14` instead of `clang` on macOS, you could use something like the following:
|
||||
|
||||
```make
|
||||
# ~/.R/Makevars
|
||||
CC=gcc-14
|
||||
CC17=gcc-14
|
||||
CXX=g++-14
|
||||
CXX17=g++-14
|
||||
```
|
||||
|
||||
To check the values R is using, run the following:
|
||||
|
||||
```shell
|
||||
R CMD config --all
|
||||
```
|
||||
|
||||
### Installing from Source with CMake <a id="install"></a>
|
||||
|
||||
You need to install git and [CMake](https://cmake.org/) first.
|
||||
|
||||
Note: this method is only supported on 64-bit systems. If you need to run LightGBM on 32-bit Windows (i386), follow the instructions in ["Installing the CRAN Package"](#installing-the-cran-package).
|
||||
|
||||
#### Windows Preparation
|
||||
|
||||
NOTE: Windows users may need to run with administrator rights (either R or the command prompt, depending on the way you are installing this package).
|
||||
|
||||
Installing a 64-bit version of [Rtools](https://cran.r-project.org/bin/windows/Rtools/) is mandatory.
|
||||
|
||||
After installing `Rtools` and `CMake`, be sure the following paths are added to the environment variable `PATH`. These may have been automatically added when installing other software.
|
||||
|
||||
* `Rtools`
|
||||
- If you have `Rtools` 4.0, example:
|
||||
- `C:\rtools40\mingw64\bin`
|
||||
- `C:\rtools40\usr\bin`
|
||||
- If you have `Rtools` 4.2+, example:
|
||||
- `C:\rtools42\x86_64-w64-mingw32.static.posix\bin`
|
||||
- `C:\rtools42\usr\bin`
|
||||
- **NOTE**: this is e.g. `rtools43\` for R 4.3
|
||||
* `CMake`
|
||||
- example: `C:\Program Files\CMake\bin`
|
||||
* `R`
|
||||
- example: `C:\Program Files\R\R-4.5.1\bin`
|
||||
|
||||
NOTE: Two `Rtools` paths are required from `Rtools` 4.0 onwards because paths and the list of included software was changed in `Rtools` 4.0.
|
||||
|
||||
NOTE: `Rtools42` and later take a very different approach to the compiler toolchain than previous releases, and how you install it changes what is required to build packages. See ["Howto: Building R 4.2 and packages on Windows"](https://cran.r-project.org/bin/windows/base/howto-R-4.2.html).
|
||||
|
||||
#### Windows Toolchain Options
|
||||
|
||||
A "toolchain" refers to the collection of software used to build the library. The R-package can be built with three different toolchains.
|
||||
|
||||
**Warning for Windows users**: it is recommended to use *Visual Studio* for its better multi-threading efficiency in Windows for many core systems. For very simple systems (dual core computers or worse), MinGW64 is recommended for maximum performance. If you do not know what to choose, it is recommended to use [Visual Studio](https://visualstudio.microsoft.com/downloads/), the default compiler. **Do not try using MinGW in Windows on many core systems. It may result in 10x slower results than Visual Studio.**
|
||||
|
||||
**Visual Studio (default)**
|
||||
|
||||
By default, the package will be built with [Visual Studio Build Tools](https://visualstudio.microsoft.com/downloads/).
|
||||
|
||||
**MSYS2 (R 4.x)**
|
||||
|
||||
If you are using R 4.x and installation fails with Visual Studio, `LightGBM` will fall back to using [MSYS2](https://www.msys2.org/). This should work with the tools already bundled in `Rtools` 4.0.
|
||||
|
||||
If you want to force `LightGBM` to use MSYS2 (for any R version), pass `--use-msys2` to the installation script.
|
||||
|
||||
```shell
|
||||
Rscript build_r.R --use-msys2
|
||||
```
|
||||
|
||||
**MinGW**
|
||||
|
||||
If you want to force `LightGBM` to use [MinGW](https://www.mingw-w64.org/) (for any R version), pass `--use-mingw` to the installation script.
|
||||
|
||||
```shell
|
||||
Rscript build_r.R --use-mingw
|
||||
```
|
||||
|
||||
#### Mac OS Preparation
|
||||
|
||||
You can perform installation either with **Apple Clang** or **gcc**. In case you prefer **Apple Clang**, you should install **OpenMP** (details for installation can be found in [Installation Guide](https://github.com/lightgbm-org/LightGBM/blob/main/docs/Installation-Guide.rst#apple-clang)) first. In case you prefer **gcc**, you need to install it (details for installation can be found in [Installation Guide](https://github.com/lightgbm-org/LightGBM/blob/main/docs/Installation-Guide.rst#gcc)) and set some environment variables to tell R to use `gcc` and `g++`. If you install these from Homebrew, your versions of `g++` and `gcc` are most likely in `/usr/local/bin`, as shown below.
|
||||
|
||||
```
|
||||
# replace 8 with version of gcc installed on your machine
|
||||
export CXX=/usr/local/bin/g++-8 CC=/usr/local/bin/gcc-8
|
||||
```
|
||||
|
||||
#### Install with CMake
|
||||
|
||||
After following the "preparation" steps above for your operating system, build and install the R-package with the following commands:
|
||||
|
||||
```sh
|
||||
git clone --recursive https://github.com/lightgbm-org/LightGBM
|
||||
cd LightGBM
|
||||
Rscript build_r.R
|
||||
```
|
||||
|
||||
The `build_r.R` script builds the package in a temporary directory called `lightgbm_r`. It will destroy and recreate that directory each time you run the script. That script supports the following command-line options:
|
||||
|
||||
- `--no-build-vignettes`: Skip building vignettes.
|
||||
- `-j[jobs]`: Number of threads to use when compiling LightGBM. E.g., `-j4` will try to compile 4 objects at a time.
|
||||
- by default, this script uses single-thread compilation
|
||||
- for best results, set `-j` to the number of physical CPUs
|
||||
- `--skip-install`: Build the package tarball, but do not install it.
|
||||
- `--use-gpu`: Build a GPU-enabled version of the library.
|
||||
- `--use-mingw`: Force the use of MinGW toolchain, regardless of R version.
|
||||
- `--use-msys2`: Force the use of MSYS2 toolchain, regardless of R version.
|
||||
|
||||
Note: for the build with Visual Studio/VS Build Tools in Windows, you should use the Windows CMD or PowerShell.
|
||||
|
||||
### Installing a GPU-enabled Build
|
||||
|
||||
You will need to install Boost and OpenCL first: details for installation can be found in [Installation-Guide](https://github.com/lightgbm-org/LightGBM/blob/main/docs/Installation-Guide.rst#build-gpu-version).
|
||||
|
||||
After installing these other libraries, follow the steps in ["Installing from Source with CMake"](#install). When you reach the step that mentions `build_r.R`, pass the flag `--use-gpu`.
|
||||
|
||||
```shell
|
||||
Rscript build_r.R --use-gpu
|
||||
```
|
||||
|
||||
You may also need or want to provide additional configuration, depending on your setup. For example, you may need to provide locations for Boost and OpenCL.
|
||||
|
||||
```shell
|
||||
Rscript build_r.R \
|
||||
--use-gpu \
|
||||
--opencl-library=/usr/lib/x86_64-linux-gnu/libOpenCL.so \
|
||||
--boost-librarydir=/usr/lib/x86_64-linux-gnu
|
||||
```
|
||||
|
||||
The following options correspond to the [CMake FindBoost options](https://cmake.org/cmake/help/latest/module/FindBoost.html) by the same names.
|
||||
|
||||
* `--boost-root`
|
||||
* `--boost-dir`
|
||||
* `--boost-include-dir`
|
||||
* `--boost-librarydir`
|
||||
|
||||
The following options correspond to the [CMake FindOpenCL options](https://cmake.org/cmake/help/latest/module/FindOpenCL.html) by the same names.
|
||||
|
||||
* `--opencl-include-dir`
|
||||
* `--opencl-library`
|
||||
|
||||
### Installing Precompiled Binaries
|
||||
|
||||
Precompiled binaries for Mac and Windows are prepared by CRAN a few days after each release to CRAN. They can be installed with the following R code.
|
||||
|
||||
```r
|
||||
install.packages(
|
||||
"lightgbm"
|
||||
, type = "both"
|
||||
, repos = "https://cran.r-project.org"
|
||||
)
|
||||
```
|
||||
|
||||
These packages do not require compilation, so they will be faster and easier to install than packages that are built from source.
|
||||
|
||||
CRAN does not prepare precompiled binaries for Linux, and as of this writing neither does this project.
|
||||
|
||||
### Installing from a Pre-compiled lib_lightgbm <a id="lib_lightgbm"></a>
|
||||
|
||||
Previous versions of LightGBM offered the ability to first compile the C++ library (`lib_lightgbm.{dll,dylib,so}`) and then build an R-package that wraps it.
|
||||
|
||||
As of version 3.0.0, this is no longer supported. If building from source is difficult for you, please [open an issue](https://github.com/lightgbm-org/LightGBM/issues).
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Please visit [demo](https://github.com/lightgbm-org/LightGBM/tree/main/R-package/demo):
|
||||
|
||||
* [Basic walkthrough of wrappers](https://github.com/lightgbm-org/LightGBM/blob/main/R-package/demo/basic_walkthrough.R)
|
||||
* [Boosting from existing prediction](https://github.com/lightgbm-org/LightGBM/blob/main/R-package/demo/boost_from_prediction.R)
|
||||
* [Early Stopping](https://github.com/lightgbm-org/LightGBM/blob/main/R-package/demo/early_stopping.R)
|
||||
* [Cross Validation](https://github.com/lightgbm-org/LightGBM/blob/main/R-package/demo/cross_validation.R)
|
||||
* [Multiclass Training/Prediction](https://github.com/lightgbm-org/LightGBM/blob/main/R-package/demo/multiclass.R)
|
||||
* [Leaf (in)Stability](https://github.com/lightgbm-org/LightGBM/blob/main/R-package/demo/leaf_stability.R)
|
||||
* [Weight-Parameter Adjustment Relationship](https://github.com/lightgbm-org/LightGBM/blob/main/R-package/demo/weight_param.R)
|
||||
|
||||
Testing
|
||||
-------
|
||||
|
||||
The R-package's unit tests are run automatically on every commit, via integrations like [GitHub Actions](https://github.com/lightgbm-org/LightGBM/actions). Adding new tests in `R-package/tests/testthat` is a valuable way to improve the reliability of the R-package.
|
||||
|
||||
### Running the Tests
|
||||
|
||||
While developing the R-package, run the code below to run the unit tests.
|
||||
|
||||
```shell
|
||||
sh build-cran-package.sh \
|
||||
--no-build-vignettes
|
||||
|
||||
R CMD INSTALL --with-keep.source lightgbm*.tar.gz
|
||||
cd R-package/tests
|
||||
Rscript testthat.R
|
||||
```
|
||||
|
||||
To run the tests with more verbose logs, set environment variable `LIGHTGBM_TEST_VERBOSITY` to a valid value for parameter [`verbosity`](https://lightgbm.readthedocs.io/en/latest/Parameters.html#verbosity).
|
||||
|
||||
```shell
|
||||
export LIGHTGBM_TEST_VERBOSITY=1
|
||||
cd R-package/tests
|
||||
Rscript testthat.R
|
||||
```
|
||||
|
||||
### Code Coverage
|
||||
|
||||
When adding tests, you may want to use test coverage to identify untested areas and to check if the tests you've added are covering all branches of the intended code.
|
||||
|
||||
The example below shows how to generate code coverage for the R-package on a macOS or Linux setup. To adjust for your environment, refer to [the customization step described above](#custom-installation-linux-mac).
|
||||
|
||||
```shell
|
||||
# Install
|
||||
sh build-cran-package.sh \
|
||||
--no-build-vignettes
|
||||
|
||||
# Get coverage
|
||||
Rscript -e " \
|
||||
library(covr);
|
||||
coverage <- covr::package_coverage('./lightgbm_r', type = 'tests', quiet = FALSE);
|
||||
print(coverage);
|
||||
covr::report(coverage, file = file.path(getwd(), 'coverage.html'), browse = TRUE);
|
||||
"
|
||||
```
|
||||
|
||||
Updating Documentation
|
||||
----------------------
|
||||
|
||||
The R-package uses [`{roxygen2}`](https://CRAN.R-project.org/package=roxygen2) to generate its documentation.
|
||||
The generated `DESCRIPTION`, `NAMESPACE`, and `man/` files are checked into source control.
|
||||
To regenerate those files, run the following.
|
||||
|
||||
```shell
|
||||
Rscript \
|
||||
--vanilla \
|
||||
-e "install.packages('roxygen2', repos = 'https://cran.rstudio.com')"
|
||||
|
||||
sh build-cran-package.sh --no-build-vignettes
|
||||
R CMD INSTALL \
|
||||
--with-keep.source \
|
||||
./lightgbm_*.tar.gz
|
||||
|
||||
cd R-package
|
||||
Rscript \
|
||||
--vanilla \
|
||||
-e "roxygen2::roxygenize(load = 'installed')"
|
||||
```
|
||||
|
||||
Preparing a CRAN Package
|
||||
------------------------
|
||||
|
||||
This section is primarily for maintainers, but may help users and contributors to understand the structure of the R-package.
|
||||
|
||||
Most of `LightGBM` uses `CMake` to handle tasks like setting compiler and linker flags, including header file locations, and linking to other libraries. Because CRAN packages typically do not assume the presence of `CMake`, the R-package uses an alternative method that is in the CRAN-supported toolchain for building R packages with C++ code: `Autoconf`.
|
||||
|
||||
For more information on this approach, see ["Writing R Extensions"](https://cran.r-project.org/doc/manuals/r-release/R-exts.html#Configure-and-cleanup).
|
||||
|
||||
### Build a CRAN Package
|
||||
|
||||
From the root of the repository, run the following.
|
||||
|
||||
```shell
|
||||
git submodule update --init --recursive
|
||||
sh build-cran-package.sh
|
||||
```
|
||||
|
||||
This will create a file `lightgbm_${VERSION}.tar.gz`, where `VERSION` is the version of `LightGBM`.
|
||||
|
||||
That script supports the following command-line options:
|
||||
|
||||
- `--no-build-vignettes`: Skip building vignettes.
|
||||
- `--r-executable=[path-to-executable]`: Use an alternative build of R.
|
||||
|
||||
### Standard Installation from CRAN Package
|
||||
|
||||
After building the package, install it with a command like the following:
|
||||
|
||||
```shell
|
||||
R CMD install lightgbm_*.tar.gz
|
||||
```
|
||||
|
||||
### Changing the CRAN Package
|
||||
|
||||
A lot of details are handled automatically by `R CMD build` and `R CMD install`, so it can be difficult to understand how the files in the R-package are related to each other. An extensive treatment of those details is available in ["Writing R Extensions"](https://cran.r-project.org/doc/manuals/r-release/R-exts.html).
|
||||
|
||||
This section briefly explains the key files for building a CRAN package. To update the package, edit the files relevant to your change and re-run the steps in [Build a CRAN Package](#build-a-cran-package).
|
||||
|
||||
**Linux or Mac**
|
||||
|
||||
At build time, `configure` will be run and used to create a file `Makevars`, using `Makevars.in` as a template.
|
||||
|
||||
1. Edit `configure.ac`.
|
||||
2. Create `configure` with `autoconf`. Do not edit it by hand. This file must be generated on Ubuntu 22.04.
|
||||
|
||||
If you have an Ubuntu 22.04 environment available, run the provided script from the root of the `LightGBM` repository.
|
||||
|
||||
```shell
|
||||
./R-package/recreate-configure.sh
|
||||
```
|
||||
|
||||
If you do not have easy access to an Ubuntu 22.04 environment, the `configure` script can be generated using Docker by running the code below from the root of this repo.
|
||||
|
||||
```shell
|
||||
docker run \
|
||||
--rm \
|
||||
-v $(pwd):/opt/LightGBM \
|
||||
-w /opt/LightGBM \
|
||||
ubuntu:22.04 \
|
||||
./R-package/recreate-configure.sh
|
||||
```
|
||||
|
||||
The version of `autoconf` used by this project is stored in `R-package/AUTOCONF_UBUNTU_VERSION`. To update that version, update that file and run the commands above. To see available versions, see https://packages.ubuntu.com/search?keywords=autoconf.
|
||||
|
||||
3. Edit `src/Makevars.in`.
|
||||
|
||||
Alternatively, GitHub Actions can re-generate this file for you.
|
||||
|
||||
1. navigate to https://github.com/lightgbm-org/LightGBM/actions/workflows/r_configure.yml
|
||||
2. click "Run workflow" (drop-down)
|
||||
3. enter the branch from the pull request for the `pr-branch` input
|
||||
4. click "Run workflow" (button)
|
||||
|
||||
**Configuring for Windows**
|
||||
|
||||
At build time, `configure.win` will be run and used to create a file `Makevars.win`, using `Makevars.win.in` as a template.
|
||||
|
||||
1. Edit `configure.win` directly.
|
||||
2. Edit `src/Makevars.win.in`.
|
||||
|
||||
### Testing the CRAN Package
|
||||
|
||||
`{lightgbm}` is tested automatically on every commit, across many combinations of operating system, R version, and compiler. This section describes how to test the package locally while you are developing.
|
||||
|
||||
#### Windows, Mac, and Linux
|
||||
|
||||
```shell
|
||||
sh build-cran-package.sh
|
||||
R CMD check --as-cran lightgbm_*.tar.gz
|
||||
```
|
||||
|
||||
#### <a id="UBSAN"></a>ASAN and UBSAN
|
||||
|
||||
All packages uploaded to CRAN must pass builds using `gcc` and `clang`, instrumented with two sanitizers: the Address Sanitizer (ASAN) and the Undefined Behavior Sanitizer (UBSAN).
|
||||
|
||||
For more background, see
|
||||
|
||||
* [this blog post](https://dirk.eddelbuettel.com/code/sanitizers.html)
|
||||
* [top-level CRAN documentation on these checks](https://cran.r-project.org/web/checks/check_issue_kinds.html)
|
||||
* [CRAN's configuration of these checks](https://www.stats.ox.ac.uk/pub/bdr/memtests/README.txt)
|
||||
|
||||
You can replicate these checks locally using Docker.
|
||||
For more information on the image used for testing, see https://github.com/wch/r-debug.
|
||||
|
||||
In the code below, environment variable `R_CUSTOMIZATION` should be set to one of two values.
|
||||
|
||||
* `"san"` = replicates CRAN's `gcc-ASAN` and `gcc-UBSAN` checks
|
||||
* `"csan"` = replicates CRAN's `clang-ASAN` and `clang-UBSAN` checks
|
||||
|
||||
```shell
|
||||
docker run \
|
||||
--rm \
|
||||
-it \
|
||||
-v $(pwd):/opt/LightGBM \
|
||||
-w /opt/LightGBM \
|
||||
--env R_CUSTOMIZATION=san \
|
||||
wch1/r-debug:latest \
|
||||
/bin/bash
|
||||
|
||||
# install dependencies
|
||||
RDscript${R_CUSTOMIZATION} \
|
||||
-e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())"
|
||||
|
||||
# install lightgbm
|
||||
sh build-cran-package.sh --r-executable=RD${R_CUSTOMIZATION}
|
||||
RD${R_CUSTOMIZATION} \
|
||||
CMD INSTALL lightgbm_*.tar.gz
|
||||
|
||||
# run tests
|
||||
cd R-package/tests
|
||||
rm -f ./tests.log
|
||||
RDscript${R_CUSTOMIZATION} testthat.R >> tests.log 2>&1
|
||||
|
||||
# check that tests passed
|
||||
echo "test exit code: $?"
|
||||
tail -300 ./tests.log
|
||||
```
|
||||
|
||||
#### Valgrind
|
||||
|
||||
All packages uploaded to CRAN must be built and tested without raising any issues from `valgrind`. `valgrind` is a profiler that can catch serious issues like memory leaks and illegal writes. For more information, see [this blog post](https://reside-ic.github.io/blog/debugging-and-fixing-crans-additional-checks-errors/).
|
||||
|
||||
You can replicate these checks locally using Docker. Note that instrumented versions of R built to use `valgrind` run much slower, and these tests may take as long as 20 minutes to run.
|
||||
|
||||
```shell
|
||||
docker run \
|
||||
--rm \
|
||||
-v $(pwd):/opt/LightGBM \
|
||||
-w /opt/LightGBM \
|
||||
-it \
|
||||
wch1/r-debug
|
||||
|
||||
RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())"
|
||||
|
||||
sh build-cran-package.sh \
|
||||
--r-executable=RDvalgrind
|
||||
|
||||
RDvalgrind CMD INSTALL \
|
||||
--preclean \
|
||||
--install-tests \
|
||||
lightgbm_*.tar.gz
|
||||
|
||||
cd R-package/tests
|
||||
|
||||
RDvalgrind \
|
||||
--no-readline \
|
||||
--vanilla \
|
||||
-d "valgrind --tool=memcheck --leak-check=full --track-origins=yes" \
|
||||
-f testthat.R \
|
||||
2>&1 \
|
||||
| tee out.log \
|
||||
| cat
|
||||
```
|
||||
|
||||
These tests can also be triggered on a pull request branch, using GitHub Actions.
|
||||
|
||||
1. navigate to https://github.com/lightgbm-org/LightGBM/actions/workflows/r_valgrind.yml
|
||||
2. click "Run workflow" (drop-down)
|
||||
3. enter the branch from the pull request for the `pr-branch` input
|
||||
4. enter the pull request ID for the `pr-number` input
|
||||
5. click "Run workflow" (button)
|
||||
|
||||
Or by using the GitHub CLI, using a command similar to this:
|
||||
|
||||
```shell
|
||||
gh workflow run \
|
||||
--repo lightgbm-org/LightGBM \
|
||||
r_valgrind.yml \
|
||||
-f pr-branch=ci/fix-rerun-workflow \
|
||||
-f pr-number=7072
|
||||
```
|
||||
|
||||
Known Issues
|
||||
------------
|
||||
|
||||
For information about known issues with the R-package, see the [R-package section of LightGBM's main FAQ page](https://lightgbm.readthedocs.io/en/latest/FAQ.html#r-package).
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
rm -f aclocal.m4
|
||||
rm -rf ./autom4te.cache
|
||||
rm -f config.log
|
||||
rm -f config.status
|
||||
rm -f src/Makevars
|
||||
+3072
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,172 @@
|
||||
### configure.ac -*- Autoconf -*-
|
||||
# Template used by Autoconf to generate 'configure' script. For more see:
|
||||
# * https://unconj.ca/blog/an-autoconf-primer-for-r-package-authors.html
|
||||
# * https://cran.r-project.org/doc/manuals/r-release/R-exts.html#Configure-and-cleanup
|
||||
|
||||
AC_PREREQ(2.69)
|
||||
AC_INIT([lightgbm], [~~VERSION~~], [], [lightgbm], [])
|
||||
|
||||
###########################
|
||||
# find compiler and flags #
|
||||
###########################
|
||||
|
||||
AC_MSG_CHECKING([location of R])
|
||||
AC_MSG_RESULT([${R_HOME}])
|
||||
|
||||
# set up CPP flags
|
||||
# find the compiler and compiler flags used by R.
|
||||
: ${R_HOME=`R HOME`}
|
||||
if test -z "${R_HOME}"; then
|
||||
echo "could not determine R_HOME"
|
||||
exit 1
|
||||
fi
|
||||
CXX17=`"${R_HOME}/bin/R" CMD config CXX17`
|
||||
CXX17STD=`"${R_HOME}/bin/R" CMD config CXX17STD`
|
||||
CXX="${CXX17} ${CXX17STD}"
|
||||
CPPFLAGS=`"${R_HOME}/bin/R" CMD config CPPFLAGS`
|
||||
CXXFLAGS=`"${R_HOME}/bin/R" CMD config CXX17FLAGS`
|
||||
LDFLAGS=`"${R_HOME}/bin/R" CMD config LDFLAGS`
|
||||
AC_LANG(C++)
|
||||
|
||||
# LightGBM-specific flags
|
||||
LGB_CPPFLAGS=""
|
||||
|
||||
#########
|
||||
# Eigen #
|
||||
#########
|
||||
|
||||
LGB_CPPFLAGS="${LGB_CPPFLAGS} -DEIGEN_MPL2_ONLY -DEIGEN_DONT_PARALLELIZE"
|
||||
|
||||
###############
|
||||
# MM_PREFETCH #
|
||||
###############
|
||||
|
||||
AC_MSG_CHECKING([whether MM_PREFETCH works])
|
||||
ac_mmprefetch=no
|
||||
AC_LANG_CONFTEST(
|
||||
[
|
||||
AC_LANG_PROGRAM(
|
||||
[[
|
||||
#include <xmmintrin.h>
|
||||
]],
|
||||
[[
|
||||
int a = 0;
|
||||
_mm_prefetch(&a, _MM_HINT_NTA);
|
||||
return 0;
|
||||
]]
|
||||
)
|
||||
]
|
||||
)
|
||||
${CXX} ${CPPFLAGS} ${CXXFLAGS} -o conftest conftest.cpp 2>/dev/null && ./conftest && ac_mmprefetch=yes
|
||||
AC_MSG_RESULT([${ac_mmprefetch}])
|
||||
if test "${ac_mmprefetch}" = yes; then
|
||||
LGB_CPPFLAGS="${LGB_CPPFLAGS} -DMM_PREFETCH=1"
|
||||
fi
|
||||
|
||||
############
|
||||
# MM_ALLOC #
|
||||
############
|
||||
|
||||
AC_MSG_CHECKING([whether MM_MALLOC works])
|
||||
ac_mm_malloc=no
|
||||
AC_LANG_CONFTEST(
|
||||
[
|
||||
AC_LANG_PROGRAM(
|
||||
[[
|
||||
#include <mm_malloc.h>
|
||||
]],
|
||||
[[
|
||||
char *a = (char*)_mm_malloc(8, 16);
|
||||
_mm_free(a);
|
||||
return 0;
|
||||
]]
|
||||
)
|
||||
]
|
||||
)
|
||||
${CXX} ${CPPFLAGS} ${CXXFLAGS} -o conftest conftest.cpp 2>/dev/null && ./conftest && ac_mm_malloc=yes
|
||||
AC_MSG_RESULT([${ac_mm_malloc}])
|
||||
if test "${ac_mm_malloc}" = yes; then
|
||||
LGB_CPPFLAGS="${LGB_CPPFLAGS} -DMM_MALLOC=1"
|
||||
fi
|
||||
|
||||
##########
|
||||
# OpenMP #
|
||||
##########
|
||||
|
||||
OPENMP_CXXFLAGS=""
|
||||
|
||||
if test `uname -s` = "Linux"
|
||||
then
|
||||
OPENMP_CXXFLAGS="\$(SHLIB_OPENMP_CXXFLAGS)"
|
||||
fi
|
||||
|
||||
if test `uname -s` = "Darwin"
|
||||
then
|
||||
OPENMP_CXXFLAGS='-Xclang -fopenmp'
|
||||
OPENMP_LIB='-lomp'
|
||||
|
||||
# libomp 15.0+ from brew is keg-only (i.e. not symlinked into the standard paths search by the linker),
|
||||
# so need to search in other locations.
|
||||
# See https://github.com/Homebrew/homebrew-core/issues/112107#issuecomment-1278042927.
|
||||
#
|
||||
# If Homebrew is found and libomp was installed with it, this code adds the necessary
|
||||
# flags for the compiler to find libomp headers and for the linker to find libomp.dylib.
|
||||
HOMEBREW_LIBOMP_PREFIX=""
|
||||
if command -v brew >/dev/null 2>&1; then
|
||||
ac_brew_openmp=no
|
||||
AC_MSG_CHECKING([whether OpenMP was installed via Homebrew])
|
||||
brew --prefix libomp >/dev/null 2>&1 && ac_brew_openmp=yes
|
||||
AC_MSG_RESULT([${ac_brew_openmp}])
|
||||
if test "${ac_brew_openmp}" = yes; then
|
||||
HOMEBREW_LIBOMP_PREFIX=`brew --prefix libomp`
|
||||
OPENMP_CXXFLAGS="${OPENMP_CXXFLAGS} -I${HOMEBREW_LIBOMP_PREFIX}/include"
|
||||
OPENMP_LIB="${OPENMP_LIB} -L${HOMEBREW_LIBOMP_PREFIX}/lib"
|
||||
fi
|
||||
fi
|
||||
ac_pkg_openmp=no
|
||||
AC_MSG_CHECKING([whether OpenMP will work in a package])
|
||||
AC_LANG_CONFTEST(
|
||||
[
|
||||
AC_LANG_PROGRAM(
|
||||
[[
|
||||
#include <omp.h>
|
||||
]],
|
||||
[[
|
||||
return (omp_get_max_threads() <= 1);
|
||||
]]
|
||||
)
|
||||
]
|
||||
)
|
||||
${CXX} ${CPPFLAGS} ${CXXFLAGS} ${LDFLAGS} ${OPENMP_CXXFLAGS} ${OPENMP_LIB} -o conftest conftest.cpp 2>/dev/null && ./conftest && ac_pkg_openmp=yes
|
||||
|
||||
# -Xclang is not portable (it is clang-specific)
|
||||
# if compilation above failed, try without that flag
|
||||
if test "${ac_pkg_openmp}" = no; then
|
||||
if test -f "./conftest"; then
|
||||
rm ./conftest
|
||||
fi
|
||||
OPENMP_CXXFLAGS="-fopenmp"
|
||||
${CXX} ${CPPFLAGS} ${CXXFLAGS} ${LDFLAGS} ${OPENMP_CXXFLAGS} ${OPENMP_LIB} -o conftest conftest.cpp 2>/dev/null && ./conftest && ac_pkg_openmp=yes
|
||||
fi
|
||||
|
||||
AC_MSG_RESULT([${ac_pkg_openmp}])
|
||||
if test "${ac_pkg_openmp}" = no; then
|
||||
OPENMP_CXXFLAGS=''
|
||||
OPENMP_LIB=''
|
||||
echo '***********************************************************************************************'
|
||||
echo ' OpenMP is unavailable on this macOS system. LightGBM code will run single-threaded as a result.'
|
||||
echo ' To use all CPU cores for training jobs, you should install OpenMP by running'
|
||||
echo ''
|
||||
echo ' brew install libomp'
|
||||
echo '***********************************************************************************************'
|
||||
fi
|
||||
fi
|
||||
|
||||
# substitute variables from this script into Makevars.in
|
||||
AC_SUBST(OPENMP_CXXFLAGS)
|
||||
AC_SUBST(OPENMP_LIB)
|
||||
AC_SUBST(LGB_CPPFLAGS)
|
||||
AC_CONFIG_FILES([src/Makevars])
|
||||
|
||||
# write out Autoconf output
|
||||
AC_OUTPUT
|
||||
Executable
+102
@@ -0,0 +1,102 @@
|
||||
# Script used to generate `Makevars.win` from `Makevars.win.in`
|
||||
# on Windows
|
||||
|
||||
###########################
|
||||
# find compiler and flags #
|
||||
###########################
|
||||
|
||||
R_EXE="${R_HOME}/bin${R_ARCH_BIN}/R"
|
||||
|
||||
CXX17=`"${R_EXE}" CMD config CXX17`
|
||||
CXX17STD=`"${R_EXE}" CMD config CXX17STD`
|
||||
CXX="${CXX17} ${CXX17STD}"
|
||||
CXXFLAGS=`"${R_EXE}" CMD config CXX17FLAGS`
|
||||
CPPFLAGS=`"${R_EXE}" CMD config CPPFLAGS`
|
||||
|
||||
# LightGBM-specific flags
|
||||
LGB_CPPFLAGS=""
|
||||
|
||||
#########
|
||||
# Eigen #
|
||||
#########
|
||||
|
||||
LGB_CPPFLAGS="${LGB_CPPFLAGS} -DEIGEN_MPL2_ONLY -DEIGEN_DONT_PARALLELIZE"
|
||||
|
||||
###############
|
||||
# MM_PREFETCH #
|
||||
###############
|
||||
|
||||
ac_mm_prefetch="no"
|
||||
|
||||
cat > conftest.cpp <<EOL
|
||||
#include <xmmintrin.h>
|
||||
int main() {
|
||||
int a = 0;
|
||||
_mm_prefetch(&a, _MM_HINT_NTA);
|
||||
return 0;
|
||||
}
|
||||
EOL
|
||||
|
||||
${CXX} ${CXXFLAGS} ${CPPFLAGS} -o conftest conftest.cpp 2>/dev/null && ./conftest && ac_mm_prefetch="yes"
|
||||
rm -f ./conftest
|
||||
rm -f ./conftest.cpp
|
||||
echo "checking whether MM_PREFETCH works...${ac_mm_prefetch}"
|
||||
|
||||
if test "${ac_mm_prefetch}" = "yes";
|
||||
then
|
||||
LGB_CPPFLAGS="${LGB_CPPFLAGS} -DMM_PREFETCH=1"
|
||||
fi
|
||||
|
||||
############
|
||||
# MM_ALLOC #
|
||||
############
|
||||
ac_mm_malloc="no"
|
||||
|
||||
cat > conftest.cpp <<EOL
|
||||
#include <mm_malloc.h>
|
||||
int main() {
|
||||
char *a = (char*)_mm_malloc(8, 16);
|
||||
_mm_free(a);
|
||||
return 0;
|
||||
}
|
||||
EOL
|
||||
|
||||
${CXX} ${CXXFLAGS} ${CPPFLAGS} -o conftest conftest.cpp 2>/dev/null && ./conftest && ac_mm_malloc="yes"
|
||||
rm -f ./conftest
|
||||
rm -f ./conftest.cpp
|
||||
echo "checking whether MM_MALLOC works...${ac_mm_malloc}"
|
||||
|
||||
if test "${ac_mm_malloc}" = "yes";
|
||||
then
|
||||
LGB_CPPFLAGS="${LGB_CPPFLAGS} -DMM_MALLOC=1"
|
||||
fi
|
||||
|
||||
#############
|
||||
# INET_PTON #
|
||||
#############
|
||||
|
||||
ac_inet_pton="no"
|
||||
|
||||
cat > conftest.cpp <<EOL
|
||||
#include <ws2tcpip.h>
|
||||
int main() {
|
||||
int (*fptr)(int, const char*, void*);
|
||||
fptr = &inet_pton;
|
||||
return 0;
|
||||
}
|
||||
EOL
|
||||
|
||||
${CXX} ${CXXFLAGS} ${CPPFLAGS} -o conftest conftest.cpp 2>/dev/null && ./conftest && ac_inet_pton="yes"
|
||||
rm -f ./conftest
|
||||
rm -f ./conftest.cpp
|
||||
echo "checking whether INET_PTON works...${ac_inet_pton}"
|
||||
|
||||
if test "${ac_inet_pton}" = "yes";
|
||||
then
|
||||
LGB_CPPFLAGS="${LGB_CPPFLAGS} -DWIN_HAS_INET_PTON=1"
|
||||
fi
|
||||
|
||||
# Generate Makevars.win from Makevars.win.in
|
||||
sed -e \
|
||||
"s/@LGB_CPPFLAGS@/$LGB_CPPFLAGS/" \
|
||||
< src/Makevars.win.in > src/Makevars.win
|
||||
@@ -0,0 +1,962 @@
|
||||
# CRAN Submission History
|
||||
|
||||
## v4.6.0 - Submission 1 - (February 13, 2025)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
This release fixed several issues reported by CRAN.
|
||||
Bashisms in `configure`
|
||||
|
||||
```text
|
||||
possible bashism in configure.ac line 63 (should be VAR="${VAR}foo"):
|
||||
LGB_CPPFLAGS+=" -DMM_PREFETCH=1"
|
||||
possible bashism in configure.ac line 89 (should be VAR="${VAR}foo"):
|
||||
LGB_CPPFLAGS+=" -DMM_MALLOC=1"
|
||||
```
|
||||
|
||||
Compilation errors on GCC 15.
|
||||
|
||||
```text
|
||||
io/json11.cpp:97:28: error: 'uint8_t' does not name a type
|
||||
97 | } else if (static_cast<uint8_t>(ch) == 0xe2 &&
|
||||
| ^~~~~~~
|
||||
io/json11.cpp:97:28: note: 'uint8_t' is defined in header '<cstdint>'; this is probably fixable by adding '#include <cstdint>'
|
||||
io/json11.cpp:98:28: error: 'uint8_t' does not name a type
|
||||
98 | static_cast<uint8_t>(value[i + 1]) == 0x80 &&
|
||||
```
|
||||
|
||||
This release contains fixes for those issues.
|
||||
|
||||
## v4.5.0 - Submission 1 - (July 25, 2024)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
This release was a response to a request from CRAN.
|
||||
On July 4, 2024, CRAN notified us that the following compiler warnings raised by `gcc` 14 needed to be fixed by August 3, 2024.
|
||||
|
||||
```text
|
||||
Result: WARN
|
||||
Found the following significant warnings:
|
||||
io/dense_bin.hpp:617:27: warning: template-id not allowed for constructor in C++20 [-Wtemplate-id-cdtor]
|
||||
io/multi_val_dense_bin.hpp:346:26: warning: template-id not allowed for constructor in C++20 [-Wtemplate-id-cdtor]
|
||||
io/multi_val_sparse_bin.hpp:433:36: warning: template-id not allowed for constructor in C++20 [-Wtemplate-id-cdtor]
|
||||
io/sparse_bin.hpp:785:19: warning: template-id not allowed for constructor in C++20 [-Wtemplate-id-cdtor]
|
||||
See ‘/data/gannet/ripley/R/packages/tests-devel/lightgbm.Rcheck/00install.out’ for details.
|
||||
```
|
||||
|
||||
This release contains fixes for those issues.
|
||||
|
||||
## v4.4.0 - Submission 1 - (June 14, 2024)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
This was a standard release of `{lightgbm}`, not intended to fix any particular R-specific issues.
|
||||
|
||||
## v4.3.0 - Submission 1 - (January 18, 2024)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
This submission was put up in response to CRAN saying the package would be archived if the following
|
||||
warning was not fixed within 14 days.
|
||||
|
||||
```text
|
||||
/usr/local/clang-trunk/bin/../include/c++/v1/__fwd/string_view.h:22:41:
|
||||
warning: 'char_traits<fmt::detail::char8_type>' is deprecated:
|
||||
char_traits<T> for T not equal to char, wchar_t, char8_t, char16_t or char32_t is non-standard and is provided for a temporary period.
|
||||
It will be removed in LLVM 19, so please migrate off of it. [-Wdeprecated-declarations]
|
||||
```
|
||||
|
||||
See https://github.com/lightgbm-org/LightGBM/issues/6264.
|
||||
|
||||
## v4.2.0 - Submission 1 - (December 7, 2023)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
This submission included many changes from the last 2 years, as well as fixes for a warning
|
||||
CRAN said could cause the package to be archived: https://github.com/lightgbm-org/LightGBM/issues/6221.
|
||||
|
||||
## v4.1.0 - not submitted
|
||||
|
||||
v4.1.0 was not submitted to CRAN, because https://github.com/lightgbm-org/LightGBM/issues/5987 had not been resolved.
|
||||
|
||||
## v4.0.0 - Submission 2 - (July 19, 2023)
|
||||
|
||||
### CRAN response
|
||||
|
||||
> Dear maintainer,
|
||||
> package lightgbm_4.0.0.tar.gz does not pass the incoming checks automatically.
|
||||
|
||||
The logs linked from those messages showed one issue remaining on Debian (0 on Windows).
|
||||
|
||||
```text
|
||||
* checking examples ... [7s/4s] NOTE
|
||||
Examples with CPU time > 2.5 times elapsed time
|
||||
user system elapsed ratio
|
||||
lgb.restore_handle 1.206 0.085 0.128 10.08
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Chose to document the issue and need for a fix in https://github.com/lightgbm-org/LightGBM/issues/5987, but not resubmit,
|
||||
to avoid annoying CRAN maintainers.
|
||||
|
||||
## v4.0.0 - Submission 1 - (July 16, 2023)
|
||||
|
||||
### CRAN response
|
||||
|
||||
> Dear maintainer,
|
||||
> package lightgbm_4.0.0.tar.gz does not pass the incoming checks automatically.
|
||||
|
||||
The logs linked from those messages showed the following issues from `R CMD check`.
|
||||
|
||||
```text
|
||||
* checking S3 generic/method consistency ... NOTE
|
||||
Mismatches for apparent methods not registered:
|
||||
merge:
|
||||
function(x, y, ...)
|
||||
merge.eval.string:
|
||||
function(env)
|
||||
|
||||
format:
|
||||
function(x, ...)
|
||||
format.eval.string:
|
||||
function(eval_res, eval_err)
|
||||
See section 'Registering S3 methods' in the 'Writing R Extensions'
|
||||
manual.
|
||||
```
|
||||
|
||||
```text
|
||||
* checking examples ... [8s/4s] NOTE
|
||||
Examples with CPU time > 2.5 times elapsed time
|
||||
user system elapsed ratio
|
||||
lgb.restore_handle 1.819 0.128 0.165 11.8
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Attempted to fix these with https://github.com/lightgbm-org/LightGBM/pull/5988 and resubmitted.
|
||||
|
||||
## v3.3.5 - Submission 2 - (January 16, 2023)
|
||||
|
||||
### CRAN response
|
||||
|
||||
> Reason was
|
||||
>
|
||||
> Flavor: r-devel-windows-x86_64
|
||||
> Check: OOverall checktime, Result: NOTE
|
||||
> Overall checktime 14 min > 10 min
|
||||
>
|
||||
> but the maintainer cannot do much to reduce this, so I triggered revdep checks now.
|
||||
> Please reply to the archival message in case the issue is not fixable easily.
|
||||
>
|
||||
> Best,
|
||||
> Uwe Ligges
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
This was technically not a "resubmission".
|
||||
We asked CRAN why the first v3.3.5 submission had been archived, and they responded with the response above... and then v3.3.5 passed all checks with no further work from LightGBM maintainers.
|
||||
|
||||
## v3.3.5 - Submission 1 - (January 11, 2023)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Archived without a response.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Submitted with the following comment.
|
||||
|
||||
> This submission contains {lightgbm} 3.3.5
|
||||
|
||||
> Per CRAN's policies, I am submitting it on behalf of the project's maintainer (Yu Shi), with his permission.
|
||||
|
||||
> This submission includes patches to address the following warnings observed on the fedora and debian CRAN checks.
|
||||
|
||||
> Found the following significant warnings:
|
||||
> io/json11.cpp:207:47: warning: unqualified call to 'std::move' [-Wunqualified-std-cast-call]
|
||||
> io/json11.cpp:216:51: warning: unqualified call to 'std::move' [-Wunqualified-std-cast-call]
|
||||
> io/json11.cpp:225:53: warning: unqualified call to 'std::move' [-Wunqualified-std-cast-call]
|
||||
> io/json11.cpp:268:60: warning: unqualified call to 'std::move' [-Wunqualified-std-cast-call]
|
||||
> io/json11.cpp:272:36: warning: unqualified call to 'std::move' [-Wunqualified-std-cast-call]
|
||||
> io/json11.cpp:276:37: warning: unqualified call to 'std::move' [-Wunqualified-std-cast-call]
|
||||
> io/json11.cpp:381:41: warning: unqualified call to 'std::move' [-Wunqualified-std-cast-call]
|
||||
> io/json11.cpp:150:39: warning: unqualified call to 'std::move' [-Wunqualified-std-cast-call]
|
||||
|
||||
Thank you very much for your time and consideration.
|
||||
|
||||
## v3.3.4 - Submission 1 - (December 15, 2022)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Submitted with the following comment:
|
||||
|
||||
> This submission contains {lightgbm} 3.3.4
|
||||
|
||||
> Per CRAN's policies, I am submitting it on behalf of the project's maintainer (Yu Shi), with his permission.
|
||||
|
||||
> This submission includes patches to address the following warnings observed on the fedora and debian CRAN checks.
|
||||
>
|
||||
> Compiled code should not call entry points which might terminate R nor write to stdout/stderr instead of to the console, nor use Fortran I/O nor system RNGs nor [v]sprintf.
|
||||
|
||||
> Thank you very much for your time and consideration.
|
||||
|
||||
## v3.3.3 - Submission 1 - (October 10, 2022)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Submitted with the following comment:
|
||||
|
||||
> This submission contains {lightgbm} 3.3.3.
|
||||
|
||||
> Per CRAN's policies, I am submitting on it on behalf of the project's maintainer (Yu Shi), with his permission (https://github.com/lightgbm-org/LightGBM/pull/5525).
|
||||
|
||||
> This submission includes two patches:
|
||||
> * a change to testing to avoid a failed test related to non-ASCII strings on the `r-devel-linux-x86_64-debian-clang` check flavor (https://github.com/lightgbm-org/LightGBM/pull/5526)
|
||||
> * modifications to allow compatibility with the RTools42 build toolchain (https://github.com/lightgbm-org/LightGBM/pull/5503)
|
||||
|
||||
> Thank you very much for your time and consideration.
|
||||
|
||||
## v3.3.2 - Submission 1 - (January 7, 2022)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN on January 14, 2022.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
In this submission, we uploaded a patch that CRAN stuff provided us via e-mail. The full text of the e-mail from CRAN:
|
||||
|
||||
```text
|
||||
Dear maintainers,
|
||||
|
||||
This concerns the CRAN packages
|
||||
|
||||
Cairo cepreader gpboost httpuv ipaddress lightgbm proj4 prophet
|
||||
RcppCWB RcppParallel RDieHarder re2 redux rgeolocate RGtk2 tth
|
||||
udunits2 unrtf
|
||||
|
||||
maintained by one of you:
|
||||
|
||||
Andreas Blaette andreas.blaette@uni-due.de: RcppCWB
|
||||
David Hall david.hall.physics@gmail.com: ipaddress
|
||||
Dirk Eddelbuettel edd@debian.org: RDieHarder
|
||||
Fabio Sigrist fabiosigrist@gmail.com: gpboost
|
||||
Friedrich Leisch Friedrich.Leisch@R-project.org: tth
|
||||
Girish Palya girishji@gmail.com: re2
|
||||
James Hiebert hiebert@uvic.ca: udunits2
|
||||
Jari Oksanen jhoksane@gmail.com: cepreader
|
||||
Kevin Ushey kevin@rstudio.com: RcppParallel
|
||||
ORPHANED: RGtk2
|
||||
Os Keyes ironholds@gmail.com: rgeolocate
|
||||
Rich FitzJohn rich.fitzjohn@gmail.com: redux
|
||||
Sean Taylor sjtz@pm.me: prophet
|
||||
Simon Urbanek simon.urbanek@r-project.org: proj4
|
||||
Simon Urbanek Simon.Urbanek@r-project.org: Cairo
|
||||
Winston Chang winston@rstudio.com: httpuv
|
||||
Yu Shi yushi2@microsoft.com: lightgbm
|
||||
|
||||
your packages need to be updated for R-devel/R 4.2 to work on Windows,
|
||||
following the recent switch to UCRT and Rtools42.
|
||||
|
||||
Sorry for the group message, please feel free to respond individually
|
||||
regarding your package or ask specifically about what needs to be fixed.
|
||||
|
||||
I've created patches for you, so please review them and fix your packages:
|
||||
|
||||
https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fsvn.r-project.org%2FR-dev-web%2Ftrunk%2FWindowsBuilds%2Fwinutf8%2Fucrt3%2Fr_packages%2Fpatches%2FCRAN%2F&data=04%7C01%7Cyushi2%40microsoft.com%7C8e6c353d1a8842c81eeb08d9bef5d835%7C72f988bf86f141af91ab2d7cd011db47%7C1%7C0%7C637750786169848244%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&sdata=rFGf7Y4Dvo6g1kzV%2BeAJDLGm1TUtzQsLsavElTw6H1U%3D&reserved=0
|
||||
|
||||
You can apply them as follows
|
||||
|
||||
tar xfz package_1.0.0.tar.gz
|
||||
|
||||
wget
|
||||
https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fsvn.r-project.org%2FR-dev-web%2Ftrunk%2FWindowsBuilds%2Fwinutf8%2Fucrt3%2Fr_packages%2Fpatches%2FCRAN%2Fpackage.diff&data=04%7C01%7Cyushi2%40microsoft.com%7C8e6c353d1a8842c81eeb08d9bef5d835%7C72f988bf86f141af91ab2d7cd011db47%7C1%7C0%7C637750786169848244%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&sdata=iyTjhoqvzj3IbQ8HGCZeh1IQl34FAGpIdVyZWkzNvO0%3D&reserved=0
|
||||
|
||||
patch --binary < package.diff
|
||||
|
||||
These patches are currently automatically applied by R-devel on Windows
|
||||
at installation time, which makes most of your packages pass their
|
||||
checks (as OK or NOTE), but please check your results carefully and
|
||||
carefully review the patches. Usually these changes were because of
|
||||
newer GCC or newer MinGW in the toolchain, but some for other reasons,
|
||||
and some of them will definitely have to be improved so that the package
|
||||
keeps building also for older versions of R using Rtools40. We have only
|
||||
been testing the patches with UCRT (and Rtools42) on Windows.
|
||||
|
||||
For more information, please see
|
||||
|
||||
https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.r-project.org%2FBlog%2Fpublic%2F2021%2F12%2F07%2Fupcoming-changes-in-r-4.2-on-windows%2F&data=04%7C01%7Cyushi2%40microsoft.com%7C8e6c353d1a8842c81eeb08d9bef5d835%7C72f988bf86f141af91ab2d7cd011db47%7C1%7C0%7C637750786169848244%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&sdata=SY77zgtbDbHvTxTgPLOoe%2Fw5OZDhXvJoxpVOoEaKoYo%3D&reserved=0
|
||||
https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.r-project.org%2FWindowsBuilds%2Fwinutf8%2Fucrt3%2Fhowto.html&data=04%7C01%7Cyushi2%40microsoft.com%7C8e6c353d1a8842c81eeb08d9bef5d835%7C72f988bf86f141af91ab2d7cd011db47%7C1%7C0%7C637750786169848244%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&sdata=dlVJ4nhQlmDPd56bHoVsWZuRfrUUorvOWxoUTmVDM%2Bg%3D&reserved=0
|
||||
|
||||
Once you add your patches/fix the issues, your package will probably
|
||||
show a warning during R CMD check (as patching would be attempted to be
|
||||
applied again). That's ok, at that point please let me know and I will
|
||||
remove my patch from the repository of automatically applied patches.
|
||||
|
||||
If you end up just applying the patch as is, there is probably no need
|
||||
testing on your end, but you can do so using Winbuilder, r-hub, github
|
||||
actions (e.g. https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithub.com%2Fkalibera%2Fucrt3&data=04%7C01%7Cyushi2%40microsoft.com%7C8e6c353d1a8842c81eeb08d9bef5d835%7C72f988bf86f141af91ab2d7cd011db47%7C1%7C0%7C637750786169848244%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&sdata=msqoPzqDStlAUn%2Bb6gGevwFPD%2FaNL5dTxiNud2Sqzy8%3D&reserved=0).
|
||||
|
||||
If you wanted to test locally on your Windows machine and do not have a
|
||||
UCRT version of R-devel yet, please uninstall your old version of
|
||||
R-devel, delete the old library used with that, install a new UCRT
|
||||
version of R-devel , and install Rtools42. You can keep Rtools40
|
||||
installed if you need it with R 4.1 or earlier.
|
||||
|
||||
Currently, the new R-devel can be downloaded from
|
||||
https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.r-project.org%2Fnosvn%2Fwinutf8%2Fucrt3%2Fweb%2Frdevel.html&data=04%7C01%7Cyushi2%40microsoft.com%7C8e6c353d1a8842c81eeb08d9bef5d835%7C72f988bf86f141af91ab2d7cd011db47%7C1%7C0%7C637750786169848244%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&sdata=0hCwONzLmcW0GIXNqiOZQEIuhNA%2BjHhQvXsofs8J98o%3D&reserved=0
|
||||
|
||||
And Rtools42 from
|
||||
https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.r-project.org%2Fnosvn%2Fwinutf8%2Fucrt3%2Fweb%2Frtools.html&data=04%7C01%7Cyushi2%40microsoft.com%7C8e6c353d1a8842c81eeb08d9bef5d835%7C72f988bf86f141af91ab2d7cd011db47%7C1%7C0%7C637750786169848244%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&sdata=WLWLbOyQKbaYz8gkfKz2sqoGknjIOtl1aGAhUF%2Bpylg%3D&reserved=0
|
||||
|
||||
If you end up testing locally, you can use R_INSTALL_TIME_PATCHES
|
||||
environment variable to disable the automated patching, see the "howto"
|
||||
document above. That way you could also see what the original issue was
|
||||
causing.
|
||||
|
||||
If you wanted to find libraries to link for yourself, e.g. in a newer
|
||||
version of your package, please look for "Using findLinkingOrder with
|
||||
Rtools42 (tiff package example)" in the "howto" document above. I
|
||||
created the patches for you manually before we finished this script, so
|
||||
you may be able to create a shorter version using it, but - it's
|
||||
probably not worth the effort.
|
||||
|
||||
If you wanted to try in a virtual machine, but did not have a license,
|
||||
you can use also an automated setup of a free trial VM from
|
||||
https://nam06.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdeveloper.r-project.org%2FBlog%2Fpublic%2F2021%2F03%2F18%2Fvirtual-windows-machine-for-checking-r-packages&data=04%7C01%7Cyushi2%40microsoft.com%7C8e6c353d1a8842c81eeb08d9bef5d835%7C72f988bf86f141af91ab2d7cd011db47%7C1%7C0%7C637750786169848244%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C3000&sdata=aFFQYuC9CoBwBiLgZHi8N3yUnSiHu5Xtdqb2YBiMIHQ%3D&reserved=0
|
||||
|
||||
(but that needs a very good and un-metered network connection to install)
|
||||
|
||||
Please let us know if you have any questions.
|
||||
|
||||
Thanks,
|
||||
Tomas & Uwe
|
||||
```
|
||||
|
||||
## v3.3.1 - Submission 1 - (October 27, 2021)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN on October 30, 2021.
|
||||
|
||||
CRAN completed its checks and preparation of binaries on November 6, 2021.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Submitted v3.3.1 to CRAN, with the following fixes for the issues that caused CRAN to reject v3.3.0 and archive the package:
|
||||
|
||||
* https://github.com/lightgbm-org/LightGBM/pull/4673
|
||||
* https://github.com/lightgbm-org/LightGBM/pull/4714
|
||||
|
||||
Submitted with the following comment:
|
||||
|
||||
> This submission contains {lightgbm} 3.3.1.
|
||||
> Per CRAN's policies, I am submitting on it on behalf of the project's maintainer (Yu Shi), with his permission (https://github.com/lightgbm-org/LightGBM/pull/4715#issuecomment-952537783).
|
||||
|
||||
> {lightgbm} was removed from CRAN on October 25, 2021 due to issues detected in the gcc-ASAN and clang-ASAN checks. To the best of our knowledge, we believe this release fixes those issues. We have introduced automated testing that we believe faithfully reproduces CRAN's tests with sanitizers (https://github.com/lightgbm-org/LightGBM/pull/4678).
|
||||
|
||||
> Thank you very much for your time and consideration.
|
||||
|
||||
Progress on the submission was tracked in https://github.com/lightgbm-org/LightGBM/issues/4713.
|
||||
|
||||
## v3.3.0 - Submission 1 - (October 8, 2021)
|
||||
|
||||
### CRAN response
|
||||
|
||||
`{lightgbm}` was removed from CRAN entirely on October 25, 2021.
|
||||
|
||||
On October 12, 2021, maintainers received the following message from CRAN (ripley@stats.ox.ac.uk):
|
||||
|
||||
> Dear maintainer,
|
||||
|
||||
> Please see the problems shown on https://cran.r-project.org/web/checks/check_results_lightgbm.html
|
||||
|
||||
> Please correct before 2021-10-25 to safely retain your package on CRAN.
|
||||
|
||||
> Do remember to look at the 'Additional issues'.
|
||||
|
||||
> The CRAN Team
|
||||
|
||||
We failed to produce a new submission prior to that date, so the package was removed entirely.
|
||||
|
||||
See https://github.com/lightgbm-org/LightGBM/issues/4713 for additional background and links explaining the specific failed CRAN checks.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
In this submission, we attempted to switch the maintainer of the package (in the CRAN official sense) from Guolin Ke to Yu Shi.
|
||||
Did this by adding a note in the CRAN submission web form explaining Guolin's departure from Microsoft.
|
||||
|
||||
## v3.2.1 - Submission 1 - (April 12, 2021)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
## v3.2.0 - Submission 1 - (March 22, 2021)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Package is failing checks in the `r-devel-linux-x86_64-debian-clang` environment (described [here](https://cran.r-project.org/web/checks/check_flavors.html#r-devel-linux-x86_64-debian-clang)). Specifically, one unit test on the use of non-ASCII feature names in `Booster$dump_model()` fails.
|
||||
|
||||
> Apparently your package fails its checks in a strict Latin-1* locale,
|
||||
e.g. under Linux using LANG=en_US.iso88591 (see the debian-clang
|
||||
results).
|
||||
|
||||
> Please correct before 2021-04-21 to safely retain your package on CRAN.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Submitted a version 3.2.1 to correct the errors noted.
|
||||
|
||||
## v3.1.1 - Submission 1 - (December 7, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN, December 8.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Submitted a fix to 3.1.0 that skips some learning-to-rank tests on 32-bit Windows.
|
||||
|
||||
## v3.1.0 - Submission 1 - (November 15, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Accepted to CRAN, November 18.
|
||||
|
||||
On November 21, found out that the CRAN's `r-oldrel-windows-ix86+x86_64` check was failing, with an issue similar to the one faced on Solaris and fixed in https://github.com/lightgbm-org/LightGBM/pull/3534.
|
||||
|
||||
CRAN did not ask for a re-submission, but this was fixed in 3.1.1.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
This package was submitted with the following information in the "optional comments" box.
|
||||
|
||||
```text
|
||||
Hello,
|
||||
|
||||
I'm submitting {lightgbm} 3.1.0 on behalf of the maintainer, Guolin Ke. I am a co-author on the package, and he has asked me to handle this submission. We saw in https://cran.r-project.org/web/packages/policies.html#Submission that this is permitted.
|
||||
|
||||
{lightgbm} was removed from CRAN in October for issues found by valgrind checks. We have invested significant effort in addressing those issues and creating an automatic test that tries to replicate CRAN's valgrind checks: https://github.com/lightgbm-org/LightGBM/blob/742d72f8bb051105484fd5cca11620493ffb0b2b/.github/workflows/r_valgrind.yml.
|
||||
|
||||
We see two warnings from valgrind that we believe are not problematic.
|
||||
|
||||
==2063== Conditional jump or move depends on uninitialised value(s)
|
||||
==2063== at 0x49CF138: gregexpr_Regexc (grep.c:2439)
|
||||
==2063== by 0x49D1F13: do_regexpr (grep.c:3100)
|
||||
==2063== by 0x49A0058: bcEval (eval.c:7121)
|
||||
==2063== by 0x498B67F: Rf_eval (eval.c:727)
|
||||
==2063== by 0x498E414: R_execClosure (eval.c:1895)
|
||||
==2063== by 0x498E0C7: Rf_applyClosure (eval.c:1821)
|
||||
==2063== by 0x499FC8C: bcEval (eval.c:7089)
|
||||
==2063== by 0x498B67F: Rf_eval (eval.c:727)
|
||||
==2063== by 0x498B1CB: forcePromise (eval.c:555)
|
||||
==2063== by 0x49963AB: FORCE_PROMISE (eval.c:5142)
|
||||
==2063== by 0x4996566: getvar (eval.c:5183)
|
||||
==2063== by 0x499D1A5: bcEval (eval.c:6873)
|
||||
==2063== Uninitialised value was created by a stack allocation
|
||||
==2063== at 0x49CEC37: gregexpr_Regexc (grep.c:2369)
|
||||
|
||||
This seems to be related to R itself and not any code in {lightgbm}.
|
||||
|
||||
==2063== 336 bytes in 1 blocks are possibly lost in loss record 153 of 2,709
|
||||
==2063== at 0x483DD99: calloc (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_memcheck-amd64-linux.so)
|
||||
==2063== by 0x40149CA: allocate_dtv (dl-tls.c:286)
|
||||
==2063== by 0x40149CA: _dl_allocate_tls (dl-tls.c:532)
|
||||
==2063== by 0x5702322: allocate_stack (allocatestack.c:622)
|
||||
==2063== by 0x5702322: pthread_create@@GLIBC_2.2.5 (pthread_create.c:660)
|
||||
==2063== by 0x56D0DDA: ??? (in /usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0)
|
||||
==2063== by 0x56C88E0: GOMP_parallel (in /usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0)
|
||||
==2063== by 0x1544D29C: LGBM_DatasetCreateFromCSC (c_api.cpp:1286)
|
||||
==2063== by 0x1546F980: LGBM_DatasetCreateFromCSC_R (lightgbm_R.cpp:91)
|
||||
==2063== by 0x4941E2F: R_doDotCall (dotcode.c:634)
|
||||
==2063== by 0x494CCC6: do_dotcall (dotcode.c:1281)
|
||||
==2063== by 0x499FB01: bcEval (eval.c:7078)
|
||||
==2063== by 0x498B67F: Rf_eval (eval.c:727)
|
||||
==2063== by 0x498E414: R_execClosure (eval.c:1895)
|
||||
|
||||
We believe this is a false positive, and related to a misunderstanding between valgrind and openmp (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=36298).
|
||||
|
||||
We have also added automated tests with ASAN/UBSAN to our testing setup, and have checked the package on Solaris 10 and found no issues.
|
||||
|
||||
Thanks for your time and consideration.
|
||||
```
|
||||
|
||||
## v3.0.0.2 - Submission 1 - (September 29, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
First response was a message talking about failing checks on 3.0.0.
|
||||
|
||||
```text
|
||||
package lightgbm_3.0.0.2.tar.gz has been auto-processed.
|
||||
The auto-check found additional issues for the last version released on CRAN:
|
||||
gcc-UBSAN <link>
|
||||
valgrind <link>
|
||||
CRAN incoming checks do not test for these additional issues and you will need an appropriately instrumented build of R to reproduce these.
|
||||
Hence please reply-all and explain: Have these been fixed?
|
||||
|
||||
Please correct before 2020-10-05 to safely retain your package on CRAN.
|
||||
|
||||
There is still a valgrind error. This did not happen when tested on
|
||||
submission, but the tests did run until timeout at 4 hours. When you
|
||||
write illegally, corruption is common.
|
||||
|
||||
Illegal writes are serious errors.
|
||||
```
|
||||
|
||||
Then in later responses to email correspondence with CRAN, CRAN expressed frustration with the number of failed submission and banned this package from new submissions for a month.
|
||||
|
||||
The content of that frustrated message was regrettable and it does not need to be preserved forever in this file.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
The 3.0.0.x series is officially not making it to CRAN. We will wait until November, and try again.
|
||||
|
||||
Detailed plan about what will be tried before November 2020 to increase the likelihood of success for that package: https://github.com/lightgbm-org/LightGBM/pull/3338#issuecomment-702756840.
|
||||
|
||||
## v3.0.0.1 - Submission 1 - (September 24, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
```text
|
||||
Thanks, we see:
|
||||
|
||||
Still lots of alignment errors, such as
|
||||
|
||||
lightgbm.Rcheck/tests/testthat.Rout:io/dataset_loader.cpp:340:59:
|
||||
runtime error: reference binding to misaligned address 0x7f51fefad81e for type 'const value_type', which requires 4 byte alignment
|
||||
lightgbm.Rcheck/tests/testthat.Rout:/usr/include/c++/10/bits/stl_vector.h:1198:21:
|
||||
runtime error: reference binding to misaligned address 0x7f51fefad81e for type 'const int', which requires 4 byte alignment lightgbm.Rcheck/tests/testthat.Rout:/usr/include/c++/10/bits/vector.tcc:449:28:runtime
|
||||
error: reference binding to misaligned address 0x7f51fefad81e for type 'const type', which requires 4 byte alignment
|
||||
lightgbm.Rcheck/tests/testthat.Rout:/usr/include/c++/10/bits/move.h:77:36:
|
||||
runtime error: reference binding to misaligned address 0x7f51fefad81e for type 'const int', which requires 4 byte alignment
|
||||
lightgbm.Rcheck/tests/testthat.Rout:/usr/include/c++/10/bits/alloc_traits.h:512:17:
|
||||
runtime error: reference binding to misaligned address 0x7f51fefad81e for type 'const type', which requires 4 byte alignment
|
||||
|
||||
Please fix and resubmit.
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Ok, these are the notes from the UBSAN tests. Was able to reproduce them with https://github.com/lightgbm-org/LightGBM/pull/3338#issuecomment-700399862, and they were fixed in https://github.com/lightgbm-org/LightGBM/pull/3415.
|
||||
|
||||
Struggling to replicate the valgrind result (running `R CMD check --use-valgrind` returns no issues), so trying submission again. Hoping that the fixes for mis-alignment fix the other errors too.
|
||||
|
||||
## v3.0.0 - Submission 6 - (September 24, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Failing pre-checks.
|
||||
|
||||
### `R CMD check` results
|
||||
|
||||
```text
|
||||
* checking CRAN incoming feasibility ... WARNING
|
||||
Maintainer: ‘Guolin Ke <guolin.ke@microsoft.com>’
|
||||
|
||||
Insufficient package version (submitted: 3.0.0, existing: 3.0.0)
|
||||
|
||||
Days since last update: 4
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Did not think the version needed to be incremented if submitting a package in response to CRAN saying "you are failing checks and will be kicked off if you don't fix it", but I guess you do!
|
||||
|
||||
This can be fixed by just re-submitting but with the version changed from `3.0.0` to `3.0.0.1`.
|
||||
|
||||
## v3.0.0 - Submission 5 - (September 11, 2020)
|
||||
|
||||
### CRAN Response
|
||||
|
||||
Accepted to CRAN!
|
||||
|
||||
Please correct the problems below before 2020-10-05 to safely retain your package on CRAN:
|
||||
|
||||
```text
|
||||
checking installed package size ... NOTE
|
||||
installed size is 49.7Mb
|
||||
sub-directories of 1Mb or more:
|
||||
libs 49.1Mb
|
||||
|
||||
"network/socket_wrapper.hpp", line 30: Error: Could not open include file<ifaddrs.h>.
|
||||
"network/socket_wrapper.hpp", line 216: Error: The type "ifaddrs" is incomplete.
|
||||
"network/socket_wrapper.hpp", line 217: Error: The type "ifaddrs" is incomplete.
|
||||
"network/socket_wrapper.hpp", line 220: Error: The type "ifaddrs" is incomplete.
|
||||
"network/socket_wrapper.hpp", line 222: Error: The type "ifaddrs" is incomplete.
|
||||
"network/socket_wrapper.hpp", line 214: Error: The function "getifaddrs" must have a prototype.
|
||||
"network/socket_wrapper.hpp", line 228: Error: The function "freeifaddrs" must have a prototype.
|
||||
"network/linkers_socket.cpp", line 76: Warning: A non-POD object of type "std::chrono::duration<double, std::ratio<1, 1000>>" passed as a variable argument to function "static LightGBM::Log::Info(const char*, ...)".
|
||||
7 Error(s) and 1 Warning(s) detected.
|
||||
*** Error code 2
|
||||
make: Fatal error: Command failed for target `network/linkers_socket.o'
|
||||
Current working directory /tmp/RtmpNfaavG/R.INSTALL40a84f70130a/lightgbm/src
|
||||
ERROR: compilation failed for package ‘lightgbm’
|
||||
* removing ‘/home/ripley/R/Lib32/lightgbm’
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Added a patch that `psutil` has used to fix missing `ifaddrs.h` on Solaris 10: https://github.com/lightgbm-org/LightGBM/issues/629#issuecomment-665091451.
|
||||
|
||||
## v3.0.0 - Submission 4 - (September 4, 2020)
|
||||
|
||||
### CRAN Response
|
||||
|
||||
> Thanks, if the running time is the only reason to wrap the examples in
|
||||
\donttest, please replace \donttest by \donttest (\donttest examples are
|
||||
not executed in the CRAN checks).
|
||||
|
||||
> Please replace cat() by message() or warning() in your functions (except
|
||||
for print() and summary() functions). Messages and warnings can be
|
||||
suppressed if needed.
|
||||
|
||||
> Missing Rd-tags:
|
||||
lightgbm/man/dimnames.lgb.Dataset.Rd: \value
|
||||
lightgbm/man/lgb.Dataset.construct.Rd: \value
|
||||
lightgbm/man/lgb.prepare.Rd: \value
|
||||
...
|
||||
|
||||
> Please add the tag and explain in detail the returned objects.
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Responded to CRAN with the following:
|
||||
|
||||
All examples have been wrapped with `\donttest` as requested. We have replied to Swetlana Herbrandt asking for clarification on the donttest news item in the R 4.0.2 changelog (https://cran.r-project.org/doc/manuals/r-devel/NEWS.html).
|
||||
|
||||
All uses of `cat()` have been replaced with `print()`. We chose `print()` over `message()` because it's important that they be written to stdout alongside all the other logs coming from the library's C++ code. `message()` and `warning()` write to stderr.
|
||||
|
||||
All exported objects now have `\value{}` statements in their documentation files in `man/`.
|
||||
|
||||
**We also replied directly to CRAN's feedback email**
|
||||
|
||||
> Swetlana,
|
||||
|
||||
> Thank you for your comments. I've just created a new submission that I believe addresses them.
|
||||
|
||||
> Can you help us understand something? In your message you said "\donttest examples are
|
||||
not executed in the CRAN checks)", but in https://cran.r-project.org/doc/manuals/r-devel/NEWS.html we see the following:
|
||||
|
||||
> > "`R CMD check --as-cran` now runs \donttest examples (which are run by example()) instead of instructing the tester to do so. This can be temporarily circumvented during development by setting environment variable `_R_CHECK_DONTTEST_EXAMPLES_` to a false value."
|
||||
|
||||
> Could you help us understand how both of those statements can be true?
|
||||
|
||||
## v3.0.0 - Submission 3 - (August 29, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
* Please write references in the description of the DESCRIPTION file in
|
||||
the form
|
||||
- authors (year) doi:...
|
||||
- authors (year) arXiv:...
|
||||
- authors (year, ISBN:...)
|
||||
* if those are not available: authors (year) https:... with no space after 'doi:', 'arXiv:', 'https:' and angle brackets for auto-linking.
|
||||
* (If you want to add a title as well please put it in quotes: "Title")
|
||||
|
||||
* \donttest{} should only be used if the example really cannot be executed (e.g. because of missing additional software, missing API keys, ...) by the user. That's why wrapping examples in \donttest{} adds the comment ("# Not run:") as a warning for the user. Does not seem necessary. Please unwrap the examples if they are executable in < 5 sec, or replace
|
||||
\donttest{} with \donttest{}.
|
||||
|
||||
* Please do not modify the global environment (e.g. by using <<-) in your
|
||||
functions. This is not allowed by the CRAN policies.
|
||||
|
||||
* Please always add all authors, contributors and copyright holders in the Authors@R field with the appropriate roles. From CRAN policies you agreed to: "The ownership of copyright and intellectual property rights of all components of the package must be clear and unambiguous (including from the authors specification in the DESCRIPTION file). Where code is copied (or derived) from the work of others (including from R itself), care must be taken that any copyright/license statements are preserved and authorship is not misrepresented." e.g.: Microsoft Corporation, Dropbox Inc. Please explain in the submission comments what you did about this issue.
|
||||
|
||||
Please fix and resubmit
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Responded to CRAN with the following:
|
||||
|
||||
The paper citation has been adjusted as requested. We were using 'glmnet' as a guide on how to include the URL but maybe they are no longer in compliance with CRAN policies: https://github.com/cran/glmnet/blob/b1a4b50de01e0cd24343959d7cf86452bac17b26/DESCRIPTION
|
||||
|
||||
All authors from the original LightGBM paper have been added to Authors@R as `"aut"`. We have also added Microsoft and DropBox, Inc. as `"cph"` (copyright holders). These roles were chosen based on the guidance in https://journal.r-project.org/archive/2012/RJ-2012-009/index.html.
|
||||
|
||||
lightgbm's code does use `<<-`, but it does not modify the global environment. The uses of `<<-` in R/lgb.interprete.R and R/callback.R are in functions which are called in an environment created by the lightgbm functions that call them, and this operator is used to reach one level up into the calling function's environment.
|
||||
|
||||
We chose to wrap our examples in `\donttest{}` because we found, through testing on https://r-hub.github.io/rhub/ and in our own continuous integration environments, that their run time varies a lot between platforms, and we cannot guarantee that all examples will run in under 5 seconds. We intentionally chose `\donttest{}` over `\donttest{}` because this item in the R 4.0.0 changelog (https://cran.r-project.org/doc/manuals/r-devel/NEWS.html) seems to indicate that \donttest will be ignored by CRAN's automated checks:
|
||||
|
||||
> "`R CMD check --as-cran` now runs \donttest examples (which are run by example()) instead of instructing the tester to do so. This can be temporarily circumvented during development by setting environment variable `_R_CHECK_DONTTEST_EXAMPLES_` to a false value."
|
||||
|
||||
We run all examples with `R CMD check --as-cran --run-dontrun` in our continuous integration tests on every commit to the package, so we have high confidence that they are working correctly.
|
||||
|
||||
## v3.0.0 - Submission 2 - (August 28, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Failing pre-checks.
|
||||
|
||||
### `R CMD check` results
|
||||
|
||||
* Debian: 2 NOTEs
|
||||
|
||||
```text
|
||||
* checking CRAN incoming feasibility ... NOTE
|
||||
Maintainer: 'Guolin Ke <guolin.ke@microsoft.com>'
|
||||
|
||||
New submission
|
||||
|
||||
Possibly mis-spelled words in DESCRIPTION:
|
||||
Guolin (13:52)
|
||||
Ke (13:48)
|
||||
LightGBM (14:20)
|
||||
al (13:62)
|
||||
et (13:59)
|
||||
|
||||
* checking top-level files ... NOTE
|
||||
Non-standard files/directories found at top level:
|
||||
'docs' 'lightgbm-hex-logo.png' 'lightgbm-hex-logo.svg'
|
||||
```
|
||||
|
||||
* Windows: 2 NOTEs
|
||||
|
||||
```text
|
||||
* checking CRAN incoming feasibility ... NOTE
|
||||
Maintainer: 'Guolin Ke <guolin.ke@microsoft.com>'
|
||||
|
||||
New submission
|
||||
|
||||
Possibly mis-spelled words in DESCRIPTION:
|
||||
Guolin (13:52)
|
||||
Ke (13:48)
|
||||
LightGBM (14:20)
|
||||
al (13:62)
|
||||
et (13:59)
|
||||
|
||||
* checking top-level files ... NOTE
|
||||
Non-standard files/directories found at top level:
|
||||
'docs' 'lightgbm-hex-logo.png' 'lightgbm-hex-logo.svg'
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
We should tell them the misspellings note is a false positive.
|
||||
|
||||
For the note about included files, that is my fault. I had extra files laying around when I generated the package. I'm surprised to see `docs/` in that list, since it is ignored in `.Rbuildignore`. I even tested that with [the exact code Rbuildignore uses](https://github.com/wch/r-source/blob/9d13622f41cfa0f36db2595bd6a5bf93e2010e21/src/library/tools/R/build.R#L85). For now, I added `rm -r docs/` to `build-cran-package.sh`. We can figure out what is happening with `.Rbuildignore` in the future, but it shouldn't block a release.
|
||||
|
||||
## v3.0.0 - Submission 1 - (August 24, 2020)
|
||||
|
||||
NOTE: 3.0.0-1 was never released to CRAN. CRAN was on vacation August 14-24, 2020, and in that time version 3.0.0-1 (a release candidate) became 3.0.0.
|
||||
|
||||
### CRAN response
|
||||
|
||||
> Please only ship the CRAN template for the MIT license.
|
||||
|
||||
> Is there some reference about the method you can add in the Description field in the form Authors (year) doi:.....?
|
||||
|
||||
> Please fix and resubmit.
|
||||
|
||||
### `R CMD check` results
|
||||
|
||||
* Debian: 1 NOTE
|
||||
|
||||
```text
|
||||
* checking CRAN incoming feasibility ... NOTE
|
||||
Maintainer: ‘Guolin Ke <guolin.ke@microsoft.com>’
|
||||
|
||||
New submission
|
||||
|
||||
License components with restrictions and base license permitting such:
|
||||
MIT + file LICENSE
|
||||
```
|
||||
|
||||
* Windows: 1 NOTE
|
||||
|
||||
```text
|
||||
* checking CRAN incoming feasibility ... NOTE
|
||||
Maintainer: 'Guolin Ke <guolin.ke@microsoft.com>'
|
||||
|
||||
New submission
|
||||
|
||||
License components with restrictions and base license permitting such:
|
||||
MIT + file LICENSE
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Tried updating `LICENSE` file to this template:
|
||||
|
||||
```yaml
|
||||
YEAR: 2016
|
||||
COPYRIGHT HOLDER: Microsoft Corporation
|
||||
```
|
||||
|
||||
Added a citation and link for [the main paper](https://proceedings.neurips.cc/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html) in `DESCRIPTION`.
|
||||
|
||||
## v3.0.0-1 - Submission 3 - (August 12, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Failing pre-checks.
|
||||
|
||||
### `R CMD check` results
|
||||
|
||||
* Debian: 1 NOTE
|
||||
|
||||
```text
|
||||
* checking CRAN incoming feasibility ... NOTE
|
||||
Maintainer: ‘Guolin Ke <guolin.ke@microsoft.com>’
|
||||
|
||||
New submission
|
||||
|
||||
License components with restrictions and base license permitting such:
|
||||
MIT + file LICENSE
|
||||
```
|
||||
|
||||
* Windows: 1 ERROR, 1 NOTE
|
||||
|
||||
```text
|
||||
* checking CRAN incoming feasibility ... NOTE
|
||||
Maintainer: ‘Guolin Ke <guolin.ke@microsoft.com>’
|
||||
|
||||
New submission
|
||||
|
||||
License components with restrictions and base license permitting such:
|
||||
MIT + file LICENSE
|
||||
|
||||
** running tests for arch 'i386' ... [9s] ERROR
|
||||
Running 'testthat.R' [8s]
|
||||
Running the tests in 'tests/testthat.R' failed.
|
||||
Complete output:
|
||||
> library(testthat)
|
||||
> library(lightgbm)
|
||||
Loading required package: R6
|
||||
>
|
||||
> test_check(
|
||||
+ package = "lightgbm"
|
||||
+ , stop_on_failure = TRUE
|
||||
+ , stop_on_warning = FALSE
|
||||
+ )
|
||||
-- 1. Error: predictions do not fail for integer input (@test_Predictor.R#7) --
|
||||
lgb.Dataset.construct: cannot create Dataset handle
|
||||
Backtrace:
|
||||
1. lightgbm::lgb.train(...)
|
||||
2. data$construct()
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
The "checking CRAN incoming feasibility" NOTE can be safely ignored. It only shows up the first time you submit a package to CRAN.
|
||||
|
||||
So the only thing I see broken right now is the test error on 32-bit Windows. This is documented in https://github.com/lightgbm-org/LightGBM/issues/3187.
|
||||
|
||||
## v3.0.0-1 - Submission 2 - (August 10, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Failing pre-checks.
|
||||
|
||||
### `R CMD check` results
|
||||
|
||||
* Debian: 2 NOTEs
|
||||
|
||||
```text
|
||||
* checking CRAN incoming feasibility ... NOTE
|
||||
Maintainer: ‘Guolin Ke <guolin.ke@microsoft.com>’
|
||||
|
||||
New submission
|
||||
|
||||
License components with restrictions and base license permitting such:
|
||||
MIT + file LICENSE
|
||||
|
||||
Non-standard files/directories found at top level:
|
||||
‘cran-comments.md’ ‘docs’
|
||||
```
|
||||
|
||||
* Windows: 1 ERROR, 2 NOTEs
|
||||
|
||||
```text
|
||||
* checking CRAN incoming feasibility ... NOTE
|
||||
Maintainer: 'Guolin Ke <guolin.ke@microsoft.com>'
|
||||
|
||||
New submission
|
||||
|
||||
License components with restrictions and base license permitting such:
|
||||
MIT + file LICENSE
|
||||
|
||||
* checking top-level files ... NOTE
|
||||
Non-standard files/directories found at top level:
|
||||
'cran-comments.md' 'docs'
|
||||
|
||||
** checking whether the package can be loaded ... ERROR
|
||||
Loading this package had a fatal error status code 1
|
||||
Loading log:
|
||||
Error: package 'lightgbm' is not installed for 'arch = i386'
|
||||
Execution halted
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Seems removing `Biarch` field didn't work. Noticed this in the install logs:
|
||||
|
||||
> Warning: this package has a non-empty 'configure.win' file, so building only the main architecture
|
||||
|
||||
Tried adding `Biarch: true` to `DESCRIPTION` to overcome this.
|
||||
|
||||
NOTE about non-standard files was the result of a mistake in `.Rbuildignore` syntax, and something strange with how `cran-comments.md` line in `.Rbuildignore` was treated. Updated `.Rbuildignore` and added an `rm cran-comments.md` to `build-cran-package.sh`.
|
||||
|
||||
## v3.0.0-1 - Submission 1 - (August 9, 2020)
|
||||
|
||||
### CRAN response
|
||||
|
||||
Failing pre-checks.
|
||||
|
||||
### `R CMD check` results
|
||||
|
||||
* Debian: 1 NOTE
|
||||
|
||||
```text
|
||||
Possibly mis-spelled words in DESCRIPTION:
|
||||
LightGBM (12:88, 19:41, 20:60, 20:264)
|
||||
```
|
||||
|
||||
* Windows: 1 ERROR, 1 NOTE
|
||||
|
||||
```text
|
||||
Possibly mis-spelled words in DESCRIPTION:
|
||||
LightGBM (12:88, 19:41, 20:60, 20:264)
|
||||
|
||||
** checking whether the package can be loaded ... ERROR
|
||||
Loading this package had a fatal error status code 1
|
||||
Loading log:
|
||||
Error: package 'lightgbm' is not installed for 'arch = i386'
|
||||
Execution halted
|
||||
```
|
||||
|
||||
### Maintainer Notes
|
||||
|
||||
Thought the issue on Windows was caused by `Biarch: false` in `DESCRIPTION`. Removed `Biarch` field.
|
||||
|
||||
Thought the "misspellings" issue could be resolved by adding single quotes around LightGBM, like `'LightGBM'`.
|
||||
@@ -0,0 +1,10 @@
|
||||
basic_walkthrough Basic feature walkthrough
|
||||
boost_from_prediction Boosting from existing prediction
|
||||
categorical_features_rules Categorical Feature Preparation with Rules
|
||||
cross_validation Cross Validation
|
||||
early_stopping Early Stop in training
|
||||
efficient_many_training Efficiency for Many Model Trainings
|
||||
multiclass Multiclass training/prediction
|
||||
multiclass_custom_objective Multiclass with Custom Objective Function
|
||||
leaf_stability Leaf (in)Stability example
|
||||
weight_param Weight-Parameter adjustment relationship
|
||||
@@ -0,0 +1,153 @@
|
||||
library(lightgbm)
|
||||
|
||||
# We load in the agaricus dataset
|
||||
# In this example, we are aiming to predict whether a mushroom is edible
|
||||
data(agaricus.train, package = "lightgbm")
|
||||
data(agaricus.test, package = "lightgbm")
|
||||
train <- agaricus.train
|
||||
test <- agaricus.test
|
||||
|
||||
# The loaded data is stored in sparseMatrix, and label is a numeric vector in {0,1}
|
||||
class(train$label)
|
||||
class(train$data)
|
||||
|
||||
# Set parameters for model training
|
||||
train_params <- list(
|
||||
num_leaves = 4L
|
||||
, learning_rate = 1.0
|
||||
, objective = "binary"
|
||||
, nthread = 2L
|
||||
)
|
||||
|
||||
#--------------------Basic Training using lightgbm----------------
|
||||
# This is the basic usage of lightgbm you can put matrix in data field
|
||||
# Note: we are putting in sparse matrix here, lightgbm naturally handles sparse input
|
||||
# Use sparse matrix when your feature is sparse (e.g. when you are using one-hot encoding vector)
|
||||
print("Training lightgbm with sparseMatrix")
|
||||
bst <- lightgbm(
|
||||
data = train$data
|
||||
, params = train_params
|
||||
, label = train$label
|
||||
, nrounds = 2L
|
||||
)
|
||||
|
||||
# Alternatively, you can put in dense matrix, i.e. basic R-matrix
|
||||
print("Training lightgbm with Matrix")
|
||||
bst <- lightgbm(
|
||||
data = as.matrix(train$data)
|
||||
, params = train_params
|
||||
, label = train$label
|
||||
, nrounds = 2L
|
||||
)
|
||||
|
||||
# You can also put in lgb.Dataset object, which stores label, data and other meta datas needed for advanced features
|
||||
print("Training lightgbm with lgb.Dataset")
|
||||
dtrain <- lgb.Dataset(
|
||||
data = train$data
|
||||
, label = train$label
|
||||
)
|
||||
bst <- lightgbm(
|
||||
data = dtrain
|
||||
, params = train_params
|
||||
, nrounds = 2L
|
||||
)
|
||||
|
||||
# Verbose = 0,1,2
|
||||
print("Train lightgbm with verbose 0, no message")
|
||||
bst <- lightgbm(
|
||||
data = dtrain
|
||||
, params = train_params
|
||||
, nrounds = 2L
|
||||
, verbose = 0L
|
||||
)
|
||||
|
||||
print("Train lightgbm with verbose 1, print evaluation metric")
|
||||
bst <- lightgbm(
|
||||
data = dtrain
|
||||
, params = train_params
|
||||
, nrounds = 2L
|
||||
, verbose = 1L
|
||||
)
|
||||
|
||||
print("Train lightgbm with verbose 2, also print information about tree")
|
||||
bst <- lightgbm(
|
||||
data = dtrain
|
||||
, params = train_params
|
||||
, nrounds = 2L
|
||||
, verbose = 2L
|
||||
)
|
||||
|
||||
# You can also specify data as file path to a LibSVM/TCV/CSV format input
|
||||
# Since we do not have this file with us, the following line is just for illustration
|
||||
# bst <- lightgbm(
|
||||
# data = "agaricus.train.svm"
|
||||
# , num_leaves = 4L
|
||||
# , learning_rate = 1.0
|
||||
# , nrounds = 2L
|
||||
# , objective = "binary"
|
||||
# )
|
||||
|
||||
#--------------------Basic prediction using lightgbm--------------
|
||||
# You can do prediction using the following line
|
||||
# You can put in Matrix, sparseMatrix, or lgb.Dataset
|
||||
pred <- predict(bst, test$data)
|
||||
err <- mean(as.numeric(pred > 0.5) != test$label)
|
||||
print(paste("test-error=", err))
|
||||
|
||||
#--------------------Save and load models-------------------------
|
||||
# Save model to binary local file
|
||||
lgb.save(bst, "lightgbm.model")
|
||||
|
||||
# Load binary model to R
|
||||
bst2 <- lgb.load("lightgbm.model")
|
||||
pred2 <- predict(bst2, test$data)
|
||||
|
||||
# pred2 should be identical to pred
|
||||
print(paste("sum(abs(pred2-pred))=", sum(abs(pred2 - pred))))
|
||||
|
||||
#--------------------Advanced features ---------------------------
|
||||
# To use advanced features, we need to put data in lgb.Dataset
|
||||
dtrain <- lgb.Dataset(data = train$data, label = train$label, free_raw_data = FALSE)
|
||||
dtest <- lgb.Dataset.create.valid(dtrain, data = test$data, label = test$label)
|
||||
|
||||
#--------------------Using validation set-------------------------
|
||||
# valids is a list of lgb.Dataset, each of them is tagged with name
|
||||
valids <- list(train = dtrain, test = dtest)
|
||||
|
||||
# To train with valids, use lgb.train, which contains more advanced features
|
||||
# valids allows us to monitor the evaluation result on all data in the list
|
||||
print("Train lightgbm using lgb.train with valids")
|
||||
bst <- lgb.train(
|
||||
data = dtrain
|
||||
, params = train_params
|
||||
, nrounds = 2L
|
||||
, valids = valids
|
||||
)
|
||||
|
||||
# We can change evaluation metrics, or use multiple evaluation metrics
|
||||
print("Train lightgbm using lgb.train with valids, watch logloss and error")
|
||||
bst <- lgb.train(
|
||||
data = dtrain
|
||||
, params = train_params
|
||||
, nrounds = 2L
|
||||
, valids = valids
|
||||
, eval = c("binary_error", "binary_logloss")
|
||||
)
|
||||
|
||||
# lgb.Dataset can also be saved using lgb.Dataset.save
|
||||
lgb.Dataset.save(dtrain, "dtrain.buffer")
|
||||
|
||||
# To load it in, simply call lgb.Dataset
|
||||
dtrain2 <- lgb.Dataset("dtrain.buffer")
|
||||
bst <- lgb.train(
|
||||
data = dtrain2
|
||||
, params = train_params
|
||||
, nrounds = 2L
|
||||
, valids = valids
|
||||
)
|
||||
|
||||
# information can be extracted from lgb.Dataset using get_field()
|
||||
label <- get_field(dtest, "label")
|
||||
pred <- predict(bst, test$data)
|
||||
err <- as.numeric(sum(as.integer(pred > 0.5) != label)) / length(label)
|
||||
print(paste("test-error=", err))
|
||||
@@ -0,0 +1,38 @@
|
||||
library(lightgbm)
|
||||
|
||||
# Load in the agaricus dataset
|
||||
data(agaricus.train, package = "lightgbm")
|
||||
data(agaricus.test, package = "lightgbm")
|
||||
dtrain <- lgb.Dataset(agaricus.train$data, label = agaricus.train$label)
|
||||
dtest <- lgb.Dataset.create.valid(dtrain, data = agaricus.test$data, label = agaricus.test$label)
|
||||
|
||||
valids <- list(eval = dtest, train = dtrain)
|
||||
#--------------------Advanced features ---------------------------
|
||||
# advanced: start from an initial base prediction
|
||||
print("Start running example to start from an initial prediction")
|
||||
|
||||
# Train lightgbm for 1 round
|
||||
param <- list(
|
||||
num_leaves = 4L
|
||||
, learning_rate = 1.0
|
||||
, nthread = 2L
|
||||
, objective = "binary"
|
||||
)
|
||||
bst <- lgb.train(param, dtrain, 1L, valids = valids)
|
||||
|
||||
# Note: we need the margin value instead of transformed prediction in set_init_score
|
||||
ptrain <- predict(bst, agaricus.train$data, type = "raw")
|
||||
ptest <- predict(bst, agaricus.test$data, type = "raw")
|
||||
|
||||
# set the init_score property of dtrain and dtest
|
||||
# base margin is the base prediction we will boost from
|
||||
set_field(dtrain, "init_score", ptrain)
|
||||
set_field(dtest, "init_score", ptest)
|
||||
|
||||
print("This is result of boost from initial prediction")
|
||||
bst <- lgb.train(
|
||||
params = param
|
||||
, data = dtrain
|
||||
, nrounds = 5L
|
||||
, valids = valids
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user