chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,423 @@
|
||||
LightGBM Python-package
|
||||
=======================
|
||||
|
||||
|License| |Python Versions| |PyPI Version| |PyPI Downloads| |conda Version| |conda Downloads| |API Docs|
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
Preparation
|
||||
'''''''''''
|
||||
|
||||
32-bit Python is not supported.
|
||||
Please install 64-bit version.
|
||||
If you have a strong need to install with 32-bit Python, refer to `Build 32-bit Version with 32-bit Python section <#build-32-bit-version-with-32-bit-python>`__.
|
||||
|
||||
|
|
||||
|
||||
Install from `PyPI <https://pypi.org/project/lightgbm>`_
|
||||
''''''''''''''''''''''''''''''''''''''''''''''''''''''''
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm
|
||||
|
||||
Compiled library that is included in the wheel file supports both **GPU** (don't confuse with CUDA version) and **CPU** versions out of the box.
|
||||
This feature is available only for **Windows** and **Linux** currently.
|
||||
To use **GPU** version you only need to install OpenCL Runtime libraries.
|
||||
For NVIDIA and AMD GPU they are included in the ordinary drivers for your graphics card, so no action is required.
|
||||
If you would like your AMD or Intel CPU to act like a GPU (for testing and debugging),
|
||||
you can install `AMD APP SDK <https://github.com/lightgbm-org/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe>`_ on **Windows** and `PoCL <https://portablecl.org>`_ on **Linux**.
|
||||
Many modern Linux distributions provide packages for PoCL, look for ``pocl-opencl-icd`` on Debian-based distributions and ``pocl`` on RedHat-based distributions.
|
||||
|
||||
For **Windows** users, `VC runtime <https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist>`_ is needed if **Visual Studio** is not installed.
|
||||
|
||||
For **macOS** users, the **OpenMP** library is needed.
|
||||
You can install it by the following command: ``brew install libomp``.
|
||||
|
||||
|
|
||||
|
||||
Install Nightly Packages
|
||||
''''''''''''''''''''''''
|
||||
|
||||
Python packages are built on each new commit to ``main`` and uploaded to https://anaconda.org/lightgbm-packages.
|
||||
|
||||
Only the latest development version is available there, and can be installed like this:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install --no-deps --index-url https://pypi.anaconda.org/lightgbm-packages/simple lightgbm
|
||||
|
||||
|
|
||||
|
||||
Use LightGBM with PyArrow
|
||||
*************************
|
||||
|
||||
To install all dependencies needed to use ``PyArrow`` in LightGBM, append ``[arrow]``.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install 'lightgbm[arrow]'
|
||||
|
||||
|
|
||||
|
||||
Use LightGBM with Polars
|
||||
*************************
|
||||
|
||||
To install all dependencies needed to use ``polars`` in LightGBM, append ``[polars]``.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install 'lightgbm[polars]'
|
||||
|
||||
|
|
||||
|
||||
Use LightGBM with Dask
|
||||
**********************
|
||||
|
||||
Warning: Dask-package is only tested on macOS and Linux.
|
||||
|
||||
To install all dependencies needed to use ``lightgbm.dask``, append ``[dask]``.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install 'lightgbm[dask]'
|
||||
|
||||
|
|
||||
|
||||
Use LightGBM with pandas
|
||||
************************
|
||||
|
||||
To install all dependencies needed to use ``pandas`` in LightGBM, append ``[pandas]``.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install 'lightgbm[pandas]'
|
||||
|
||||
|
|
||||
|
||||
Use LightGBM Plotting Capabilities
|
||||
**********************************
|
||||
|
||||
To install all dependencies needed to use ``lightgbm.plotting``, append ``[plotting]``.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install 'lightgbm[plotting]'
|
||||
|
||||
|
|
||||
|
||||
Use LightGBM with scikit-learn
|
||||
******************************
|
||||
|
||||
To install all dependencies needed to use ``lightgbm.sklearn``, append ``[scikit-learn]``.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install 'lightgbm[scikit-learn]'
|
||||
|
||||
|
|
||||
|
||||
Build from Sources
|
||||
******************
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --no-binary lightgbm
|
||||
|
||||
For **macOS** users, you can perform installation either with **Apple Clang** or **gcc**.
|
||||
|
||||
- In case you prefer **Apple Clang**, you should install **OpenMP** (details for installation can be found in `Installation Guide <https://github.com/lightgbm-org/LightGBM/blob/main/docs/Installation-Guide.rst#apple-clang>`__) first.
|
||||
|
||||
- In case you prefer **gcc**, you need to install it (details for installation can be found in `Installation Guide <https://github.com/lightgbm-org/LightGBM/blob/main/docs/Installation-Guide.rst#gcc-1>`__) and specify compilers by running ``export CXX=g++-7 CC=gcc-7`` (replace "7" with version of **gcc** installed on your machine) first.
|
||||
|
||||
For **Windows** users, **Visual Studio** (or `VS Build Tools <https://visualstudio.microsoft.com/downloads/>`_) is needed.
|
||||
|
||||
|
|
||||
|
||||
Build Threadless Version
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --no-binary lightgbm --config-settings=cmake.define.USE_OPENMP=OFF
|
||||
|
||||
All requirements, except the **OpenMP** requirement, from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
|
||||
|
||||
It is **strongly not recommended** to use this version of LightGBM!
|
||||
|
||||
|
|
||||
|
||||
Build MPI Version
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --no-binary lightgbm --config-settings=cmake.define.USE_MPI=ON
|
||||
|
||||
All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
|
||||
|
||||
For **Windows** users, compilation with **MinGW-w64** is not supported.
|
||||
|
||||
**MPI** libraries are needed: details for installation can be found in `Installation Guide <https://github.com/lightgbm-org/LightGBM/blob/main/docs/Installation-Guide.rst#build-mpi-version>`__.
|
||||
|
||||
|
|
||||
|
||||
Build GPU Version
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --no-binary lightgbm --config-settings=cmake.define.USE_GPU=ON
|
||||
|
||||
All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
|
||||
|
||||
For **macOS** users, the GPU version is not supported.
|
||||
|
||||
**Boost** and **OpenCL** are needed: details for installation can be found in `Installation Guide <https://github.com/lightgbm-org/LightGBM/blob/main/docs/Installation-Guide.rst#build-gpu-version>`__.
|
||||
Almost always you also need to pass ``OpenCL_INCLUDE_DIR``, ``OpenCL_LIBRARY`` options for **Linux** and ``BOOST_ROOT``, ``BOOST_LIBRARYDIR`` options for **Windows** to **CMake** via ``pip`` options, like
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --no-binary lightgbm --config-settings=cmake.define.USE_GPU=ON --config-settings=cmake.define.OpenCL_INCLUDE_DIR="/usr/local/cuda/include/" --config-settings=cmake.define.OpenCL_LIBRARY="/usr/local/cuda/lib64/libOpenCL.so"
|
||||
|
||||
All available options that can be passed via ``cmake.define.{option}``.
|
||||
|
||||
- BOOST_ROOT
|
||||
|
||||
- Boost_DIR
|
||||
|
||||
- Boost_INCLUDE_DIR
|
||||
|
||||
- BOOST_LIBRARYDIR
|
||||
|
||||
- OpenCL_INCLUDE_DIR
|
||||
|
||||
- OpenCL_LIBRARY
|
||||
|
||||
For more details see `FindBoost <https://cmake.org/cmake/help/latest/module/FindBoost.html>`__ and `FindOpenCL <https://cmake.org/cmake/help/latest/module/FindOpenCL.html>`__.
|
||||
|
||||
Don't confuse with `CUDA version <#build-cuda-version>`__.
|
||||
To use the GPU version within Python, pass ``{"device": "gpu"}`` respectively in parameters.
|
||||
|
||||
|
|
||||
|
||||
Build CUDA Version
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --no-binary lightgbm --config-settings=cmake.define.USE_CUDA=ON
|
||||
|
||||
|
||||
By default, the library will be built with support for a hard-coded list of GPU architectures
|
||||
based on the detected CUDA Toolkit version.
|
||||
|
||||
To build the library with support for more architectures, set ``CMAKE_CUDA_ARCHITECTURES``.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
# example: all Blackwell arches, including DGX Spark
|
||||
pip install \
|
||||
--no-binary lightgbm \
|
||||
--config-settings=cmake.define.USE_CUDA=ON \
|
||||
--config-settings=cmake.define.CMAKE_CUDA_ARCHITECTURES='100;120;121-real;121-virtual' \
|
||||
'lightgbm>=4.7.0'
|
||||
|
||||
# example: just the local GPU
|
||||
pip install \
|
||||
--no-binary lightgbm \
|
||||
--config-settings=cmake.define.USE_CUDA=ON \
|
||||
--config-settings=cmake.define.CMAKE_CUDA_ARCHITECTURES='native' \
|
||||
'lightgbm>=4.7.0'
|
||||
|
||||
All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
|
||||
|
||||
For **macOS** and **Windows** users, the CUDA version is not supported.
|
||||
|
||||
**CUDA** library is needed: details for installation can be found in `Installation Guide <https://github.com/lightgbm-org/LightGBM/blob/main/docs/Installation-Guide.rst#build-cuda-version>`__.
|
||||
|
||||
Don't confuse with `GPU version <#build-gpu-version>`__.
|
||||
To use the CUDA version within Python, pass ``{"device": "cuda"}`` respectively in parameters.
|
||||
|
||||
|
|
||||
|
||||
Build with MinGW-w64 on Windows
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --no-binary lightgbm --config-settings=cmake.define.CMAKE_SH=CMAKE_SH-NOTFOUND --config-settings=cmake.args="-GMinGW Makefiles"
|
||||
|
||||
`MinGW-w64 <https://www.mingw-w64.org/>`_ should be installed first.
|
||||
|
||||
It is recommended to use **Visual Studio** for its better multithreading efficiency in **Windows** for many-core systems
|
||||
(see `Question 4 <https://github.com/lightgbm-org/LightGBM/blob/main/docs/FAQ.rst#4-i-am-using-windows-should-i-use-visual-studio-or-mingw-for-compiling-lightgbm>`__
|
||||
and `Question 8 <https://github.com/lightgbm-org/LightGBM/blob/main/docs/FAQ.rst#8-cpu-usage-is-low-like-10-in-windows-when-using-lightgbm-on-very-large-datasets-with-many-core-systems>`__).
|
||||
|
||||
|
|
||||
|
||||
Build 32-bit Version with 32-bit Python
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --no-binary lightgbm --config-settings=cmake.args="-AWin32"
|
||||
|
||||
For **Windows** users, compilation with **MinGW-w64** is not supported.
|
||||
|
||||
For **macOS** and **Linux** users, the 32-bit version is not supported.
|
||||
|
||||
It is **strongly not recommended** to use this version of LightGBM!
|
||||
|
||||
|
|
||||
|
||||
Build with Time Costs Output
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install lightgbm --no-binary lightgbm --config-settings=cmake.define.USE_TIMETAG=ON
|
||||
|
||||
Use this option to make LightGBM output time costs for different internal routines, to investigate and benchmark its performance.
|
||||
|
||||
|
|
||||
|
||||
Install from `conda-forge channel <https://anaconda.org/conda-forge/lightgbm>`_
|
||||
'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
|
||||
|
||||
``lightgbm`` conda packages are available from the ``conda-forge`` channel.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
conda install -c conda-forge lightgbm
|
||||
|
||||
These packages support **CPU**, **GPU** and **CUDA** versions out of the box.
|
||||
|
||||
**GPU**-enabled version is available only for **Windows** and **Linux** currently.
|
||||
|
||||
**CUDA**-enabled version (since ``lightgbm>=4.4.0``) is available only for **Linux** currently and will be automatically selected if you are on a system where CUDA is installed.
|
||||
|
||||
|
|
||||
|
||||
Install from GitHub
|
||||
'''''''''''''''''''
|
||||
|
||||
All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
git clone --recursive https://github.com/lightgbm-org/LightGBM.git
|
||||
cd LightGBM
|
||||
# export CXX=g++-14 CC=gcc-14 # macOS users, if you decided to compile with gcc, don't forget to specify compilers
|
||||
sh ./build-python.sh install
|
||||
|
||||
Note: ``sudo`` (or administrator rights in **Windows**) may be needed to perform the command.
|
||||
Run ``sh ./build-python.sh install --user`` to install into user-specific instead of global site-packages directory.
|
||||
|
||||
Run ``sh ./build-python.sh install --no-isolation`` to assume all build and install dependencies are already installed, don't go to the internet to get them.
|
||||
|
||||
|
|
||||
|
||||
Run ``sh ./build-python.sh install --nomp`` to disable **OpenMP** support.
|
||||
All requirements from `Build Threadless Version section <#build-threadless-version>`__ apply for this installation option as well.
|
||||
|
||||
Run ``sh ./build-python.sh install --mpi`` to enable **MPI** support.
|
||||
All requirements from `Build MPI Version section <#build-mpi-version>`__ apply for this installation option as well.
|
||||
|
||||
Run ``sh ./build-python.sh install --gpu`` to enable GPU support.
|
||||
All requirements from `Build GPU Version section <#build-gpu-version>`__ apply for this installation option as well.
|
||||
To pass additional options to **CMake** use the following syntax: ``sh ./build-python.sh install --gpu --opencl-include-dir="/usr/local/cuda/include/"``,
|
||||
see `Build GPU Version section <#build-gpu-version>`__ for the complete list of them.
|
||||
|
||||
Run ``sh ./build-python.sh install --cuda`` to enable CUDA support.
|
||||
All requirements from `Build CUDA Version section <#build-cuda-version>`__ apply for this installation option as well.
|
||||
|
||||
Run ``sh ./build-python.sh install --mingw``, if you want to use **MinGW-w64** on **Windows** instead of **Visual Studio**.
|
||||
All requirements from `Build with MinGW-w64 on Windows section <#build-with-mingw-w64-on-windows>`__ apply for this installation option as well.
|
||||
|
||||
Run ``sh ./build-python.sh install --bit32``, if you want to use 32-bit version.
|
||||
All requirements from `Build 32-bit Version with 32-bit Python section <#build-32-bit-version-with-32-bit-python>`__ apply for this installation option as well.
|
||||
|
||||
Run ``sh ./build-python.sh install --time-costs``, if you want to output time costs for different internal routines.
|
||||
All requirements from `Build with Time Costs Output section <#build-with-time-costs-output>`__ apply for this installation option as well.
|
||||
|
||||
|
|
||||
|
||||
If you get any errors during installation or due to any other reasons,
|
||||
you may want to build dynamic library from sources by any method you prefer
|
||||
(see `Installation Guide <https://github.com/lightgbm-org/LightGBM/blob/main/docs/Installation-Guide.rst>`__).
|
||||
For example, you can use ``MSBuild`` tool and `solution file <https://github.com/lightgbm-org/LightGBM/blob/main/windows/LightGBM.sln>`__ from the repo.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
MSBuild.exe windows/LightGBM.sln /p:Configuration=DLL /p:Platform=x64 /p:PlatformToolset=v143
|
||||
|
||||
After compiling dynamic library just run ``sh ./build-python.sh install --precompile`` to install the Python-package using that library.
|
||||
|
||||
|
|
||||
|
||||
Build Wheel File
|
||||
****************
|
||||
|
||||
You can run ``sh ./build-python.sh bdist_wheel`` to build a wheel file but not install it.
|
||||
|
||||
That script requires some dependencies like ``build``, ``scikit-build-core``, and ``wheel``.
|
||||
In environments with restricted or no internet access, install those tools and then pass ``--no-isolation``.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
sh ./build-python.sh bdist_wheel --no-isolation
|
||||
|
||||
Troubleshooting
|
||||
---------------
|
||||
|
||||
Refer to `FAQ <https://github.com/lightgbm-org/LightGBM/tree/main/docs/FAQ.rst>`_.
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Refer to the walk through examples in `Python guide folder <https://github.com/lightgbm-org/LightGBM/tree/main/examples/python-guide>`_.
|
||||
|
||||
Supported Python Versions
|
||||
-------------------------
|
||||
|
||||
This project supports all Python versions until they reach end-of-life.
|
||||
For details on the support calendar for Python versions, see https://devguide.python.org/versions/.
|
||||
|
||||
Development Guide
|
||||
-----------------
|
||||
|
||||
To check that a contribution to the package matches its style expectations, run the following from the root of the repo.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pre-commit run --all-files
|
||||
|
||||
To run the tests locally and compute test coverage, install the Python package using one of the options mentioned above.
|
||||
Then run the following from the root of the repo.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pytest \
|
||||
--cov=lightgbm \
|
||||
--cov-report="term" \
|
||||
--cov-report="html:htmlcov" \
|
||||
tests/python_package_test/
|
||||
|
||||
Then open `htmlcov/index.html` to view a clickable coverage report.
|
||||
|
||||
.. |License| image:: https://img.shields.io/github/license/lightgbm-org/lightgbm.svg
|
||||
:target: https://github.com/lightgbm-org/LightGBM/blob/main/LICENSE
|
||||
.. |Python Versions| image:: https://img.shields.io/pypi/pyversions/lightgbm.svg?logo=python&logoColor=white
|
||||
:target: https://pypi.org/project/lightgbm
|
||||
.. |PyPI Version| image:: https://img.shields.io/pypi/v/lightgbm.svg?logo=pypi&logoColor=white
|
||||
:target: https://pypi.org/project/lightgbm
|
||||
.. |PyPI Downloads| image:: https://img.shields.io/pepy/dt/lightgbm?logo=pypi&logoColor=white&label=pypi%20downloads
|
||||
:target: https://pepy.tech/project/lightgbm
|
||||
.. |conda Version| image:: https://img.shields.io/conda/vn/conda-forge/lightgbm?logo=conda-forge&logoColor=white&label=conda
|
||||
:target: https://anaconda.org/conda-forge/lightgbm
|
||||
.. |conda Downloads| image:: https://img.shields.io/conda/d/conda-forge/lightgbm?logo=conda-forge&logoColor=white&label=conda%20downloads
|
||||
:target: https://anaconda.org/conda-forge/lightgbm/files
|
||||
.. |API Docs| image:: https://readthedocs.org/projects/lightgbm/badge/?version=latest
|
||||
:target: https://lightgbm.readthedocs.io/en/latest/Python-API.html
|
||||
@@ -0,0 +1,58 @@
|
||||
# coding: utf-8
|
||||
"""LightGBM, Light Gradient Boosting Machine.
|
||||
|
||||
Contributors: https://github.com/lightgbm-org/LightGBM/graphs/contributors.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
# .basic is intentionally loaded as early as possible, to dlopen() lib_lightgbm.{dll,dylib,so}
|
||||
# and its dependencies as early as possible
|
||||
from .basic import Booster, Dataset, Sequence, register_logger
|
||||
from .callback import EarlyStopException, early_stopping, log_evaluation, record_evaluation, reset_parameter
|
||||
from .engine import CVBooster, cv, train
|
||||
|
||||
try:
|
||||
from .sklearn import LGBMClassifier, LGBMModel, LGBMRanker, LGBMRegressor
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
from .plotting import create_tree_digraph, plot_importance, plot_metric, plot_split_value_histogram, plot_tree
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
from .dask import DaskLGBMClassifier, DaskLGBMRanker, DaskLGBMRegressor
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
_version_path = Path(__file__).resolve().parent / "VERSION.txt"
|
||||
if _version_path.is_file():
|
||||
__version__ = _version_path.read_text(encoding="utf-8").strip()
|
||||
|
||||
__all__ = [
|
||||
"Dataset",
|
||||
"Booster",
|
||||
"CVBooster",
|
||||
"Sequence",
|
||||
"register_logger",
|
||||
"train",
|
||||
"cv",
|
||||
"LGBMModel",
|
||||
"LGBMRegressor",
|
||||
"LGBMClassifier",
|
||||
"LGBMRanker",
|
||||
"DaskLGBMRegressor",
|
||||
"DaskLGBMClassifier",
|
||||
"DaskLGBMRanker",
|
||||
"log_evaluation",
|
||||
"record_evaluation",
|
||||
"reset_parameter",
|
||||
"early_stopping",
|
||||
"EarlyStopException",
|
||||
"plot_importance",
|
||||
"plot_split_value_histogram",
|
||||
"plot_metric",
|
||||
"plot_tree",
|
||||
"create_tree_digraph",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,511 @@
|
||||
# coding: utf-8
|
||||
"""Callbacks library."""
|
||||
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from .basic import (
|
||||
Booster,
|
||||
_ConfigAliases,
|
||||
_LGBM_BoosterEvalMethodResultType,
|
||||
_LGBM_BoosterEvalMethodResultWithStandardDeviationType,
|
||||
_log_info,
|
||||
_log_warning,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .engine import CVBooster
|
||||
|
||||
__all__ = [
|
||||
"EarlyStopException",
|
||||
"early_stopping",
|
||||
"log_evaluation",
|
||||
"record_evaluation",
|
||||
"reset_parameter",
|
||||
]
|
||||
|
||||
_EvalResultDict = Dict[str, Dict[str, List[Any]]]
|
||||
_EvalResultTuple = Union[
|
||||
_LGBM_BoosterEvalMethodResultType,
|
||||
_LGBM_BoosterEvalMethodResultWithStandardDeviationType,
|
||||
]
|
||||
_ListOfEvalResultTuples = Union[
|
||||
List[_LGBM_BoosterEvalMethodResultType],
|
||||
List[_LGBM_BoosterEvalMethodResultWithStandardDeviationType],
|
||||
]
|
||||
|
||||
|
||||
class EarlyStopException(Exception):
|
||||
"""Exception of early stopping.
|
||||
|
||||
Raise this from a callback passed in via keyword argument ``callbacks``
|
||||
in ``cv()`` or ``train()`` to trigger early stopping.
|
||||
"""
|
||||
|
||||
def __init__(self, best_iteration: int, best_score: _ListOfEvalResultTuples) -> None:
|
||||
"""Create early stopping exception.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
best_iteration : int
|
||||
The best iteration stopped.
|
||||
0-based... pass ``best_iteration=2`` to indicate that the third iteration was the best one.
|
||||
best_score : list of (dataset_name, metric_name, metric_value, maximize) tuple or (dataset_name, metric_name, metric_value, maximize, metric_std_dev) tuple
|
||||
Scores for each metric, on each validation set, as of the best iteration.
|
||||
"""
|
||||
super().__init__()
|
||||
self.best_iteration = best_iteration
|
||||
self.best_score = best_score
|
||||
|
||||
|
||||
# Callback environment used by callbacks
|
||||
@dataclass
|
||||
class CallbackEnv:
|
||||
model: Union[Booster, "CVBooster"]
|
||||
params: Dict[str, Any]
|
||||
iteration: int
|
||||
begin_iteration: int
|
||||
end_iteration: int
|
||||
evaluation_result_list: Optional[_ListOfEvalResultTuples]
|
||||
|
||||
|
||||
def _is_using_cv(env: CallbackEnv) -> bool:
|
||||
"""Check if model in callback env is a CVBooster."""
|
||||
# this import is here to avoid a circular import
|
||||
from .engine import CVBooster # noqa: PLC0415
|
||||
|
||||
return isinstance(env.model, CVBooster)
|
||||
|
||||
|
||||
def _format_eval_result(value: _EvalResultTuple, show_stdv: bool) -> str:
|
||||
"""Format metric string."""
|
||||
dataset_name, metric_name, metric_value, *_ = value
|
||||
out = f"{dataset_name}'s {metric_name}: {metric_value:g}"
|
||||
# tuples from cv() sometimes have a 5th item, with standard deviation of
|
||||
# the evaluation metric (taken over all cross-validation folds)
|
||||
if show_stdv and len(value) == 5:
|
||||
out += f" + {value[4]:g}"
|
||||
return out
|
||||
|
||||
|
||||
class _LogEvaluationCallback:
|
||||
"""Internal log evaluation callable class."""
|
||||
|
||||
def __init__(self, period: int = 1, show_stdv: bool = True) -> None:
|
||||
self.order = 10
|
||||
self.before_iteration = False
|
||||
|
||||
self.period = period
|
||||
self.show_stdv = show_stdv
|
||||
|
||||
def __call__(self, env: CallbackEnv) -> None:
|
||||
if self.period > 0 and env.evaluation_result_list and (env.iteration + 1) % self.period == 0:
|
||||
result = "\t".join([_format_eval_result(x, self.show_stdv) for x in env.evaluation_result_list])
|
||||
_log_info(f"[{env.iteration + 1}]\t{result}")
|
||||
|
||||
|
||||
def log_evaluation(period: int = 1, show_stdv: bool = True) -> _LogEvaluationCallback:
|
||||
"""Create a callback that logs the evaluation results.
|
||||
|
||||
By default, standard output resource is used.
|
||||
Use ``register_logger()`` function to register a custom logger.
|
||||
|
||||
Note
|
||||
----
|
||||
Requires at least one validation data.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
period : int, optional (default=1)
|
||||
The period to log the evaluation results.
|
||||
The last boosting stage or the boosting stage found by using ``early_stopping`` callback is also logged.
|
||||
show_stdv : bool, optional (default=True)
|
||||
Whether to log stdv (if provided).
|
||||
|
||||
Returns
|
||||
-------
|
||||
callback : _LogEvaluationCallback
|
||||
The callback that logs the evaluation results every ``period`` boosting iteration(s).
|
||||
"""
|
||||
return _LogEvaluationCallback(period=period, show_stdv=show_stdv)
|
||||
|
||||
|
||||
class _RecordEvaluationCallback:
|
||||
"""Internal record evaluation callable class."""
|
||||
|
||||
def __init__(self, eval_result: _EvalResultDict) -> None:
|
||||
self.order = 20
|
||||
self.before_iteration = False
|
||||
|
||||
if not isinstance(eval_result, dict):
|
||||
raise TypeError("eval_result should be a dictionary")
|
||||
self.eval_result = eval_result
|
||||
|
||||
def _init(self, env: CallbackEnv) -> None:
|
||||
if env.evaluation_result_list is None:
|
||||
raise RuntimeError(
|
||||
"record_evaluation() callback enabled but no evaluation results found. This is a probably bug in LightGBM. "
|
||||
"Please report it at https://github.com/lightgbm-org/LightGBM/issues"
|
||||
)
|
||||
self.eval_result.clear()
|
||||
for item in env.evaluation_result_list:
|
||||
dataset_name, metric_name, *_ = item
|
||||
self.eval_result.setdefault(dataset_name, OrderedDict())
|
||||
if len(item) == 4:
|
||||
self.eval_result[dataset_name].setdefault(metric_name, [])
|
||||
else:
|
||||
self.eval_result[dataset_name].setdefault(f"{metric_name}-mean", [])
|
||||
self.eval_result[dataset_name].setdefault(f"{metric_name}-stdv", [])
|
||||
|
||||
def __call__(self, env: CallbackEnv) -> None:
|
||||
if env.iteration == env.begin_iteration:
|
||||
self._init(env)
|
||||
if env.evaluation_result_list is None:
|
||||
raise RuntimeError(
|
||||
"record_evaluation() callback enabled but no evaluation results found. This is a probably bug in LightGBM. "
|
||||
"Please report it at https://github.com/lightgbm-org/LightGBM/issues"
|
||||
)
|
||||
for item in env.evaluation_result_list:
|
||||
# for cv(), 'metric_value' is actually a mean of metric values over all CV folds
|
||||
dataset_name, metric_name, metric_value, *_ = item
|
||||
if len(item) == 4:
|
||||
# train()
|
||||
self.eval_result[dataset_name][metric_name].append(metric_value)
|
||||
else:
|
||||
# cv()
|
||||
metric_std_dev = item[4] # type: ignore[misc]
|
||||
self.eval_result[dataset_name][f"{metric_name}-mean"].append(metric_value)
|
||||
self.eval_result[dataset_name][f"{metric_name}-stdv"].append(metric_std_dev)
|
||||
|
||||
|
||||
def record_evaluation(eval_result: Dict[str, Dict[str, List[Any]]]) -> Callable:
|
||||
"""Create a callback that records the evaluation history into ``eval_result``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
eval_result : dict
|
||||
Dictionary used to store all evaluation results of all validation sets.
|
||||
This should be initialized outside of your call to ``record_evaluation()`` and should be empty.
|
||||
Any initial contents of the dictionary will be deleted.
|
||||
|
||||
.. rubric:: Example
|
||||
|
||||
With two validation sets named 'eval' and 'train', and one evaluation metric named 'logloss'
|
||||
this dictionary after finishing a model training process will have the following structure:
|
||||
|
||||
.. code-block::
|
||||
|
||||
{
|
||||
'train':
|
||||
{
|
||||
'logloss': [0.48253, 0.35953, ...]
|
||||
},
|
||||
'eval':
|
||||
{
|
||||
'logloss': [0.480385, 0.357756, ...]
|
||||
}
|
||||
}
|
||||
|
||||
Returns
|
||||
-------
|
||||
callback : _RecordEvaluationCallback
|
||||
The callback that records the evaluation history into the passed dictionary.
|
||||
"""
|
||||
return _RecordEvaluationCallback(eval_result=eval_result)
|
||||
|
||||
|
||||
class _ResetParameterCallback:
|
||||
"""Internal reset parameter callable class."""
|
||||
|
||||
def __init__(self, **kwargs: Union[list, Callable]) -> None:
|
||||
self.order = 10
|
||||
self.before_iteration = True
|
||||
|
||||
self.kwargs = kwargs
|
||||
|
||||
def __call__(self, env: CallbackEnv) -> None:
|
||||
new_parameters = {}
|
||||
for key, value in self.kwargs.items():
|
||||
if isinstance(value, list):
|
||||
if len(value) != env.end_iteration - env.begin_iteration:
|
||||
raise ValueError(f"Length of list {key!r} has to be equal to 'num_boost_round'.")
|
||||
new_param = value[env.iteration - env.begin_iteration]
|
||||
elif callable(value):
|
||||
new_param = value(env.iteration - env.begin_iteration)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Only list and callable values are supported "
|
||||
"as a mapping from boosting round index to new parameter value."
|
||||
)
|
||||
if new_param != env.params.get(key, None):
|
||||
new_parameters[key] = new_param
|
||||
if new_parameters:
|
||||
if isinstance(env.model, Booster):
|
||||
env.model.reset_parameter(new_parameters)
|
||||
else:
|
||||
# CVBooster holds a list of Booster objects, each needs to be updated
|
||||
for booster in env.model.boosters:
|
||||
booster.reset_parameter(new_parameters)
|
||||
env.params.update(new_parameters)
|
||||
|
||||
|
||||
def reset_parameter(**kwargs: Union[list, Callable]) -> Callable:
|
||||
"""Create a callback that resets the parameter after the first iteration.
|
||||
|
||||
.. note::
|
||||
|
||||
The initial parameter will still take in-effect on first iteration.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
**kwargs : value should be list or callable
|
||||
List of parameters for each boosting round
|
||||
or a callable that calculates the parameter in terms of
|
||||
current number of round (e.g. yields learning rate decay).
|
||||
If list lst, parameter = lst[current_round].
|
||||
If callable func, parameter = func(current_round).
|
||||
|
||||
Returns
|
||||
-------
|
||||
callback : _ResetParameterCallback
|
||||
The callback that resets the parameter after the first iteration.
|
||||
"""
|
||||
return _ResetParameterCallback(**kwargs)
|
||||
|
||||
|
||||
class _EarlyStoppingCallback:
|
||||
"""Internal early stopping callable class."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
stopping_rounds: int,
|
||||
first_metric_only: bool = False,
|
||||
verbose: bool = True,
|
||||
min_delta: Union[float, List[float]] = 0.0,
|
||||
) -> None:
|
||||
self.enabled = _should_enable_early_stopping(stopping_rounds)
|
||||
|
||||
self.order = 30
|
||||
self.before_iteration = False
|
||||
|
||||
self.stopping_rounds = stopping_rounds
|
||||
self.first_metric_only = first_metric_only
|
||||
self.verbose = verbose
|
||||
self.min_delta = min_delta
|
||||
|
||||
self._reset_storages()
|
||||
|
||||
def _reset_storages(self) -> None:
|
||||
self.best_score: List[float] = []
|
||||
self.best_iter: List[int] = []
|
||||
self.best_score_list: List[_ListOfEvalResultTuples] = []
|
||||
self.cmp_op: List[Callable[[float, float, float], bool]] = []
|
||||
self.first_metric = ""
|
||||
|
||||
def _gt_delta(self, *, curr_score: float, best_score: float, delta: float) -> bool:
|
||||
return curr_score > best_score + delta
|
||||
|
||||
def _lt_delta(self, *, curr_score: float, best_score: float, delta: float) -> bool:
|
||||
return curr_score < best_score - delta
|
||||
|
||||
def _is_train_set(self, *, dataset_name: str, env: CallbackEnv) -> bool:
|
||||
"""Check, by name, if a given Dataset is the training data."""
|
||||
# for lgb.cv() with eval_train_metric=True, evaluation is also done on the training set
|
||||
# and those metrics are considered for early stopping
|
||||
if _is_using_cv(env) and dataset_name == "train":
|
||||
return True
|
||||
|
||||
# for lgb.train(), it's possible to pass the training data via valid_sets with any name
|
||||
if isinstance(env.model, Booster) and dataset_name == env.model._train_data_name:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _init(self, env: CallbackEnv) -> None:
|
||||
if env.evaluation_result_list is None or env.evaluation_result_list == []:
|
||||
raise ValueError("For early stopping, at least one dataset and eval metric is required for evaluation")
|
||||
|
||||
is_dart = any(env.params.get(alias, "") == "dart" for alias in _ConfigAliases.get("boosting"))
|
||||
if is_dart:
|
||||
self.enabled = False
|
||||
_log_warning("Early stopping is not available in dart mode")
|
||||
return
|
||||
|
||||
# get details of the first dataset
|
||||
first_dataset_name, first_metric_name, *_ = env.evaluation_result_list[0]
|
||||
|
||||
# validation sets are guaranteed to not be identical to the training data in cv()
|
||||
if isinstance(env.model, Booster):
|
||||
only_train_set = len(env.evaluation_result_list) == 1 and self._is_train_set(
|
||||
dataset_name=first_dataset_name,
|
||||
env=env,
|
||||
)
|
||||
if only_train_set:
|
||||
self.enabled = False
|
||||
_log_warning("Only training set found, disabling early stopping.")
|
||||
return
|
||||
|
||||
if self.verbose:
|
||||
_log_info(f"Training until validation scores don't improve for {self.stopping_rounds} rounds")
|
||||
|
||||
self._reset_storages()
|
||||
|
||||
n_metrics = len({m[1] for m in env.evaluation_result_list})
|
||||
n_datasets = len(env.evaluation_result_list) // n_metrics
|
||||
if isinstance(self.min_delta, list):
|
||||
if not all(t >= 0 for t in self.min_delta):
|
||||
raise ValueError("Values for early stopping min_delta must be non-negative.")
|
||||
if len(self.min_delta) == 0:
|
||||
if self.verbose:
|
||||
_log_info("Disabling min_delta for early stopping.")
|
||||
deltas = [0.0] * n_datasets * n_metrics
|
||||
elif len(self.min_delta) == 1:
|
||||
if self.verbose:
|
||||
_log_info(f"Using {self.min_delta[0]} as min_delta for all metrics.")
|
||||
deltas = self.min_delta * n_datasets * n_metrics
|
||||
else:
|
||||
if len(self.min_delta) != n_metrics:
|
||||
raise ValueError("Must provide a single value for min_delta or as many as metrics.")
|
||||
if self.first_metric_only and self.verbose:
|
||||
_log_info(f"Using only {self.min_delta[0]} as early stopping min_delta.")
|
||||
deltas = self.min_delta * n_datasets
|
||||
else:
|
||||
if self.min_delta < 0:
|
||||
raise ValueError("Early stopping min_delta must be non-negative.")
|
||||
if self.min_delta > 0 and n_metrics > 1 and not self.first_metric_only and self.verbose:
|
||||
_log_info(f"Using {self.min_delta} as min_delta for all metrics.")
|
||||
deltas = [self.min_delta] * n_datasets * n_metrics
|
||||
|
||||
self.first_metric = first_metric_name
|
||||
for eval_ret, delta in zip(env.evaluation_result_list, deltas, strict=True):
|
||||
self.best_iter.append(0)
|
||||
if eval_ret[3]: # greater is better
|
||||
self.best_score.append(float("-inf"))
|
||||
self.cmp_op.append(partial(self._gt_delta, delta=delta))
|
||||
else:
|
||||
self.best_score.append(float("inf"))
|
||||
self.cmp_op.append(partial(self._lt_delta, delta=delta))
|
||||
|
||||
def _final_iteration_check(self, *, env: CallbackEnv, metric_name: str, i: int) -> None:
|
||||
if env.iteration == env.end_iteration - 1:
|
||||
if self.verbose:
|
||||
best_score_str = "\t".join([_format_eval_result(x, show_stdv=True) for x in self.best_score_list[i]])
|
||||
_log_info(
|
||||
f"Did not meet early stopping. Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}"
|
||||
)
|
||||
if self.first_metric_only:
|
||||
_log_info(f"Evaluated only: {metric_name}")
|
||||
raise EarlyStopException(
|
||||
best_iteration=self.best_iter[i],
|
||||
best_score=self.best_score_list[i],
|
||||
)
|
||||
|
||||
def __call__(self, env: CallbackEnv) -> None:
|
||||
if env.iteration == env.begin_iteration:
|
||||
self._init(env)
|
||||
if not self.enabled:
|
||||
return
|
||||
if env.evaluation_result_list is None:
|
||||
raise RuntimeError(
|
||||
"early_stopping() callback enabled but no evaluation results found. This is a probably bug in LightGBM. "
|
||||
"Please report it at https://github.com/lightgbm-org/LightGBM/issues"
|
||||
)
|
||||
# self.best_score_list is initialized to an empty list
|
||||
first_time_updating_best_score_list = self.best_score_list == []
|
||||
for i in range(len(env.evaluation_result_list)):
|
||||
dataset_name, metric_name, metric_value, *_ = env.evaluation_result_list[i]
|
||||
if first_time_updating_best_score_list or self.cmp_op[i]( # type: ignore[call-arg]
|
||||
curr_score=metric_value, best_score=self.best_score[i]
|
||||
):
|
||||
self.best_score[i] = metric_value
|
||||
self.best_iter[i] = env.iteration
|
||||
if first_time_updating_best_score_list:
|
||||
self.best_score_list.append(env.evaluation_result_list)
|
||||
else:
|
||||
self.best_score_list[i] = env.evaluation_result_list
|
||||
if self.first_metric_only and self.first_metric != metric_name:
|
||||
continue # use only the first metric for early stopping
|
||||
if self._is_train_set(
|
||||
dataset_name=dataset_name,
|
||||
env=env,
|
||||
):
|
||||
continue # train data for lgb.cv or sklearn wrapper (underlying lgb.train)
|
||||
elif env.iteration - self.best_iter[i] >= self.stopping_rounds:
|
||||
if self.verbose:
|
||||
eval_result_str = "\t".join(
|
||||
[_format_eval_result(x, show_stdv=True) for x in self.best_score_list[i]]
|
||||
)
|
||||
_log_info(f"Early stopping, best iteration is:\n[{self.best_iter[i] + 1}]\t{eval_result_str}")
|
||||
if self.first_metric_only:
|
||||
_log_info(f"Evaluated only: {metric_name}")
|
||||
raise EarlyStopException(
|
||||
best_iteration=self.best_iter[i],
|
||||
best_score=self.best_score_list[i],
|
||||
)
|
||||
self._final_iteration_check(env=env, metric_name=metric_name, i=i)
|
||||
|
||||
|
||||
def _should_enable_early_stopping(stopping_rounds: Any) -> bool:
|
||||
"""Check if early stopping should be activated.
|
||||
|
||||
This function will evaluate to True if the early stopping callback should be
|
||||
activated (i.e. stopping_rounds > 0). It also provides an informative error if the
|
||||
type is not int.
|
||||
"""
|
||||
if not isinstance(stopping_rounds, int):
|
||||
raise TypeError(f"early_stopping_round should be an integer. Got '{type(stopping_rounds).__name__}'")
|
||||
return stopping_rounds > 0
|
||||
|
||||
|
||||
def early_stopping(
|
||||
stopping_rounds: int,
|
||||
first_metric_only: bool = False,
|
||||
verbose: bool = True,
|
||||
min_delta: Union[float, List[float]] = 0.0,
|
||||
) -> _EarlyStoppingCallback:
|
||||
"""Create a callback that activates early stopping.
|
||||
|
||||
Activates early stopping.
|
||||
The model will train until the validation score doesn't improve by at least ``min_delta``.
|
||||
Validation score needs to improve at least every ``stopping_rounds`` round(s)
|
||||
to continue training.
|
||||
Requires at least one validation data and one metric.
|
||||
If there's more than one, will check all of them. But the training data is ignored anyway.
|
||||
To check only the first metric set ``first_metric_only`` to True.
|
||||
The index of iteration that has the best performance will be saved in the ``best_iteration`` attribute of a model.
|
||||
|
||||
.. note::
|
||||
|
||||
If using ``boosting_type="dart"``, this callback has no effect and early stopping
|
||||
will not be performed.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
stopping_rounds : int
|
||||
The possible number of rounds without the trend occurrence.
|
||||
first_metric_only : bool, optional (default=False)
|
||||
Whether to use only the first metric for early stopping.
|
||||
verbose : bool, optional (default=True)
|
||||
Whether to log message with early stopping information.
|
||||
By default, standard output resource is used.
|
||||
Use ``register_logger()`` function to register a custom logger.
|
||||
min_delta : float or list of float, optional (default=0.0)
|
||||
Minimum improvement in score to keep training.
|
||||
If float, this single value is used for all metrics.
|
||||
If list, its length should match the total number of metrics.
|
||||
|
||||
.. versionadded:: 4.0.0
|
||||
|
||||
Returns
|
||||
-------
|
||||
callback : _EarlyStoppingCallback
|
||||
The callback that activates early stopping.
|
||||
"""
|
||||
return _EarlyStoppingCallback(
|
||||
stopping_rounds=stopping_rounds,
|
||||
first_metric_only=first_metric_only,
|
||||
verbose=verbose,
|
||||
min_delta=min_delta,
|
||||
)
|
||||
@@ -0,0 +1,223 @@
|
||||
# coding: utf-8
|
||||
"""Compatibility library."""
|
||||
|
||||
import inspect
|
||||
from typing import TYPE_CHECKING, Any, List
|
||||
|
||||
# scikit-learn is intentionally imported first here,
|
||||
# see https://github.com/lightgbm-org/LightGBM/issues/6509
|
||||
"""sklearn"""
|
||||
try:
|
||||
from sklearn import __version__ as _sklearn_version
|
||||
from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
from sklearn.utils.class_weight import compute_sample_weight
|
||||
from sklearn.utils.multiclass import check_classification_targets
|
||||
from sklearn.utils.validation import assert_all_finite, check_array, check_X_y
|
||||
|
||||
try:
|
||||
from sklearn.exceptions import NotFittedError
|
||||
from sklearn.model_selection import BaseCrossValidator, GroupKFold, StratifiedKFold
|
||||
except ImportError:
|
||||
from sklearn.cross_validation import BaseCrossValidator, GroupKFold, StratifiedKFold
|
||||
from sklearn.utils.validation import NotFittedError
|
||||
try:
|
||||
from sklearn.utils.validation import _check_sample_weight
|
||||
|
||||
# As of https://github.com/scikit-learn/scikit-learn/pull/32212, scikit-learn started raising an error
|
||||
# when sample weights are all 0. This argument allow_all_zero_weights can be used switch back
|
||||
# to the old behavior of allowing them.
|
||||
#
|
||||
# This can be removed when the minimum scikit-learn version supported here is v1.9.
|
||||
SKLEARN_CHECK_SAMPLE_WEIGHT_HAS_ALLOW_ZERO_WEIGHTS_ARG = (
|
||||
"allow_all_zero_weights" in inspect.signature(_check_sample_weight).parameters
|
||||
)
|
||||
|
||||
except ImportError:
|
||||
from sklearn.utils.validation import check_consistent_length
|
||||
|
||||
SKLEARN_CHECK_SAMPLE_WEIGHT_HAS_ALLOW_ZERO_WEIGHTS_ARG = False
|
||||
|
||||
# dummy function to support older version of scikit-learn
|
||||
def _check_sample_weight(sample_weight: Any, X: Any, dtype: Any = None) -> Any:
|
||||
check_consistent_length(sample_weight, X)
|
||||
return sample_weight
|
||||
|
||||
try:
|
||||
from sklearn.utils.validation import validate_data
|
||||
except ImportError:
|
||||
# validate_data() was added in scikit-learn 1.6, this function roughly imitates it for older versions.
|
||||
# It can be removed when lightgbm's minimum scikit-learn version is at least 1.6.
|
||||
def validate_data(
|
||||
_estimator: Any,
|
||||
X: Any,
|
||||
y: Any = "no_validation",
|
||||
accept_sparse: bool = True,
|
||||
# 'force_all_finite' was renamed to 'ensure_all_finite' in scikit-learn 1.6
|
||||
ensure_all_finite: bool = False,
|
||||
ensure_min_samples: int = 1,
|
||||
# trap other keyword arguments that only work on scikit-learn >=1.6, like 'reset'
|
||||
**ignored_kwargs: Any,
|
||||
) -> Any:
|
||||
# it's safe to import _num_features unconditionally because:
|
||||
#
|
||||
# * it was first added in scikit-learn 0.24.2
|
||||
# * lightgbm cannot be used with scikit-learn versions older than that
|
||||
# * this validate_data() re-implementation will not be called in scikit-learn>=1.6
|
||||
#
|
||||
from sklearn.utils.validation import _num_features # noqa: PLC0415
|
||||
|
||||
# _num_features() raises a TypeError on 1-dimensional input. That's a problem
|
||||
# because scikit-learn's 'check_fit1d' estimator check sets that expectation that
|
||||
# estimators must raise a ValueError when a 1-dimensional input is passed to fit().
|
||||
#
|
||||
# So here, lightgbm avoids calling _num_features() on 1-dimensional inputs.
|
||||
if hasattr(X, "shape") and len(X.shape) == 1:
|
||||
n_features_in_ = 1
|
||||
else:
|
||||
n_features_in_ = _num_features(X)
|
||||
|
||||
no_val_y = isinstance(y, str) and y == "no_validation"
|
||||
|
||||
# NOTE: check_X_y() calls check_array() internally, so only need to call one or the other of them here
|
||||
if no_val_y:
|
||||
X = check_array(
|
||||
X,
|
||||
accept_sparse=accept_sparse,
|
||||
force_all_finite=ensure_all_finite,
|
||||
ensure_min_samples=ensure_min_samples,
|
||||
)
|
||||
else:
|
||||
X, y = check_X_y(
|
||||
X,
|
||||
y,
|
||||
accept_sparse=accept_sparse,
|
||||
force_all_finite=ensure_all_finite,
|
||||
ensure_min_samples=ensure_min_samples,
|
||||
)
|
||||
|
||||
# this only needs to be updated at fit() time
|
||||
_estimator.n_features_in_ = n_features_in_
|
||||
|
||||
# raise the same error that scikit-learn's `validate_data()` does on scikit-learn>=1.6
|
||||
if _estimator.__sklearn_is_fitted__() and _estimator._n_features != n_features_in_:
|
||||
raise ValueError(
|
||||
f"X has {n_features_in_} features, but {_estimator.__class__.__name__} "
|
||||
f"is expecting {_estimator._n_features} features as input."
|
||||
)
|
||||
|
||||
if no_val_y:
|
||||
return X
|
||||
else:
|
||||
return X, y
|
||||
|
||||
SKLEARN_INSTALLED = True
|
||||
_LGBMBaseCrossValidator = BaseCrossValidator
|
||||
_LGBMModelBase = BaseEstimator
|
||||
_LGBMRegressorBase = RegressorMixin
|
||||
_LGBMClassifierBase = ClassifierMixin
|
||||
_LGBMLabelEncoder = LabelEncoder
|
||||
LGBMNotFittedError = NotFittedError
|
||||
_LGBMStratifiedKFold = StratifiedKFold
|
||||
_LGBMGroupKFold = GroupKFold
|
||||
_LGBMCheckSampleWeight = _check_sample_weight
|
||||
_LGBMAssertAllFinite = assert_all_finite
|
||||
_LGBMCheckClassificationTargets = check_classification_targets
|
||||
_LGBMComputeSampleWeight = compute_sample_weight
|
||||
_LGBMValidateData = validate_data
|
||||
except ImportError:
|
||||
SKLEARN_INSTALLED = False
|
||||
SKLEARN_CHECK_SAMPLE_WEIGHT_HAS_ALLOW_ZERO_WEIGHTS_ARG = False
|
||||
|
||||
class _LGBMModelBase: # type: ignore
|
||||
"""Dummy class for sklearn.base.BaseEstimator."""
|
||||
|
||||
pass
|
||||
|
||||
class _LGBMClassifierBase: # type: ignore
|
||||
"""Dummy class for sklearn.base.ClassifierMixin."""
|
||||
|
||||
pass
|
||||
|
||||
class _LGBMRegressorBase: # type: ignore
|
||||
"""Dummy class for sklearn.base.RegressorMixin."""
|
||||
|
||||
pass
|
||||
|
||||
_LGBMBaseCrossValidator = None
|
||||
_LGBMLabelEncoder = None
|
||||
LGBMNotFittedError = ValueError
|
||||
_LGBMStratifiedKFold = None
|
||||
_LGBMGroupKFold = None
|
||||
_LGBMCheckSampleWeight = None
|
||||
_LGBMAssertAllFinite = None
|
||||
_LGBMCheckClassificationTargets = None
|
||||
_LGBMComputeSampleWeight = None
|
||||
_LGBMValidateData = None
|
||||
_sklearn_version = None
|
||||
|
||||
# additional scikit-learn imports only for type hints
|
||||
if TYPE_CHECKING:
|
||||
# sklearn.utils.Tags can be imported unconditionally once
|
||||
# lightgbm's minimum scikit-learn version is 1.6 or higher
|
||||
try:
|
||||
from sklearn.utils import Tags as _sklearn_Tags
|
||||
except ImportError:
|
||||
_sklearn_Tags = None
|
||||
|
||||
"""pandas"""
|
||||
try:
|
||||
from pandas import DataFrame as pd_DataFrame
|
||||
from pandas import Series as pd_Series
|
||||
from pandas import concat
|
||||
|
||||
try:
|
||||
from pandas import CategoricalDtype as pd_CategoricalDtype
|
||||
except ImportError:
|
||||
from pandas.api.types import CategoricalDtype as pd_CategoricalDtype
|
||||
PANDAS_INSTALLED = True
|
||||
except ImportError:
|
||||
PANDAS_INSTALLED = False
|
||||
|
||||
class pd_Series: # type: ignore
|
||||
"""Dummy class for pandas.Series."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
pass
|
||||
|
||||
class pd_DataFrame: # type: ignore
|
||||
"""Dummy class for pandas.DataFrame."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
pass
|
||||
|
||||
class pd_CategoricalDtype: # type: ignore
|
||||
"""Dummy class for pandas.CategoricalDtype."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
pass
|
||||
|
||||
concat = None
|
||||
|
||||
"""cpu_count()"""
|
||||
|
||||
|
||||
def _LGBMCpuCount(only_physical_cores: bool = True) -> int:
|
||||
ret: int
|
||||
try:
|
||||
from joblib import cpu_count # noqa: I001,PLC0415
|
||||
|
||||
ret = cpu_count(only_physical_cores=only_physical_cores)
|
||||
except ImportError:
|
||||
try:
|
||||
from psutil import cpu_count # noqa: I001,PLC0415
|
||||
|
||||
ret = cpu_count(logical=not only_physical_cores) or 1
|
||||
except ImportError:
|
||||
from multiprocessing import cpu_count # noqa: I001,PLC0415
|
||||
|
||||
ret = cpu_count()
|
||||
return ret
|
||||
|
||||
|
||||
__all__: List[str] = []
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,872 @@
|
||||
# coding: utf-8
|
||||
"""Library with training routines of LightGBM."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from collections import OrderedDict, defaultdict
|
||||
from operator import attrgetter
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
from . import callback
|
||||
from .basic import (
|
||||
Booster,
|
||||
Dataset,
|
||||
LightGBMError,
|
||||
_choose_param_value,
|
||||
_ConfigAliases,
|
||||
_InnerPredictor,
|
||||
_LGBM_BoosterEvalMethodResultType,
|
||||
_LGBM_BoosterEvalMethodResultWithStandardDeviationType,
|
||||
_LGBM_CustomObjectiveFunction,
|
||||
_LGBM_EvalFunctionResultType,
|
||||
_log_warning,
|
||||
)
|
||||
from .compat import SKLEARN_INSTALLED, _LGBMBaseCrossValidator, _LGBMGroupKFold, _LGBMStratifiedKFold
|
||||
|
||||
__all__ = [
|
||||
"cv",
|
||||
"CVBooster",
|
||||
"train",
|
||||
]
|
||||
|
||||
|
||||
_LGBM_CustomMetricFunction = Union[
|
||||
Callable[
|
||||
[np.ndarray, Dataset],
|
||||
_LGBM_EvalFunctionResultType,
|
||||
],
|
||||
Callable[
|
||||
[np.ndarray, Dataset],
|
||||
List[_LGBM_EvalFunctionResultType],
|
||||
],
|
||||
]
|
||||
|
||||
_LGBM_PreprocFunction = Callable[
|
||||
[Dataset, Dataset, Dict[str, Any]],
|
||||
Tuple[Dataset, Dataset, Dict[str, Any]],
|
||||
]
|
||||
|
||||
|
||||
def _choose_num_iterations(*, num_boost_round_kwarg: int, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Choose number of boosting rounds.
|
||||
|
||||
In ``train()`` and ``cv()``, there are multiple ways to provide configuration for
|
||||
the number of boosting rounds to perform:
|
||||
|
||||
* the ``num_boost_round`` keyword argument
|
||||
* any of the ``num_iterations`` or its aliases via the ``params`` dictionary
|
||||
|
||||
These should be preferred in the following order (first one found wins):
|
||||
|
||||
1. ``num_iterations`` provided via ``params`` (because it's the main parameter name)
|
||||
2. any other aliases of ``num_iterations`` provided via ``params``
|
||||
3. the ``num_boost_round`` keyword argument
|
||||
|
||||
This function handles that choice, and issuing helpful warnings in the cases where the
|
||||
result might be surprising.
|
||||
|
||||
Returns
|
||||
-------
|
||||
params : dict
|
||||
Parameters, with ``"num_iterations"`` set to the preferred value and all other
|
||||
aliases of ``num_iterations`` removed.
|
||||
"""
|
||||
num_iteration_configs_provided = {
|
||||
alias: params[alias] for alias in _ConfigAliases.get("num_iterations") if alias in params
|
||||
}
|
||||
|
||||
# now that the relevant information has been pulled out of params, it's safe to overwrite it
|
||||
# with the content that should be used for training (i.e. with aliases resolved)
|
||||
params = _choose_param_value(
|
||||
main_param_name="num_iterations",
|
||||
params=params,
|
||||
default_value=num_boost_round_kwarg,
|
||||
)
|
||||
|
||||
# if there were not multiple boosting rounds configurations provided in params,
|
||||
# then by definition they cannot have conflicting values... no need to warn
|
||||
if len(num_iteration_configs_provided) <= 1:
|
||||
return params
|
||||
|
||||
# if all the aliases have the same value, no need to warn
|
||||
if len(set(num_iteration_configs_provided.values())) <= 1:
|
||||
return params
|
||||
|
||||
# if this line is reached, lightgbm should warn
|
||||
value_string = ", ".join(f"{alias}={val}" for alias, val in num_iteration_configs_provided.items())
|
||||
_log_warning(
|
||||
f"Found conflicting values for num_iterations provided via 'params': {value_string}. "
|
||||
f"LightGBM will perform up to {params['num_iterations']} boosting rounds. "
|
||||
"To be confident in the maximum number of boosting rounds LightGBM will perform and to "
|
||||
"suppress this warning, modify 'params' so that only one of those is present."
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
def train(
|
||||
params: Dict[str, Any],
|
||||
train_set: Dataset,
|
||||
num_boost_round: int = 100,
|
||||
valid_sets: Optional[List[Dataset]] = None,
|
||||
valid_names: Optional[List[str]] = None,
|
||||
feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None,
|
||||
init_model: Optional[Union[str, Path, Booster]] = None,
|
||||
keep_training_booster: bool = False,
|
||||
callbacks: Optional[List[Callable]] = None,
|
||||
) -> Booster:
|
||||
"""Perform the training with given parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
params : dict
|
||||
Parameters for training. Values passed through ``params`` take precedence over those
|
||||
supplied via arguments.
|
||||
train_set : Dataset
|
||||
Data to be trained on.
|
||||
num_boost_round : int, optional (default=100)
|
||||
Number of boosting iterations.
|
||||
valid_sets : list of Dataset, or None, optional (default=None)
|
||||
List of data to be evaluated on during training.
|
||||
valid_names : list of str, or None, optional (default=None)
|
||||
Names of ``valid_sets``.
|
||||
feval : callable, list of callable, or None, optional (default=None)
|
||||
Customized evaluation function.
|
||||
Each evaluation function should accept two parameters: preds, eval_data,
|
||||
and return (metric_name, metric_value, maximize) or list of such tuples.
|
||||
|
||||
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
|
||||
The predicted values.
|
||||
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
|
||||
If custom objective function is used, predicted values are returned before any transformation,
|
||||
e.g. they are raw margin instead of probability of positive class for binary task in this case.
|
||||
eval_data : Dataset
|
||||
A ``Dataset`` to evaluate.
|
||||
metric_name : str
|
||||
Unique identifier for the metric (e.g. "custom_adjusted_mse").
|
||||
metric_value : float
|
||||
Value of the evaluation metric.
|
||||
maximize : bool
|
||||
Are higher values better? e.g. ``True`` for AUC and ``False`` for binary error.
|
||||
|
||||
To ignore the default metric corresponding to the used objective,
|
||||
set the ``metric`` parameter to the string ``"None"`` in ``params``.
|
||||
init_model : str, pathlib.Path, Booster or None, optional (default=None)
|
||||
Filename of LightGBM model or Booster instance used for continue training.
|
||||
keep_training_booster : bool, optional (default=False)
|
||||
Whether the returned Booster will be used to keep training.
|
||||
If False, the returned value will be converted into _InnerPredictor before returning.
|
||||
This means you won't be able to use ``eval``, ``eval_train`` or ``eval_valid`` methods of the returned Booster.
|
||||
When your model is very large and cause the memory error,
|
||||
you can try to set this param to ``True`` to avoid the model conversion performed during the internal call of ``model_to_string``.
|
||||
You can still use _InnerPredictor as ``init_model`` for future continue training.
|
||||
callbacks : list of callable, or None, optional (default=None)
|
||||
List of callback functions that are applied at each iteration.
|
||||
See Callbacks in Python API for more information.
|
||||
|
||||
Note
|
||||
----
|
||||
A custom objective function can be provided for the ``objective`` parameter.
|
||||
It should accept two parameters: preds, train_data and return (grad, hess).
|
||||
|
||||
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
|
||||
The predicted values.
|
||||
Predicted values are returned before any transformation,
|
||||
e.g. they are raw margin instead of probability of positive class for binary task.
|
||||
train_data : Dataset
|
||||
The training dataset.
|
||||
grad : numpy 1-D array or numpy 2-D array (for multi-class task)
|
||||
The value of the first order derivative (gradient) of the loss
|
||||
with respect to the elements of preds for each sample point.
|
||||
hess : numpy 1-D array or numpy 2-D array (for multi-class task)
|
||||
The value of the second order derivative (Hessian) of the loss
|
||||
with respect to the elements of preds for each sample point.
|
||||
|
||||
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
|
||||
and grad and hess should be returned in the same format.
|
||||
|
||||
Returns
|
||||
-------
|
||||
booster : Booster
|
||||
The trained Booster model.
|
||||
"""
|
||||
if not isinstance(train_set, Dataset):
|
||||
raise TypeError(f"train() only accepts Dataset object, train_set has type '{type(train_set).__name__}'.")
|
||||
|
||||
if isinstance(valid_sets, list):
|
||||
for i, valid_item in enumerate(valid_sets):
|
||||
if not isinstance(valid_item, Dataset):
|
||||
raise TypeError(
|
||||
"Every item in valid_sets must be a Dataset object. "
|
||||
f"Item {i} has type '{type(valid_item).__name__}'."
|
||||
)
|
||||
|
||||
# create predictor first
|
||||
params = copy.deepcopy(params)
|
||||
params = _choose_param_value(
|
||||
main_param_name="objective",
|
||||
params=params,
|
||||
default_value=None,
|
||||
)
|
||||
fobj: Optional[_LGBM_CustomObjectiveFunction] = None
|
||||
if callable(params["objective"]):
|
||||
fobj = params["objective"]
|
||||
params["objective"] = "none"
|
||||
|
||||
params = _choose_num_iterations(num_boost_round_kwarg=num_boost_round, params=params)
|
||||
num_boost_round = params["num_iterations"]
|
||||
if num_boost_round <= 0:
|
||||
raise ValueError(f"Number of boosting rounds must be greater than 0. Got {num_boost_round}.")
|
||||
|
||||
# setting early stopping via global params should be possible
|
||||
params = _choose_param_value(
|
||||
main_param_name="early_stopping_round",
|
||||
params=params,
|
||||
default_value=None,
|
||||
)
|
||||
if params["early_stopping_round"] is None:
|
||||
params.pop("early_stopping_round")
|
||||
first_metric_only = params.get("first_metric_only", False)
|
||||
|
||||
predictor: Optional[_InnerPredictor] = None
|
||||
if isinstance(init_model, (str, Path)):
|
||||
predictor = _InnerPredictor.from_model_file(model_file=init_model, pred_parameter=params)
|
||||
elif isinstance(init_model, Booster):
|
||||
predictor = _InnerPredictor.from_booster(booster=init_model, pred_parameter=dict(init_model.params, **params))
|
||||
|
||||
if predictor is not None:
|
||||
init_iteration = predictor.current_iteration()
|
||||
else:
|
||||
init_iteration = 0
|
||||
|
||||
train_set._update_params(params)._set_predictor(predictor)
|
||||
|
||||
is_valid_contain_train = False
|
||||
train_data_name = "training"
|
||||
reduced_valid_sets = []
|
||||
name_valid_sets = []
|
||||
if valid_sets is not None:
|
||||
if isinstance(valid_sets, Dataset):
|
||||
valid_sets = [valid_sets]
|
||||
if isinstance(valid_names, str):
|
||||
valid_names = [valid_names]
|
||||
for i, valid_data in enumerate(valid_sets):
|
||||
# reduce cost for prediction training data
|
||||
if valid_data is train_set:
|
||||
is_valid_contain_train = True
|
||||
if valid_names is not None:
|
||||
train_data_name = valid_names[i]
|
||||
continue
|
||||
reduced_valid_sets.append(valid_data._update_params(params).set_reference(train_set))
|
||||
if valid_names is not None and len(valid_names) > i:
|
||||
name_valid_sets.append(valid_names[i])
|
||||
else:
|
||||
name_valid_sets.append(f"valid_{i}")
|
||||
# process callbacks
|
||||
if callbacks is None:
|
||||
callbacks_set = set()
|
||||
else:
|
||||
for i, cb in enumerate(callbacks):
|
||||
cb.__dict__.setdefault("order", i - len(callbacks))
|
||||
callbacks_set = set(callbacks)
|
||||
|
||||
if callback._should_enable_early_stopping(params.get("early_stopping_round", 0)):
|
||||
callbacks_set.add(
|
||||
callback.early_stopping(
|
||||
stopping_rounds=params["early_stopping_round"], # type: ignore[arg-type]
|
||||
first_metric_only=first_metric_only,
|
||||
min_delta=params.get("early_stopping_min_delta", 0.0),
|
||||
verbose=_choose_param_value(
|
||||
main_param_name="verbosity",
|
||||
params=params,
|
||||
default_value=1,
|
||||
).pop("verbosity")
|
||||
> 0,
|
||||
)
|
||||
)
|
||||
|
||||
callbacks_before_iter_set = {cb for cb in callbacks_set if getattr(cb, "before_iteration", False)}
|
||||
callbacks_after_iter_set = callbacks_set - callbacks_before_iter_set
|
||||
callbacks_before_iter = sorted(callbacks_before_iter_set, key=attrgetter("order"))
|
||||
callbacks_after_iter = sorted(callbacks_after_iter_set, key=attrgetter("order"))
|
||||
|
||||
# construct booster
|
||||
try:
|
||||
booster = Booster(params=params, train_set=train_set)
|
||||
if is_valid_contain_train:
|
||||
booster.set_train_data_name(train_data_name)
|
||||
for valid_set, name_valid_set in zip(reduced_valid_sets, name_valid_sets, strict=True):
|
||||
booster.add_valid(valid_set, name_valid_set)
|
||||
finally:
|
||||
train_set._reverse_update_params()
|
||||
for valid_set in reduced_valid_sets:
|
||||
valid_set._reverse_update_params()
|
||||
booster.best_iteration = 0
|
||||
|
||||
# start training
|
||||
for i in range(init_iteration, init_iteration + num_boost_round):
|
||||
for cb in callbacks_before_iter:
|
||||
cb(
|
||||
callback.CallbackEnv(
|
||||
model=booster,
|
||||
params=params,
|
||||
iteration=i,
|
||||
begin_iteration=init_iteration,
|
||||
end_iteration=init_iteration + num_boost_round,
|
||||
evaluation_result_list=None,
|
||||
)
|
||||
)
|
||||
|
||||
booster.update(fobj=fobj)
|
||||
|
||||
evaluation_result_list: List[_LGBM_BoosterEvalMethodResultType] = []
|
||||
# check evaluation result.
|
||||
if valid_sets is not None:
|
||||
if is_valid_contain_train:
|
||||
evaluation_result_list.extend(booster.eval_train(feval))
|
||||
evaluation_result_list.extend(booster.eval_valid(feval))
|
||||
try:
|
||||
for cb in callbacks_after_iter:
|
||||
cb(
|
||||
callback.CallbackEnv(
|
||||
model=booster,
|
||||
params=params,
|
||||
iteration=i,
|
||||
begin_iteration=init_iteration,
|
||||
end_iteration=init_iteration + num_boost_round,
|
||||
evaluation_result_list=evaluation_result_list,
|
||||
)
|
||||
)
|
||||
except callback.EarlyStopException as earlyStopException:
|
||||
booster.best_iteration = earlyStopException.best_iteration + 1
|
||||
# eval results from cv() have a 5th element with the standard deviation of metrics,
|
||||
# which is not needed for early stopping
|
||||
evaluation_result_list = [item[:4] for item in earlyStopException.best_score]
|
||||
break
|
||||
booster.best_score = defaultdict(OrderedDict)
|
||||
for dataset_name, metric_name, metric_value, _ in evaluation_result_list:
|
||||
booster.best_score[dataset_name][metric_name] = metric_value
|
||||
if not keep_training_booster:
|
||||
booster.model_from_string(booster.model_to_string()).free_dataset()
|
||||
return booster
|
||||
|
||||
|
||||
class CVBooster:
|
||||
"""CVBooster in LightGBM.
|
||||
|
||||
Auxiliary data structure to hold and redirect all boosters of ``cv()`` function.
|
||||
This class has the same methods as Booster class.
|
||||
All method calls, except for the following methods, are actually performed for underlying Boosters and
|
||||
then all returned results are returned in a list.
|
||||
|
||||
- ``model_from_string()``
|
||||
- ``model_to_string()``
|
||||
- ``save_model()``
|
||||
|
||||
Attributes
|
||||
----------
|
||||
boosters : list of Booster
|
||||
The list of underlying fitted models.
|
||||
best_iteration : int
|
||||
The best iteration of fitted model.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_file: Optional[Union[str, Path]] = None,
|
||||
):
|
||||
"""Initialize the CVBooster.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
model_file : str, pathlib.Path or None, optional (default=None)
|
||||
Path to the CVBooster model file.
|
||||
"""
|
||||
self.boosters: List[Booster] = []
|
||||
self.best_iteration = -1
|
||||
|
||||
if model_file is not None:
|
||||
with open(model_file, "r") as file:
|
||||
self._from_dict(json.load(file))
|
||||
|
||||
def _from_dict(self, models: Dict[str, Any]) -> None:
|
||||
"""Load CVBooster from dict."""
|
||||
self.best_iteration = models["best_iteration"]
|
||||
self.boosters = []
|
||||
for model_str in models["boosters"]:
|
||||
self.boosters.append(Booster(model_str=model_str))
|
||||
|
||||
def _to_dict(
|
||||
self,
|
||||
*,
|
||||
num_iteration: Optional[int],
|
||||
start_iteration: int,
|
||||
importance_type: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""Serialize CVBooster to dict."""
|
||||
models_str = []
|
||||
for booster in self.boosters:
|
||||
models_str.append(
|
||||
booster.model_to_string(
|
||||
num_iteration=num_iteration, start_iteration=start_iteration, importance_type=importance_type
|
||||
)
|
||||
)
|
||||
return {"boosters": models_str, "best_iteration": self.best_iteration}
|
||||
|
||||
def __getattr__(self, name: str) -> Callable[[Any, Any], List[Any]]:
|
||||
"""Redirect methods call of CVBooster."""
|
||||
|
||||
def handler_function(*args: Any, **kwargs: Any) -> List[Any]:
|
||||
"""Call methods with each booster, and concatenate their results."""
|
||||
ret = []
|
||||
for booster in self.boosters:
|
||||
ret.append(getattr(booster, name)(*args, **kwargs))
|
||||
return ret
|
||||
|
||||
return handler_function
|
||||
|
||||
def __getstate__(self) -> Dict[str, Any]:
|
||||
return vars(self)
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
vars(self).update(state)
|
||||
|
||||
def model_from_string(self, model_str: str) -> "CVBooster":
|
||||
"""Load CVBooster from a string.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
model_str : str
|
||||
Model will be loaded from this string.
|
||||
|
||||
Returns
|
||||
-------
|
||||
self : CVBooster
|
||||
Loaded CVBooster object.
|
||||
"""
|
||||
self._from_dict(json.loads(model_str))
|
||||
return self
|
||||
|
||||
def model_to_string(
|
||||
self,
|
||||
num_iteration: Optional[int] = None,
|
||||
start_iteration: int = 0,
|
||||
importance_type: str = "split",
|
||||
) -> str:
|
||||
"""Save CVBooster to JSON string.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
num_iteration : int or None, optional (default=None)
|
||||
Index of the iteration that should be saved.
|
||||
If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
|
||||
If <= 0, all iterations are saved.
|
||||
start_iteration : int, optional (default=0)
|
||||
Start index of the iteration that should be saved.
|
||||
importance_type : str, optional (default="split")
|
||||
What type of feature importance should be saved.
|
||||
If "split", result contains numbers of times the feature is used in a model.
|
||||
If "gain", result contains total gains of splits which use the feature.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str_repr : str
|
||||
JSON string representation of CVBooster.
|
||||
"""
|
||||
return json.dumps(
|
||||
self._to_dict(num_iteration=num_iteration, start_iteration=start_iteration, importance_type=importance_type)
|
||||
)
|
||||
|
||||
def save_model(
|
||||
self,
|
||||
filename: Union[str, Path],
|
||||
num_iteration: Optional[int] = None,
|
||||
start_iteration: int = 0,
|
||||
importance_type: str = "split",
|
||||
) -> "CVBooster":
|
||||
"""Save CVBooster to a file as JSON text.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str or pathlib.Path
|
||||
Filename to save CVBooster.
|
||||
num_iteration : int or None, optional (default=None)
|
||||
Index of the iteration that should be saved.
|
||||
If None, if the best iteration exists, it is saved; otherwise, all iterations are saved.
|
||||
If <= 0, all iterations are saved.
|
||||
start_iteration : int, optional (default=0)
|
||||
Start index of the iteration that should be saved.
|
||||
importance_type : str, optional (default="split")
|
||||
What type of feature importance should be saved.
|
||||
If "split", result contains numbers of times the feature is used in a model.
|
||||
If "gain", result contains total gains of splits which use the feature.
|
||||
|
||||
Returns
|
||||
-------
|
||||
self : CVBooster
|
||||
Returns self.
|
||||
"""
|
||||
with open(filename, "w") as file:
|
||||
json.dump(
|
||||
self._to_dict(
|
||||
num_iteration=num_iteration, start_iteration=start_iteration, importance_type=importance_type
|
||||
),
|
||||
file,
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
|
||||
def _make_n_folds(
|
||||
*,
|
||||
full_data: Dataset,
|
||||
folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]],
|
||||
nfold: int,
|
||||
params: Dict[str, Any],
|
||||
seed: int,
|
||||
fpreproc: Optional[_LGBM_PreprocFunction],
|
||||
stratified: bool,
|
||||
shuffle: bool,
|
||||
eval_train_metric: bool,
|
||||
) -> CVBooster:
|
||||
"""Make a n-fold list of Booster from random indices."""
|
||||
full_data = full_data.construct()
|
||||
num_data = full_data.num_data()
|
||||
if folds is not None:
|
||||
if not hasattr(folds, "__iter__") and not hasattr(folds, "split"):
|
||||
raise AttributeError(
|
||||
"folds should be a generator or iterator of (train_idx, test_idx) tuples "
|
||||
"or scikit-learn splitter object with split method"
|
||||
)
|
||||
if hasattr(folds, "split"):
|
||||
group_info = full_data.get_group()
|
||||
if group_info is not None:
|
||||
group_info = np.asarray(group_info, dtype=np.int32)
|
||||
flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
|
||||
else:
|
||||
flatted_group = np.zeros(num_data, dtype=np.int32)
|
||||
folds = folds.split(X=np.empty(num_data), y=full_data.get_label(), groups=flatted_group)
|
||||
else:
|
||||
if any(
|
||||
params.get(obj_alias, "")
|
||||
in {"lambdarank", "rank_xendcg", "xendcg", "xe_ndcg", "xe_ndcg_mart", "xendcg_mart"}
|
||||
for obj_alias in _ConfigAliases.get("objective")
|
||||
):
|
||||
if not SKLEARN_INSTALLED:
|
||||
raise LightGBMError("scikit-learn is required for ranking cv")
|
||||
# ranking task, split according to groups
|
||||
group_info = np.asarray(full_data.get_group(), dtype=np.int32)
|
||||
flatted_group = np.repeat(range(len(group_info)), repeats=group_info)
|
||||
group_kfold = _LGBMGroupKFold(n_splits=nfold)
|
||||
folds = group_kfold.split(X=np.empty(num_data), groups=flatted_group)
|
||||
elif stratified:
|
||||
if not SKLEARN_INSTALLED:
|
||||
raise LightGBMError("scikit-learn is required for stratified cv")
|
||||
skf = _LGBMStratifiedKFold(n_splits=nfold, shuffle=shuffle, random_state=seed)
|
||||
folds = skf.split(X=np.empty(num_data), y=full_data.get_label())
|
||||
else:
|
||||
if shuffle:
|
||||
randidx = np.random.RandomState(seed).permutation(num_data)
|
||||
else:
|
||||
randidx = np.arange(num_data)
|
||||
test_id = np.array_split(randidx, nfold)
|
||||
train_id = [np.concatenate([test_id[i] for i in range(nfold) if k != i]) for k in range(nfold)]
|
||||
folds = zip(train_id, test_id, strict=True)
|
||||
|
||||
ret = CVBooster()
|
||||
for train_idx, test_idx in folds:
|
||||
train_set = full_data.subset(sorted(train_idx))
|
||||
valid_set = full_data.subset(sorted(test_idx))
|
||||
# run preprocessing on the data set if needed
|
||||
if fpreproc is not None:
|
||||
train_set, valid_set, tparam = fpreproc(train_set, valid_set, params.copy())
|
||||
else:
|
||||
tparam = params
|
||||
booster_for_fold = Booster(tparam, train_set)
|
||||
if eval_train_metric:
|
||||
booster_for_fold.add_valid(train_set, "train")
|
||||
booster_for_fold.add_valid(valid_set, "valid")
|
||||
ret.boosters.append(booster_for_fold)
|
||||
return ret
|
||||
|
||||
|
||||
def _agg_cv_result(
|
||||
raw_results: List[List[_LGBM_BoosterEvalMethodResultType]],
|
||||
) -> List[_LGBM_BoosterEvalMethodResultWithStandardDeviationType]:
|
||||
"""Aggregate cross-validation results."""
|
||||
# build up 2 maps, of the form:
|
||||
#
|
||||
# OrderedDict{
|
||||
# (<dataset_name>, <metric_name>): <maximize>
|
||||
# }
|
||||
#
|
||||
# OrderedDict{
|
||||
# (<dataset_name>, <metric_name>): list[<metric_value>]
|
||||
# }
|
||||
#
|
||||
metric_types: Dict[Tuple[str, str], bool] = OrderedDict()
|
||||
metric_values: Dict[Tuple[str, str], List[float]] = OrderedDict()
|
||||
for one_result in raw_results:
|
||||
for dataset_name, metric_name, metric_value, maximize in one_result:
|
||||
key = (dataset_name, metric_name)
|
||||
metric_types[key] = maximize
|
||||
metric_values.setdefault(key, [])
|
||||
metric_values[key].append(metric_value)
|
||||
|
||||
# turn that into a list of tuples of the form:
|
||||
#
|
||||
# [
|
||||
# (<dataset_name>, <metric_name>, mean(<values>), <maximize>, std_dev(<values>))
|
||||
# ]
|
||||
return [(k[0], k[1], float(np.mean(v)), metric_types[k], float(np.std(v))) for k, v in metric_values.items()]
|
||||
|
||||
|
||||
def cv(
|
||||
params: Dict[str, Any],
|
||||
train_set: Dataset,
|
||||
num_boost_round: int = 100,
|
||||
folds: Optional[Union[Iterable[Tuple[np.ndarray, np.ndarray]], _LGBMBaseCrossValidator]] = None,
|
||||
nfold: int = 5,
|
||||
stratified: bool = True,
|
||||
shuffle: bool = True,
|
||||
metrics: Optional[Union[str, List[str]]] = None,
|
||||
feval: Optional[Union[_LGBM_CustomMetricFunction, List[_LGBM_CustomMetricFunction]]] = None,
|
||||
init_model: Optional[Union[str, Path, Booster]] = None,
|
||||
fpreproc: Optional[_LGBM_PreprocFunction] = None,
|
||||
seed: int = 0,
|
||||
callbacks: Optional[List[Callable]] = None,
|
||||
eval_train_metric: bool = False,
|
||||
return_cvbooster: bool = False,
|
||||
) -> Dict[str, Union[List[float], CVBooster]]:
|
||||
"""Perform the cross-validation with given parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
params : dict
|
||||
Parameters for training. Values passed through ``params`` take precedence over those
|
||||
supplied via arguments.
|
||||
train_set : Dataset
|
||||
Data to be trained on.
|
||||
num_boost_round : int, optional (default=100)
|
||||
Number of boosting iterations.
|
||||
folds : generator or iterator of (train_idx, test_idx) tuples, scikit-learn splitter object or None, optional (default=None)
|
||||
If generator or iterator, it should yield the train and test indices for each fold.
|
||||
If object, it should be one of the scikit-learn splitter classes
|
||||
(https://scikit-learn.org/stable/modules/classes.html#splitter-classes)
|
||||
and have ``split`` method.
|
||||
This argument has highest priority over other data split arguments.
|
||||
nfold : int, optional (default=5)
|
||||
Number of folds in CV.
|
||||
stratified : bool, optional (default=True)
|
||||
Whether to perform stratified sampling.
|
||||
shuffle : bool, optional (default=True)
|
||||
Whether to shuffle before splitting data.
|
||||
metrics : str, list of str, or None, optional (default=None)
|
||||
Evaluation metrics to be monitored while CV.
|
||||
If not None, the metric in ``params`` will be overridden.
|
||||
feval : callable, list of callable, or None, optional (default=None)
|
||||
Customized evaluation function.
|
||||
Each evaluation function should accept two parameters: preds, eval_data,
|
||||
and return (metric_name, metric_value, maximize) or list of such tuples.
|
||||
|
||||
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
|
||||
The predicted values.
|
||||
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes].
|
||||
If custom objective function is used, predicted values are returned before any transformation,
|
||||
e.g. they are raw margin instead of probability of positive class for binary task in this case.
|
||||
eval_data : Dataset
|
||||
A ``Dataset`` to evaluate.
|
||||
metric_name : str
|
||||
Unique identifier for the metric (e.g. "custom_adjusted_mse").
|
||||
metric_value : float
|
||||
Value of the evaluation metric.
|
||||
maximize : bool
|
||||
Are higher values better? e.g. ``True`` for AUC and ``False`` for binary error.
|
||||
|
||||
To ignore the default metric corresponding to the used objective,
|
||||
set ``metrics`` to the string ``"None"``.
|
||||
init_model : str, pathlib.Path, Booster or None, optional (default=None)
|
||||
Filename of LightGBM model or Booster instance used for continue training.
|
||||
fpreproc : callable or None, optional (default=None)
|
||||
Preprocessing function that takes (dtrain, dtest, params)
|
||||
and returns transformed versions of those.
|
||||
seed : int, optional (default=0)
|
||||
Seed used to generate the folds (passed to numpy.random.seed).
|
||||
callbacks : list of callable, or None, optional (default=None)
|
||||
List of callback functions that are applied at each iteration.
|
||||
See Callbacks in Python API for more information.
|
||||
eval_train_metric : bool, optional (default=False)
|
||||
Whether to display the train metric in progress.
|
||||
The score of the metric is calculated again after each training step, so there is some impact on performance.
|
||||
return_cvbooster : bool, optional (default=False)
|
||||
Whether to return Booster models trained on each fold through ``CVBooster``.
|
||||
|
||||
Note
|
||||
----
|
||||
A custom objective function can be provided for the ``objective`` parameter.
|
||||
It should accept two parameters: preds, train_data and return (grad, hess).
|
||||
|
||||
preds : numpy 1-D array or numpy 2-D array (for multi-class task)
|
||||
The predicted values.
|
||||
Predicted values are returned before any transformation,
|
||||
e.g. they are raw margin instead of probability of positive class for binary task.
|
||||
train_data : Dataset
|
||||
The training dataset.
|
||||
grad : numpy 1-D array or numpy 2-D array (for multi-class task)
|
||||
The value of the first order derivative (gradient) of the loss
|
||||
with respect to the elements of preds for each sample point.
|
||||
hess : numpy 1-D array or numpy 2-D array (for multi-class task)
|
||||
The value of the second order derivative (Hessian) of the loss
|
||||
with respect to the elements of preds for each sample point.
|
||||
|
||||
For multi-class task, preds are numpy 2-D array of shape = [n_samples, n_classes],
|
||||
and grad and hess should be returned in the same format.
|
||||
|
||||
Returns
|
||||
-------
|
||||
eval_results : dict
|
||||
History of evaluation results of each metric.
|
||||
The dictionary has the following format:
|
||||
{'valid metric1-mean': [values], 'valid metric1-stdv': [values],
|
||||
'valid metric2-mean': [values], 'valid metric2-stdv': [values],
|
||||
...}.
|
||||
If ``return_cvbooster=True``, also returns trained boosters wrapped in a ``CVBooster`` object via ``cvbooster`` key.
|
||||
If ``eval_train_metric=True``, also returns the train metric history.
|
||||
In this case, the dictionary has the following format:
|
||||
{'train metric1-mean': [values], 'valid metric1-mean': [values],
|
||||
'train metric2-mean': [values], 'valid metric2-mean': [values],
|
||||
...}.
|
||||
"""
|
||||
if not isinstance(train_set, Dataset):
|
||||
raise TypeError(f"cv() only accepts Dataset object, train_set has type '{type(train_set).__name__}'.")
|
||||
|
||||
params = copy.deepcopy(params)
|
||||
params = _choose_param_value(
|
||||
main_param_name="objective",
|
||||
params=params,
|
||||
default_value=None,
|
||||
)
|
||||
fobj: Optional[_LGBM_CustomObjectiveFunction] = None
|
||||
if callable(params["objective"]):
|
||||
fobj = params["objective"]
|
||||
params["objective"] = "none"
|
||||
|
||||
params = _choose_num_iterations(num_boost_round_kwarg=num_boost_round, params=params)
|
||||
num_boost_round = params["num_iterations"]
|
||||
if num_boost_round <= 0:
|
||||
raise ValueError(f"Number of boosting rounds must be greater than 0. Got {num_boost_round}.")
|
||||
|
||||
# setting early stopping via global params should be possible
|
||||
params = _choose_param_value(
|
||||
main_param_name="early_stopping_round",
|
||||
params=params,
|
||||
default_value=None,
|
||||
)
|
||||
if params["early_stopping_round"] is None:
|
||||
params.pop("early_stopping_round")
|
||||
first_metric_only = params.get("first_metric_only", False)
|
||||
|
||||
if isinstance(init_model, (str, Path)):
|
||||
predictor = _InnerPredictor.from_model_file(
|
||||
model_file=init_model,
|
||||
pred_parameter=params,
|
||||
)
|
||||
elif isinstance(init_model, Booster):
|
||||
predictor = _InnerPredictor.from_booster(
|
||||
booster=init_model,
|
||||
pred_parameter=dict(init_model.params, **params),
|
||||
)
|
||||
else:
|
||||
predictor = None
|
||||
|
||||
if metrics is not None:
|
||||
for metric_alias in _ConfigAliases.get("metric"):
|
||||
params.pop(metric_alias, None)
|
||||
params["metric"] = metrics
|
||||
|
||||
train_set._update_params(params)._set_predictor(predictor)
|
||||
|
||||
results = defaultdict(list)
|
||||
cvbooster = _make_n_folds(
|
||||
full_data=train_set,
|
||||
folds=folds,
|
||||
nfold=nfold,
|
||||
params=params,
|
||||
seed=seed,
|
||||
fpreproc=fpreproc,
|
||||
stratified=stratified,
|
||||
shuffle=shuffle,
|
||||
eval_train_metric=eval_train_metric,
|
||||
)
|
||||
|
||||
# setup callbacks
|
||||
if callbacks is None:
|
||||
callbacks_set = set()
|
||||
else:
|
||||
for i, cb in enumerate(callbacks):
|
||||
cb.__dict__.setdefault("order", i - len(callbacks))
|
||||
callbacks_set = set(callbacks)
|
||||
|
||||
if callback._should_enable_early_stopping(params.get("early_stopping_round", 0)):
|
||||
callbacks_set.add(
|
||||
callback.early_stopping(
|
||||
stopping_rounds=params["early_stopping_round"], # type: ignore[arg-type]
|
||||
first_metric_only=first_metric_only,
|
||||
min_delta=params.get("early_stopping_min_delta", 0.0),
|
||||
verbose=_choose_param_value(
|
||||
main_param_name="verbosity",
|
||||
params=params,
|
||||
default_value=1,
|
||||
).pop("verbosity")
|
||||
> 0,
|
||||
)
|
||||
)
|
||||
|
||||
callbacks_before_iter_set = {cb for cb in callbacks_set if getattr(cb, "before_iteration", False)}
|
||||
callbacks_after_iter_set = callbacks_set - callbacks_before_iter_set
|
||||
callbacks_before_iter = sorted(callbacks_before_iter_set, key=attrgetter("order"))
|
||||
callbacks_after_iter = sorted(callbacks_after_iter_set, key=attrgetter("order"))
|
||||
|
||||
for i in range(num_boost_round):
|
||||
for cb in callbacks_before_iter:
|
||||
cb(
|
||||
callback.CallbackEnv(
|
||||
model=cvbooster,
|
||||
params=params,
|
||||
iteration=i,
|
||||
begin_iteration=0,
|
||||
end_iteration=num_boost_round,
|
||||
evaluation_result_list=None,
|
||||
)
|
||||
)
|
||||
cvbooster.update(fobj=fobj) # type: ignore[call-arg]
|
||||
res = _agg_cv_result(cvbooster.eval_valid(feval)) # type: ignore[call-arg]
|
||||
for dataset_name, metric_name, metric_mean, _, metric_std_dev in res:
|
||||
results[f"{dataset_name} {metric_name}-mean"].append(metric_mean)
|
||||
results[f"{dataset_name} {metric_name}-stdv"].append(metric_std_dev)
|
||||
try:
|
||||
for cb in callbacks_after_iter:
|
||||
cb(
|
||||
callback.CallbackEnv(
|
||||
model=cvbooster,
|
||||
params=params,
|
||||
iteration=i,
|
||||
begin_iteration=0,
|
||||
end_iteration=num_boost_round,
|
||||
evaluation_result_list=res,
|
||||
)
|
||||
)
|
||||
except callback.EarlyStopException as earlyStopException:
|
||||
cvbooster.best_iteration = earlyStopException.best_iteration + 1
|
||||
for bst in cvbooster.boosters:
|
||||
bst.best_iteration = cvbooster.best_iteration
|
||||
for k in results:
|
||||
results[k] = results[k][: cvbooster.best_iteration]
|
||||
break
|
||||
|
||||
if return_cvbooster:
|
||||
results["cvbooster"] = cvbooster # type: ignore[assignment]
|
||||
|
||||
return dict(results)
|
||||
@@ -0,0 +1,49 @@
|
||||
# coding: utf-8
|
||||
"""Find the path to LightGBM dynamic library files."""
|
||||
|
||||
import ctypes
|
||||
from os import environ
|
||||
from pathlib import Path
|
||||
from platform import system
|
||||
from typing import List
|
||||
|
||||
__all__: List[str] = []
|
||||
|
||||
|
||||
def _find_lib_path() -> List[str]:
|
||||
"""Find the path to LightGBM library files.
|
||||
|
||||
Returns
|
||||
-------
|
||||
lib_path: list of str
|
||||
List of all found library paths to LightGBM.
|
||||
"""
|
||||
curr_path = Path(__file__).resolve()
|
||||
dll_path = [
|
||||
curr_path.parents[1],
|
||||
curr_path.parents[0] / "bin",
|
||||
curr_path.parents[0] / "lib",
|
||||
]
|
||||
if system() in ("Windows", "Microsoft"):
|
||||
dll_path.append(curr_path.parents[1] / "Release")
|
||||
dll_path.append(curr_path.parents[1] / "windows" / "x64" / "DLL")
|
||||
dll_path = [p / "lib_lightgbm.dll" for p in dll_path]
|
||||
elif system() == "Darwin":
|
||||
dll_path = [p / "lib_lightgbm.dylib" for p in dll_path]
|
||||
else:
|
||||
dll_path = [p / "lib_lightgbm.so" for p in dll_path]
|
||||
lib_path = [str(p) for p in dll_path if p.is_file()]
|
||||
if not lib_path:
|
||||
dll_path_joined = "\n".join(map(str, dll_path))
|
||||
raise Exception(f"Cannot find lightgbm library file in following paths:\n{dll_path_joined}")
|
||||
return lib_path
|
||||
|
||||
|
||||
# we don't need lib_lightgbm while building docs
|
||||
_LIB: ctypes.CDLL
|
||||
if environ.get("LIGHTGBM_BUILD_DOC", "False") == "True":
|
||||
from unittest.mock import Mock # isort: skip
|
||||
|
||||
_LIB = Mock(ctypes.CDLL) # type: ignore
|
||||
else:
|
||||
_LIB = ctypes.cdll.LoadLibrary(_find_lib_path()[0])
|
||||
@@ -0,0 +1,849 @@
|
||||
# coding: utf-8
|
||||
"""Plotting library."""
|
||||
|
||||
import math
|
||||
from copy import deepcopy
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .basic import Booster, _data_from_pandas, _is_zero, _log_warning, _MissingType
|
||||
from .compat import pd_DataFrame
|
||||
from .sklearn import LGBMModel
|
||||
|
||||
__all__ = [
|
||||
"create_tree_digraph",
|
||||
"plot_importance",
|
||||
"plot_metric",
|
||||
"plot_split_value_histogram",
|
||||
"plot_tree",
|
||||
]
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import matplotlib
|
||||
|
||||
|
||||
def _check_not_tuple_of_2_elements(obj: Any, obj_name: str) -> None:
|
||||
"""Check object is not tuple or does not have 2 elements."""
|
||||
if not isinstance(obj, tuple) or len(obj) != 2:
|
||||
raise TypeError(f"{obj_name} must be a tuple of 2 elements.")
|
||||
|
||||
|
||||
def _float2str(value: float, precision: Optional[int]) -> str:
|
||||
return f"{value:.{precision}f}" if precision is not None and not isinstance(value, str) else str(value)
|
||||
|
||||
|
||||
def plot_importance(
|
||||
booster: Union[Booster, LGBMModel],
|
||||
ax: "Optional[matplotlib.axes.Axes]" = None,
|
||||
height: float = 0.2,
|
||||
xlim: Optional[Tuple[float, float]] = None,
|
||||
ylim: Optional[Tuple[float, float]] = None,
|
||||
title: Optional[str] = "Feature importance",
|
||||
xlabel: Optional[str] = "Feature importance",
|
||||
ylabel: Optional[str] = "Features",
|
||||
importance_type: str = "auto",
|
||||
max_num_features: Optional[int] = None,
|
||||
ignore_zero: bool = True,
|
||||
figsize: Optional[Tuple[float, float]] = None,
|
||||
dpi: Optional[int] = None,
|
||||
grid: bool = True,
|
||||
precision: Optional[int] = 3,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Plot model's feature importances.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
booster : Booster or LGBMModel
|
||||
Booster or LGBMModel instance which feature importance should be plotted.
|
||||
ax : matplotlib.axes.Axes or None, optional (default=None)
|
||||
Target axes instance.
|
||||
If None, new figure and axes will be created.
|
||||
height : float, optional (default=0.2)
|
||||
Bar height, passed to ``ax.barh()``.
|
||||
xlim : tuple of 2 elements or None, optional (default=None)
|
||||
Tuple passed to ``ax.xlim()``.
|
||||
ylim : tuple of 2 elements or None, optional (default=None)
|
||||
Tuple passed to ``ax.ylim()``.
|
||||
title : str or None, optional (default="Feature importance")
|
||||
Axes title.
|
||||
If None, title is disabled.
|
||||
xlabel : str or None, optional (default="Feature importance")
|
||||
X-axis title label.
|
||||
If None, title is disabled.
|
||||
@importance_type@ placeholder can be used, and it will be replaced with the value of ``importance_type`` parameter.
|
||||
ylabel : str or None, optional (default="Features")
|
||||
Y-axis title label.
|
||||
If None, title is disabled.
|
||||
importance_type : str, optional (default="auto")
|
||||
How the importance is calculated.
|
||||
If "auto", if ``booster`` parameter is LGBMModel, ``booster.importance_type`` attribute is used; "split" otherwise.
|
||||
If "split", result contains numbers of times the feature is used in a model.
|
||||
If "gain", result contains total gains of splits which use the feature.
|
||||
max_num_features : int or None, optional (default=None)
|
||||
Max number of top features displayed on plot.
|
||||
If None or <1, all features will be displayed.
|
||||
ignore_zero : bool, optional (default=True)
|
||||
Whether to ignore features with zero importance.
|
||||
figsize : tuple of 2 elements or None, optional (default=None)
|
||||
Figure size.
|
||||
dpi : int or None, optional (default=None)
|
||||
Resolution of the figure.
|
||||
grid : bool, optional (default=True)
|
||||
Whether to add a grid for axes.
|
||||
precision : int or None, optional (default=3)
|
||||
Used to restrict the display of floating point values to a certain precision.
|
||||
**kwargs
|
||||
Other parameters passed to ``ax.barh()``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ax : matplotlib.axes.Axes
|
||||
The plot with model's feature importances.
|
||||
"""
|
||||
try:
|
||||
import matplotlib.pyplot as plt # noqa: PLC0415
|
||||
except ImportError as err:
|
||||
raise ImportError("You must install matplotlib and restart your session to plot importance.") from err
|
||||
|
||||
if isinstance(booster, LGBMModel):
|
||||
if importance_type == "auto":
|
||||
importance_type = booster.importance_type
|
||||
booster = booster.booster_
|
||||
elif isinstance(booster, Booster):
|
||||
if importance_type == "auto":
|
||||
importance_type = "split"
|
||||
else:
|
||||
raise TypeError("booster must be Booster or LGBMModel.")
|
||||
|
||||
importance = booster.feature_importance(importance_type=importance_type)
|
||||
feature_name = booster.feature_name()
|
||||
|
||||
if not len(importance):
|
||||
raise ValueError("Booster's feature_importance is empty.")
|
||||
|
||||
tuples = sorted(zip(feature_name, importance, strict=True), key=lambda x: x[1])
|
||||
if ignore_zero:
|
||||
tuples = [x for x in tuples if x[1] > 0]
|
||||
if max_num_features is not None and max_num_features > 0:
|
||||
tuples = tuples[-max_num_features:]
|
||||
if not tuples:
|
||||
raise ValueError(
|
||||
"No non-zero feature importances found. The model may have no splits. "
|
||||
"Use ignore_zero=False to show all features."
|
||||
)
|
||||
labels, values = zip(*tuples, strict=True)
|
||||
|
||||
if ax is None:
|
||||
if figsize is not None:
|
||||
_check_not_tuple_of_2_elements(figsize, "figsize")
|
||||
_, ax = plt.subplots(1, 1, figsize=figsize, dpi=dpi)
|
||||
|
||||
ylocs = np.arange(len(values))
|
||||
ax.barh(ylocs, values, align="center", height=height, **kwargs)
|
||||
|
||||
for x, y in zip(values, ylocs, strict=True):
|
||||
ax.text(x + 1, float(y), _float2str(x, precision) if importance_type == "gain" else x, va="center")
|
||||
|
||||
ax.set_yticks(ylocs)
|
||||
ax.set_yticklabels(labels)
|
||||
|
||||
if xlim is not None:
|
||||
_check_not_tuple_of_2_elements(xlim, "xlim")
|
||||
else:
|
||||
xlim = (0, max(values) * 1.1)
|
||||
ax.set_xlim(xlim)
|
||||
|
||||
if ylim is not None:
|
||||
_check_not_tuple_of_2_elements(ylim, "ylim")
|
||||
else:
|
||||
ylim = (-1, len(values))
|
||||
ax.set_ylim(ylim)
|
||||
|
||||
if title is not None:
|
||||
ax.set_title(title)
|
||||
if xlabel is not None:
|
||||
xlabel = xlabel.replace("@importance_type@", importance_type)
|
||||
ax.set_xlabel(xlabel)
|
||||
if ylabel is not None:
|
||||
ax.set_ylabel(ylabel)
|
||||
ax.grid(grid)
|
||||
return ax
|
||||
|
||||
|
||||
def plot_split_value_histogram(
|
||||
booster: Union[Booster, LGBMModel],
|
||||
feature: Union[int, str],
|
||||
bins: Union[int, str, None] = None,
|
||||
ax: "Optional[matplotlib.axes.Axes]" = None,
|
||||
width_coef: float = 0.8,
|
||||
xlim: Optional[Tuple[float, float]] = None,
|
||||
ylim: Optional[Tuple[float, float]] = None,
|
||||
title: Optional[str] = "Split value histogram for feature with @index/name@ @feature@",
|
||||
xlabel: Optional[str] = "Feature split value",
|
||||
ylabel: Optional[str] = "Count",
|
||||
figsize: Optional[Tuple[float, float]] = None,
|
||||
dpi: Optional[int] = None,
|
||||
grid: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Plot split value histogram for the specified feature of the model.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
booster : Booster or LGBMModel
|
||||
Booster or LGBMModel instance of which feature split value histogram should be plotted.
|
||||
feature : int or str
|
||||
The feature name or index the histogram is plotted for.
|
||||
If int, interpreted as index.
|
||||
If str, interpreted as name.
|
||||
bins : int, str or None, optional (default=None)
|
||||
The maximum number of bins.
|
||||
If None, the number of bins equals number of unique split values.
|
||||
If str, it should be one from the list of the supported values by ``numpy.histogram()`` function.
|
||||
ax : matplotlib.axes.Axes or None, optional (default=None)
|
||||
Target axes instance.
|
||||
If None, new figure and axes will be created.
|
||||
width_coef : float, optional (default=0.8)
|
||||
Coefficient for histogram bar width.
|
||||
xlim : tuple of 2 elements or None, optional (default=None)
|
||||
Tuple passed to ``ax.xlim()``.
|
||||
ylim : tuple of 2 elements or None, optional (default=None)
|
||||
Tuple passed to ``ax.ylim()``.
|
||||
title : str or None, optional (default="Split value histogram for feature with @index/name@ @feature@")
|
||||
Axes title.
|
||||
If None, title is disabled.
|
||||
@feature@ placeholder can be used, and it will be replaced with the value of ``feature`` parameter.
|
||||
@index/name@ placeholder can be used,
|
||||
and it will be replaced with ``index`` word in case of ``int`` type ``feature`` parameter
|
||||
or ``name`` word in case of ``str`` type ``feature`` parameter.
|
||||
xlabel : str or None, optional (default="Feature split value")
|
||||
X-axis title label.
|
||||
If None, title is disabled.
|
||||
ylabel : str or None, optional (default="Count")
|
||||
Y-axis title label.
|
||||
If None, title is disabled.
|
||||
figsize : tuple of 2 elements or None, optional (default=None)
|
||||
Figure size.
|
||||
dpi : int or None, optional (default=None)
|
||||
Resolution of the figure.
|
||||
grid : bool, optional (default=True)
|
||||
Whether to add a grid for axes.
|
||||
**kwargs
|
||||
Other parameters passed to ``ax.bar()``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ax : matplotlib.axes.Axes
|
||||
The plot with specified model's feature split value histogram.
|
||||
"""
|
||||
try:
|
||||
import matplotlib.pyplot as plt # noqa: PLC0415
|
||||
from matplotlib.ticker import MaxNLocator # noqa: PLC0415
|
||||
except ImportError as err:
|
||||
raise ImportError(
|
||||
"You must install matplotlib and restart your session to plot split value histogram."
|
||||
) from err
|
||||
|
||||
if isinstance(booster, LGBMModel):
|
||||
booster = booster.booster_
|
||||
elif not isinstance(booster, Booster):
|
||||
raise TypeError("booster must be Booster or LGBMModel.")
|
||||
|
||||
hist, split_bins = booster.get_split_value_histogram(feature=feature, bins=bins, xgboost_style=False)
|
||||
if np.count_nonzero(hist) == 0:
|
||||
raise ValueError(f"Cannot plot split value histogram, because feature {feature} was not used in splitting")
|
||||
width = width_coef * (split_bins[1] - split_bins[0])
|
||||
centred = (split_bins[:-1] + split_bins[1:]) / 2
|
||||
|
||||
if ax is None:
|
||||
if figsize is not None:
|
||||
_check_not_tuple_of_2_elements(figsize, "figsize")
|
||||
_, ax = plt.subplots(1, 1, figsize=figsize, dpi=dpi)
|
||||
|
||||
ax.bar(centred, hist, align="center", width=width, **kwargs)
|
||||
|
||||
if xlim is not None:
|
||||
_check_not_tuple_of_2_elements(xlim, "xlim")
|
||||
else:
|
||||
range_result = split_bins[-1] - split_bins[0]
|
||||
xlim = (split_bins[0] - range_result * 0.2, split_bins[-1] + range_result * 0.2)
|
||||
ax.set_xlim(xlim)
|
||||
|
||||
ax.yaxis.set_major_locator(MaxNLocator(integer=True))
|
||||
if ylim is not None:
|
||||
_check_not_tuple_of_2_elements(ylim, "ylim")
|
||||
else:
|
||||
ylim = (0, max(hist) * 1.1)
|
||||
ax.set_ylim(ylim)
|
||||
|
||||
if title is not None:
|
||||
title = title.replace("@feature@", str(feature))
|
||||
title = title.replace("@index/name@", ("name" if isinstance(feature, str) else "index"))
|
||||
ax.set_title(title)
|
||||
if xlabel is not None:
|
||||
ax.set_xlabel(xlabel)
|
||||
if ylabel is not None:
|
||||
ax.set_ylabel(ylabel)
|
||||
ax.grid(grid)
|
||||
return ax
|
||||
|
||||
|
||||
def plot_metric(
|
||||
booster: Union[Dict, LGBMModel],
|
||||
metric: Optional[str] = None,
|
||||
dataset_names: Optional[List[str]] = None,
|
||||
ax: "Optional[matplotlib.axes.Axes]" = None,
|
||||
xlim: Optional[Tuple[float, float]] = None,
|
||||
ylim: Optional[Tuple[float, float]] = None,
|
||||
title: Optional[str] = "Metric during training",
|
||||
xlabel: Optional[str] = "Iterations",
|
||||
ylabel: Optional[str] = "@metric@",
|
||||
figsize: Optional[Tuple[float, float]] = None,
|
||||
dpi: Optional[int] = None,
|
||||
grid: bool = True,
|
||||
) -> Any:
|
||||
"""Plot one metric during training.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
booster : dict or LGBMModel
|
||||
Dictionary returned from ``lightgbm.train()`` or LGBMModel instance.
|
||||
metric : str or None, optional (default=None)
|
||||
The metric name to plot.
|
||||
Only one metric supported because different metrics have various scales.
|
||||
If None, first metric picked from dictionary (according to hashcode).
|
||||
dataset_names : list of str, or None, optional (default=None)
|
||||
List of the dataset names which are used to calculate metric to plot.
|
||||
If None, all datasets are used.
|
||||
ax : matplotlib.axes.Axes or None, optional (default=None)
|
||||
Target axes instance.
|
||||
If None, new figure and axes will be created.
|
||||
xlim : tuple of 2 elements or None, optional (default=None)
|
||||
Tuple passed to ``ax.xlim()``.
|
||||
ylim : tuple of 2 elements or None, optional (default=None)
|
||||
Tuple passed to ``ax.ylim()``.
|
||||
title : str or None, optional (default="Metric during training")
|
||||
Axes title.
|
||||
If None, title is disabled.
|
||||
xlabel : str or None, optional (default="Iterations")
|
||||
X-axis title label.
|
||||
If None, title is disabled.
|
||||
ylabel : str or None, optional (default="@metric@")
|
||||
Y-axis title label.
|
||||
If 'auto', metric name is used.
|
||||
If None, title is disabled.
|
||||
@metric@ placeholder can be used, and it will be replaced with metric name.
|
||||
figsize : tuple of 2 elements or None, optional (default=None)
|
||||
Figure size.
|
||||
dpi : int or None, optional (default=None)
|
||||
Resolution of the figure.
|
||||
grid : bool, optional (default=True)
|
||||
Whether to add a grid for axes.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ax : matplotlib.axes.Axes
|
||||
The plot with metric's history over the training.
|
||||
"""
|
||||
try:
|
||||
import matplotlib.pyplot as plt # noqa: PLC0415
|
||||
except ImportError as err:
|
||||
raise ImportError("You must install matplotlib and restart your session to plot metric.") from err
|
||||
|
||||
if isinstance(booster, LGBMModel):
|
||||
eval_results = deepcopy(booster.evals_result_)
|
||||
elif isinstance(booster, dict):
|
||||
eval_results = deepcopy(booster)
|
||||
elif isinstance(booster, Booster):
|
||||
raise TypeError(
|
||||
"booster must be dict or LGBMModel. To use plot_metric with Booster type, first record the metrics using record_evaluation callback then pass that to plot_metric as argument `booster`"
|
||||
)
|
||||
else:
|
||||
raise TypeError("booster must be dict or LGBMModel.")
|
||||
|
||||
num_data = len(eval_results)
|
||||
|
||||
if not num_data:
|
||||
raise ValueError("eval results cannot be empty.")
|
||||
|
||||
if ax is None:
|
||||
if figsize is not None:
|
||||
_check_not_tuple_of_2_elements(figsize, "figsize")
|
||||
_, ax = plt.subplots(1, 1, figsize=figsize, dpi=dpi)
|
||||
|
||||
if dataset_names is None:
|
||||
dataset_names_iter = iter(eval_results.keys())
|
||||
elif not isinstance(dataset_names, (list, tuple, set)) or not dataset_names:
|
||||
raise ValueError("dataset_names should be iterable and cannot be empty")
|
||||
else:
|
||||
dataset_names_iter = iter(dataset_names)
|
||||
|
||||
name = next(dataset_names_iter) # take one as sample
|
||||
metrics_for_one = eval_results[name]
|
||||
num_metric = len(metrics_for_one)
|
||||
if metric is None:
|
||||
if num_metric > 1:
|
||||
_log_warning("More than one metric available, picking one to plot.")
|
||||
metric, results = metrics_for_one.popitem()
|
||||
else:
|
||||
if metric not in metrics_for_one:
|
||||
raise KeyError("No given metric in eval results.")
|
||||
results = metrics_for_one[metric]
|
||||
num_iteration = len(results)
|
||||
max_result = max(results)
|
||||
min_result = min(results)
|
||||
x_ = range(num_iteration)
|
||||
ax.plot(x_, results, label=name)
|
||||
|
||||
for name in dataset_names_iter:
|
||||
metrics_for_one = eval_results[name]
|
||||
results = metrics_for_one[metric]
|
||||
max_result = max(*results, max_result)
|
||||
min_result = min(*results, min_result)
|
||||
ax.plot(x_, results, label=name)
|
||||
|
||||
ax.legend(loc="best")
|
||||
|
||||
if xlim is not None:
|
||||
_check_not_tuple_of_2_elements(xlim, "xlim")
|
||||
else:
|
||||
xlim = (0, num_iteration)
|
||||
ax.set_xlim(xlim)
|
||||
|
||||
if ylim is not None:
|
||||
_check_not_tuple_of_2_elements(ylim, "ylim")
|
||||
else:
|
||||
range_result = max_result - min_result
|
||||
ylim = (min_result - range_result * 0.2, max_result + range_result * 0.2)
|
||||
ax.set_ylim(ylim)
|
||||
|
||||
if title is not None:
|
||||
ax.set_title(title)
|
||||
if xlabel is not None:
|
||||
ax.set_xlabel(xlabel)
|
||||
if ylabel is not None:
|
||||
ylabel = ylabel.replace("@metric@", metric)
|
||||
ax.set_ylabel(ylabel)
|
||||
ax.grid(grid)
|
||||
return ax
|
||||
|
||||
|
||||
def _determine_direction_for_numeric_split(
|
||||
*,
|
||||
fval: float,
|
||||
threshold: float,
|
||||
missing_type_str: str,
|
||||
default_left: bool,
|
||||
) -> str:
|
||||
missing_type = _MissingType(missing_type_str)
|
||||
if math.isnan(fval) and missing_type != _MissingType.NAN:
|
||||
fval = 0.0
|
||||
if (missing_type == _MissingType.ZERO and _is_zero(fval)) or (
|
||||
missing_type == _MissingType.NAN and math.isnan(fval)
|
||||
):
|
||||
direction = "left" if default_left else "right"
|
||||
else:
|
||||
direction = "left" if fval <= threshold else "right"
|
||||
return direction
|
||||
|
||||
|
||||
def _determine_direction_for_categorical_split(fval: float, thresholds: str) -> str:
|
||||
if math.isnan(fval) or int(fval) < 0:
|
||||
return "right"
|
||||
int_thresholds = {int(t) for t in thresholds.split("||")}
|
||||
return "left" if int(fval) in int_thresholds else "right"
|
||||
|
||||
|
||||
def _to_graphviz(
|
||||
*,
|
||||
tree_info: Dict[str, Any],
|
||||
show_info: List[str],
|
||||
feature_names: Union[List[str], None],
|
||||
precision: Optional[int],
|
||||
orientation: str,
|
||||
constraints: Optional[List[int]],
|
||||
example_case: Optional[Union[np.ndarray, pd_DataFrame]],
|
||||
max_category_values: int,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Convert specified tree to graphviz instance.
|
||||
|
||||
See:
|
||||
- https://graphviz.readthedocs.io/en/stable/api.html#digraph
|
||||
"""
|
||||
try:
|
||||
from graphviz import Digraph # noqa: PLC0415
|
||||
except ImportError as err:
|
||||
raise ImportError("You must install graphviz and restart your session to plot tree.") from err
|
||||
|
||||
def add(
|
||||
root: Dict[str, Any], total_count: int, parent: Optional[str], decision: Optional[str], highlight: bool
|
||||
) -> None:
|
||||
"""Recursively add node or edge."""
|
||||
fillcolor = "white"
|
||||
style = ""
|
||||
tooltip = None
|
||||
if highlight:
|
||||
color = "blue"
|
||||
penwidth = "3"
|
||||
else:
|
||||
color = "black"
|
||||
penwidth = "1"
|
||||
if "split_index" in root: # non-leaf
|
||||
shape = "rectangle"
|
||||
l_dec = "yes"
|
||||
r_dec = "no"
|
||||
threshold = root["threshold"]
|
||||
if root["decision_type"] == "<=":
|
||||
operator = "≤"
|
||||
elif root["decision_type"] == "==":
|
||||
operator = "="
|
||||
else:
|
||||
raise ValueError("Invalid decision type in tree model.")
|
||||
name = f"split{root['split_index']}"
|
||||
split_feature = root["split_feature"]
|
||||
if feature_names is not None:
|
||||
label = f"<B>{feature_names[split_feature]}</B> {operator}"
|
||||
else:
|
||||
label = f"feature <B>{split_feature}</B> {operator} "
|
||||
direction = None
|
||||
if example_case is not None:
|
||||
if root["decision_type"] == "==":
|
||||
direction = _determine_direction_for_categorical_split(
|
||||
fval=example_case[split_feature], thresholds=root["threshold"]
|
||||
)
|
||||
else:
|
||||
direction = _determine_direction_for_numeric_split(
|
||||
fval=example_case[split_feature],
|
||||
threshold=root["threshold"],
|
||||
missing_type_str=root["missing_type"],
|
||||
default_left=root["default_left"],
|
||||
)
|
||||
if root["decision_type"] == "==":
|
||||
category_values = root["threshold"].split("||")
|
||||
if len(category_values) > max_category_values:
|
||||
tooltip = root["threshold"]
|
||||
threshold = "||".join(category_values[:2]) + "||...||" + category_values[-1]
|
||||
|
||||
label += f"<B>{_float2str(threshold, precision)}</B>"
|
||||
for info in ["split_gain", "internal_value", "internal_weight", "internal_count", "data_percentage"]:
|
||||
if info in show_info:
|
||||
output = info.split("_")[-1]
|
||||
if info in {"split_gain", "internal_value", "internal_weight"}:
|
||||
label += f"<br/>{_float2str(root[info], precision)} {output}"
|
||||
elif info == "internal_count":
|
||||
label += f"<br/>{output}: {root[info]}"
|
||||
elif info == "data_percentage":
|
||||
label += f"<br/>{_float2str(root['internal_count'] / total_count * 100, 2)}% of data"
|
||||
|
||||
if constraints:
|
||||
if constraints[root["split_feature"]] == 1:
|
||||
fillcolor = "#ddffdd" # light green
|
||||
if constraints[root["split_feature"]] == -1:
|
||||
fillcolor = "#ffdddd" # light red
|
||||
style = "filled"
|
||||
label = f"<{label}>"
|
||||
add(
|
||||
root=root["left_child"],
|
||||
total_count=total_count,
|
||||
parent=name,
|
||||
decision=l_dec,
|
||||
highlight=highlight and direction == "left",
|
||||
)
|
||||
add(
|
||||
root=root["right_child"],
|
||||
total_count=total_count,
|
||||
parent=name,
|
||||
decision=r_dec,
|
||||
highlight=highlight and direction == "right",
|
||||
)
|
||||
else: # leaf
|
||||
shape = "ellipse"
|
||||
name = f"leaf{root['leaf_index']}"
|
||||
label = f"leaf {root['leaf_index']}: "
|
||||
label += f"<B>{_float2str(root['leaf_value'], precision)}</B>"
|
||||
if "leaf_weight" in show_info:
|
||||
label += f"<br/>{_float2str(root['leaf_weight'], precision)} weight"
|
||||
if "leaf_count" in show_info:
|
||||
label += f"<br/>count: {root['leaf_count']}"
|
||||
if "data_percentage" in show_info:
|
||||
label += f"<br/>{_float2str(root['leaf_count'] / total_count * 100, 2)}% of data"
|
||||
label = f"<{label}>"
|
||||
graph.node(
|
||||
name,
|
||||
label=label,
|
||||
shape=shape,
|
||||
style=style,
|
||||
fillcolor=fillcolor,
|
||||
color=color,
|
||||
penwidth=penwidth,
|
||||
tooltip=tooltip,
|
||||
)
|
||||
if parent is not None:
|
||||
graph.edge(parent, name, decision, color=color, penwidth=penwidth)
|
||||
|
||||
graph = Digraph(**kwargs)
|
||||
rankdir = "LR" if orientation == "horizontal" else "TB"
|
||||
graph.attr("graph", nodesep="0.05", ranksep="0.3", rankdir=rankdir)
|
||||
if "internal_count" in tree_info["tree_structure"]:
|
||||
add(
|
||||
root=tree_info["tree_structure"],
|
||||
total_count=tree_info["tree_structure"]["internal_count"],
|
||||
parent=None,
|
||||
decision=None,
|
||||
highlight=example_case is not None,
|
||||
)
|
||||
else:
|
||||
raise Exception("Cannot plot trees with no split")
|
||||
|
||||
if constraints:
|
||||
# "#ddffdd" is light green, "#ffdddd" is light red
|
||||
legend = """<
|
||||
<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0" CELLPADDING="4">
|
||||
<TR>
|
||||
<TD COLSPAN="2"><B>Monotone constraints</B></TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD>Increasing</TD>
|
||||
<TD BGCOLOR="#ddffdd"></TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD>Decreasing</TD>
|
||||
<TD BGCOLOR="#ffdddd"></TD>
|
||||
</TR>
|
||||
</TABLE>
|
||||
>"""
|
||||
graph.node("legend", label=legend, shape="rectangle", color="white")
|
||||
return graph
|
||||
|
||||
|
||||
def create_tree_digraph(
|
||||
booster: Union[Booster, LGBMModel],
|
||||
tree_index: int = 0,
|
||||
show_info: Optional[List[str]] = None,
|
||||
precision: Optional[int] = 3,
|
||||
orientation: str = "horizontal",
|
||||
example_case: Optional[Union[np.ndarray, pd_DataFrame]] = None,
|
||||
max_category_values: int = 10,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Create a digraph representation of specified tree.
|
||||
|
||||
Each node in the graph represents a node in the tree.
|
||||
|
||||
Non-leaf nodes have labels like ``Column_10 <= 875.9``, which means
|
||||
"this node splits on the feature named "Column_10", with threshold 875.9".
|
||||
|
||||
Leaf nodes have labels like ``leaf 2: 0.422``, which means "this node is a
|
||||
leaf node, and the predicted value for records that fall into this node
|
||||
is 0.422". The number (``2``) is an internal unique identifier and doesn't
|
||||
have any special meaning.
|
||||
|
||||
.. note::
|
||||
|
||||
For more information please visit
|
||||
https://graphviz.readthedocs.io/en/stable/api.html#digraph.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
booster : Booster or LGBMModel
|
||||
Booster or LGBMModel instance to be converted.
|
||||
tree_index : int, optional (default=0)
|
||||
The index of a target tree to convert.
|
||||
show_info : list of str, or None, optional (default=None)
|
||||
What information should be shown in nodes.
|
||||
|
||||
- ``'split_gain'`` : gain from adding this split to the model
|
||||
- ``'internal_value'`` : raw predicted value that would be produced by this node if it was a leaf node
|
||||
- ``'internal_count'`` : number of records from the training data that fall into this non-leaf node
|
||||
- ``'internal_weight'`` : total weight of all nodes that fall into this non-leaf node
|
||||
- ``'leaf_count'`` : number of records from the training data that fall into this leaf node
|
||||
- ``'leaf_weight'`` : total weight (sum of Hessian) of all observations that fall into this leaf node
|
||||
- ``'data_percentage'`` : percentage of training data that fall into this node
|
||||
precision : int or None, optional (default=3)
|
||||
Used to restrict the display of floating point values to a certain precision.
|
||||
orientation : str, optional (default='horizontal')
|
||||
Orientation of the tree.
|
||||
Can be 'horizontal' or 'vertical'.
|
||||
example_case : numpy 2-D array, pandas DataFrame or None, optional (default=None)
|
||||
Single row with the same structure as the training data.
|
||||
If not None, the plot will highlight the path that sample takes through the tree.
|
||||
|
||||
.. versionadded:: 4.0.0
|
||||
|
||||
max_category_values : int, optional (default=10)
|
||||
The maximum number of category values to display in tree nodes, if the number of thresholds is greater than this value, thresholds will be collapsed and displayed on the label tooltip instead.
|
||||
|
||||
.. warning::
|
||||
|
||||
Consider wrapping the SVG string of the tree graph with ``IPython.display.HTML`` when running on JupyterLab to get the `tooltip <https://graphviz.org/docs/attrs/tooltip>`_ working right.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from IPython.display import HTML
|
||||
|
||||
graph = lgb.create_tree_digraph(clf, max_category_values=5)
|
||||
HTML(graph._repr_image_svg_xml())
|
||||
|
||||
.. versionadded:: 4.0.0
|
||||
|
||||
**kwargs
|
||||
Other parameters passed to ``Digraph`` constructor.
|
||||
Check https://graphviz.readthedocs.io/en/stable/api.html#digraph for the full list of supported parameters.
|
||||
|
||||
Returns
|
||||
-------
|
||||
graph : graphviz.Digraph
|
||||
The digraph representation of specified tree.
|
||||
"""
|
||||
if isinstance(booster, LGBMModel):
|
||||
booster = booster.booster_
|
||||
elif not isinstance(booster, Booster):
|
||||
raise TypeError("booster must be Booster or LGBMModel.")
|
||||
|
||||
model = booster.dump_model()
|
||||
tree_infos = model["tree_info"]
|
||||
feature_names = model.get("feature_names", None)
|
||||
monotone_constraints = model.get("monotone_constraints", None)
|
||||
|
||||
if tree_index < len(tree_infos):
|
||||
tree_info = tree_infos[tree_index]
|
||||
else:
|
||||
raise IndexError("tree_index is out of range.")
|
||||
|
||||
if show_info is None:
|
||||
show_info = []
|
||||
|
||||
if example_case is not None:
|
||||
if not isinstance(example_case, (np.ndarray, pd_DataFrame)) or example_case.ndim != 2:
|
||||
raise ValueError("example_case must be a numpy 2-D array or a pandas DataFrame")
|
||||
if example_case.shape[0] != 1:
|
||||
raise ValueError("example_case must have a single row.")
|
||||
if isinstance(example_case, pd_DataFrame):
|
||||
example_case = _data_from_pandas(
|
||||
data=example_case,
|
||||
feature_name="auto",
|
||||
categorical_feature="auto",
|
||||
pandas_categorical=booster.pandas_categorical,
|
||||
)[0]
|
||||
example_case = example_case[0]
|
||||
|
||||
return _to_graphviz(
|
||||
tree_info=tree_info,
|
||||
show_info=show_info,
|
||||
feature_names=feature_names,
|
||||
precision=precision,
|
||||
orientation=orientation,
|
||||
constraints=monotone_constraints,
|
||||
example_case=example_case,
|
||||
max_category_values=max_category_values,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def plot_tree(
|
||||
booster: Union[Booster, LGBMModel],
|
||||
ax: "Optional[matplotlib.axes.Axes]" = None,
|
||||
tree_index: int = 0,
|
||||
figsize: Optional[Tuple[float, float]] = None,
|
||||
dpi: Optional[int] = None,
|
||||
show_info: Optional[List[str]] = None,
|
||||
precision: Optional[int] = 3,
|
||||
orientation: str = "horizontal",
|
||||
example_case: Optional[Union[np.ndarray, pd_DataFrame]] = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Plot specified tree.
|
||||
|
||||
Each node in the graph represents a node in the tree.
|
||||
|
||||
Non-leaf nodes have labels like ``Column_10 <= 875.9``, which means
|
||||
"this node splits on the feature named "Column_10", with threshold 875.9".
|
||||
|
||||
Leaf nodes have labels like ``leaf 2: 0.422``, which means "this node is a
|
||||
leaf node, and the predicted value for records that fall into this node
|
||||
is 0.422". The number (``2``) is an internal unique identifier and doesn't
|
||||
have any special meaning.
|
||||
|
||||
.. note::
|
||||
|
||||
It is preferable to use ``create_tree_digraph()`` because of its lossless quality
|
||||
and returned objects can be also rendered and displayed directly inside a Jupyter notebook.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
booster : Booster or LGBMModel
|
||||
Booster or LGBMModel instance to be plotted.
|
||||
ax : matplotlib.axes.Axes or None, optional (default=None)
|
||||
Target axes instance.
|
||||
If None, new figure and axes will be created.
|
||||
tree_index : int, optional (default=0)
|
||||
The index of a target tree to plot.
|
||||
figsize : tuple of 2 elements or None, optional (default=None)
|
||||
Figure size.
|
||||
dpi : int or None, optional (default=None)
|
||||
Resolution of the figure.
|
||||
show_info : list of str, or None, optional (default=None)
|
||||
What information should be shown in nodes.
|
||||
|
||||
- ``'split_gain'`` : gain from adding this split to the model
|
||||
- ``'internal_value'`` : raw predicted value that would be produced by this node if it was a leaf node
|
||||
- ``'internal_count'`` : number of records from the training data that fall into this non-leaf node
|
||||
- ``'internal_weight'`` : total weight of all nodes that fall into this non-leaf node
|
||||
- ``'leaf_count'`` : number of records from the training data that fall into this leaf node
|
||||
- ``'leaf_weight'`` : total weight (sum of Hessian) of all observations that fall into this leaf node
|
||||
- ``'data_percentage'`` : percentage of training data that fall into this node
|
||||
precision : int or None, optional (default=3)
|
||||
Used to restrict the display of floating point values to a certain precision.
|
||||
orientation : str, optional (default='horizontal')
|
||||
Orientation of the tree.
|
||||
Can be 'horizontal' or 'vertical'.
|
||||
example_case : numpy 2-D array, pandas DataFrame or None, optional (default=None)
|
||||
Single row with the same structure as the training data.
|
||||
If not None, the plot will highlight the path that sample takes through the tree.
|
||||
|
||||
.. versionadded:: 4.0.0
|
||||
|
||||
**kwargs
|
||||
Other parameters passed to ``Digraph`` constructor.
|
||||
Check https://graphviz.readthedocs.io/en/stable/api.html#digraph for the full list of supported parameters.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ax : matplotlib.axes.Axes
|
||||
The plot with single tree.
|
||||
"""
|
||||
try:
|
||||
import matplotlib.image # noqa: PLC0415
|
||||
import matplotlib.pyplot as plt # noqa: PLC0415
|
||||
except ImportError as err:
|
||||
raise ImportError("You must install matplotlib and restart your session to plot tree.") from err
|
||||
|
||||
if ax is None:
|
||||
if figsize is not None:
|
||||
_check_not_tuple_of_2_elements(figsize, "figsize")
|
||||
_, ax = plt.subplots(1, 1, figsize=figsize, dpi=dpi)
|
||||
|
||||
graph = create_tree_digraph(
|
||||
booster=booster,
|
||||
tree_index=tree_index,
|
||||
show_info=show_info,
|
||||
precision=precision,
|
||||
orientation=orientation,
|
||||
example_case=example_case,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
s = BytesIO()
|
||||
s.write(graph.pipe(format="png"))
|
||||
s.seek(0)
|
||||
img = matplotlib.image.imread(s)
|
||||
|
||||
ax.imshow(img)
|
||||
ax.axis("off")
|
||||
return ax
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,229 @@
|
||||
[project]
|
||||
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Intended Audience :: Science/Research",
|
||||
"Natural Language :: English",
|
||||
"Operating System :: MacOS",
|
||||
"Operating System :: Microsoft :: Windows",
|
||||
"Operating System :: POSIX",
|
||||
"Operating System :: Unix",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3.14",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence"
|
||||
]
|
||||
dependencies = [
|
||||
"narwhals>=1.15",
|
||||
"numpy>=1.21.3",
|
||||
"scipy"
|
||||
]
|
||||
description = "LightGBM Python-package"
|
||||
license = "MIT"
|
||||
license-files = [
|
||||
"LICENSE"
|
||||
]
|
||||
maintainers = [
|
||||
{name = "Yu Shi", email = "yushi@microsoft.com"}
|
||||
]
|
||||
name = "lightgbm"
|
||||
readme = "README.rst"
|
||||
requires-python = ">=3.10"
|
||||
version = "4.6.0.99"
|
||||
|
||||
[project.optional-dependencies]
|
||||
arrow = [
|
||||
"narwhals[pyarrow]",
|
||||
"pyarrow>=16.0.0"
|
||||
]
|
||||
dask = [
|
||||
"dask[array,dataframe,distributed]>=2.0.0",
|
||||
"narwhals[pandas]",
|
||||
"pandas>=1.3.4"
|
||||
]
|
||||
pandas = [
|
||||
"narwhals[pandas]",
|
||||
"pandas>=1.3.4"
|
||||
]
|
||||
plotting = [
|
||||
"graphviz",
|
||||
"matplotlib"
|
||||
]
|
||||
polars = [
|
||||
"narwhals[polars]",
|
||||
"polars>=1.0.0"
|
||||
]
|
||||
scikit-learn = [
|
||||
"scikit-learn>=1.0.2"
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
homepage = "https://github.com/lightgbm-org/LightGBM"
|
||||
documentation = "https://lightgbm.readthedocs.io/en/latest/"
|
||||
repository = "https://github.com/lightgbm-org/LightGBM.git"
|
||||
changelog = "https://github.com/lightgbm-org/LightGBM/releases"
|
||||
|
||||
# start:build-system
|
||||
[build-system]
|
||||
|
||||
requires = ["scikit-build-core>=0.11.0"]
|
||||
build-backend = "scikit_build_core.build"
|
||||
|
||||
# based on https://github.com/scikit-build/scikit-build-core#configuration
|
||||
[tool.scikit-build]
|
||||
ninja.version = ">=1.11"
|
||||
ninja.make-fallback = true
|
||||
build.verbose = false
|
||||
build.targets = ["_lightgbm"]
|
||||
install.strip = true
|
||||
logging.level = "INFO"
|
||||
sdist.reproducible = true
|
||||
wheel.py-api = "py3"
|
||||
experimental = false
|
||||
strict-config = false
|
||||
minimum-version = "build-system.requires"
|
||||
|
||||
[tool.scikit-build.cmake]
|
||||
version = "CMakeLists.txt"
|
||||
build-type = "Release"
|
||||
|
||||
[tool.scikit-build.cmake.define]
|
||||
__BUILD_FOR_PYTHON = "ON"
|
||||
|
||||
# end:build-system
|
||||
|
||||
[tool.mypy]
|
||||
disallow_untyped_defs = true
|
||||
exclude = 'build/*|compile/*|docs/*|examples/*|external_libs/*|lightgbm-python/*|.pixi/*|tests/*'
|
||||
ignore_missing_imports = true
|
||||
|
||||
[tool.rstcheck]
|
||||
report_level = "WARNING"
|
||||
ignore_directives = [
|
||||
"autoclass",
|
||||
"autofunction",
|
||||
"autosummary",
|
||||
"doxygenfile"
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
exclude = [
|
||||
"build",
|
||||
"compile",
|
||||
"external_libs",
|
||||
"lightgbm-python",
|
||||
]
|
||||
line-length = 120
|
||||
|
||||
# this should be set to the oldest version of python LightGBM supports
|
||||
target-version = "py310"
|
||||
|
||||
[tool.ruff.format]
|
||||
docstring-code-format = false
|
||||
exclude = [
|
||||
"build/*.py",
|
||||
"compile/*.py",
|
||||
"external_libs/*.py",
|
||||
"lightgbm-python/*.py",
|
||||
]
|
||||
indent-style = "space"
|
||||
quote-style = "double"
|
||||
skip-magic-trailing-comma = false
|
||||
|
||||
[tool.ruff.lint]
|
||||
ignore = [
|
||||
# (pydocstyle) Missing docstring in magic method
|
||||
"D105",
|
||||
# (pycodestyle) Line too long
|
||||
"E501",
|
||||
# (pylint) Too many branches
|
||||
"PLR0912",
|
||||
# (pylint) Too many arguments in function definition
|
||||
"PLR0913",
|
||||
# (pylint) Too many statements
|
||||
"PLR0915",
|
||||
# (pylint) Consider merging multiple comparisons
|
||||
"PLR1714",
|
||||
# (pylint) Magic value used in comparison
|
||||
"PLR2004",
|
||||
# (pylint) for loop variable overwritten by assignment target
|
||||
"PLW2901",
|
||||
# (pylint) use 'elif' instead of 'else' then 'if', to reduce indentation
|
||||
"PLR5501",
|
||||
# (flake8-pytest-style) `scope='function'` is implied in `@pytest.fixture()`
|
||||
"PT003"
|
||||
]
|
||||
select = [
|
||||
# flake8-bugbear
|
||||
"B",
|
||||
# flake8-comprehensions
|
||||
"C4",
|
||||
# pydocstyle
|
||||
"D",
|
||||
# pycodestyle (errors)
|
||||
"E",
|
||||
# pyflakes
|
||||
"F",
|
||||
# isort
|
||||
"I",
|
||||
# NumPy-specific rules
|
||||
"NPY",
|
||||
# pylint
|
||||
"PL",
|
||||
# flake8-pytest-style
|
||||
"PT",
|
||||
# flake8-return: unnecessary assignment before return
|
||||
"RET504",
|
||||
# flake8-return: superfluous-else-raise
|
||||
"RET506",
|
||||
# flake8-simplify: use dict.get() instead of an if-else block
|
||||
"SIM401",
|
||||
# flake8-print
|
||||
"T",
|
||||
# pycodestyle (warnings)
|
||||
"W",
|
||||
]
|
||||
|
||||
[tool.ruff.lint.flake8-pytest-style]
|
||||
raises-extend-require-match-for = ["*Exception", "*Error"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
".ci/create-nuget.py" = [
|
||||
# (flake8-print) flake8-print
|
||||
"T201"
|
||||
]
|
||||
"docs/conf.py" = [
|
||||
# (flake8-bugbear) raise exceptions with "raise ... from err"
|
||||
"B904",
|
||||
# (flake8-print) flake8-print
|
||||
"T"
|
||||
]
|
||||
"examples/*" = [
|
||||
# pydocstyle
|
||||
"D",
|
||||
# flake8-print
|
||||
"T"
|
||||
]
|
||||
"python-package/lightgbm/basic.py" = [
|
||||
# (pylint) Using the global statement is discouraged
|
||||
"PLW0603"
|
||||
]
|
||||
"tests/*" = [
|
||||
# (flake8-bugbear) Found useless expression
|
||||
"B018",
|
||||
# (pycodestyle) Module level import not at top of file
|
||||
"E402",
|
||||
# pydocstyle
|
||||
"D",
|
||||
# flake8-print
|
||||
"T"
|
||||
]
|
||||
|
||||
[tool.ruff.lint.pydocstyle]
|
||||
convention = "numpy"
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
known-first-party = ["lightgbm"]
|
||||
Reference in New Issue
Block a user