chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,32 @@
|
||||
verbose = "info"
|
||||
no_progress = false
|
||||
cache = false
|
||||
scheme = ["http", "https", "file"]
|
||||
include_mail = false
|
||||
include_fragments = true
|
||||
no_ignore = true
|
||||
insecure = false
|
||||
require_https = true
|
||||
accept = ["100..=103", "200..=299"]
|
||||
user_agent = "curl/7.88.1"
|
||||
header = {"User-Agent" = "curl/7.88.1"}
|
||||
timeout = 30
|
||||
retry_wait_time = 10
|
||||
max_concurrency = 10
|
||||
# remove anchors from GitHub URLs to overcome https://github.com/lycheeverse/lychee/issues/1729
|
||||
remap = [
|
||||
'(?P<host>^https://github\.com)/(?P<path>.*)#(?P<anchor>.*)$ $host/$path/',
|
||||
]
|
||||
exclude = [
|
||||
'^https://www\.swig\.org/download\.html$',
|
||||
'^https://proceedings\.neurips\.cc/.*',
|
||||
'^https://www\.amd\.com/en/support\.html$',
|
||||
'^https://www\.jstor\.org/stable/2281952$',
|
||||
'^https://dl\.acm\.org/doi/10\.1145/3298689\.3347033$',
|
||||
'^https://packages\.ubuntu\.com/search.*',
|
||||
'^https://stackoverflow\.com/.*',
|
||||
'^https://.*\.stackexchange\.com/.*',
|
||||
]
|
||||
exclude_path = [
|
||||
"(^|/)docs/.*\\.rst",
|
||||
]
|
||||
@@ -0,0 +1,121 @@
|
||||
Advanced Topics
|
||||
===============
|
||||
|
||||
Missing Value Handle
|
||||
--------------------
|
||||
|
||||
- LightGBM enables the missing value handle by default. Disable it by setting ``use_missing=false``.
|
||||
|
||||
- LightGBM uses NA (NaN) to represent missing values by default. Change it to use zero by setting ``zero_as_missing=true``.
|
||||
|
||||
- When ``zero_as_missing=false`` (default), the unrecorded values in sparse matrices (and LightSVM) are treated as zeros.
|
||||
|
||||
- When ``zero_as_missing=true``, NA and zeros (including unrecorded values in sparse matrices (and LightSVM)) are treated as missing.
|
||||
|
||||
Categorical Feature Support
|
||||
---------------------------
|
||||
|
||||
- LightGBM offers good accuracy with integer-encoded categorical features. LightGBM applies
|
||||
`Fisher (1958) <https://www.jstor.org/stable/2281952>`_
|
||||
to find the optimal split over categories as
|
||||
`described here <./Features.rst#optimal-split-for-categorical-features>`_. This often performs better than one-hot encoding.
|
||||
|
||||
- Use ``categorical_feature`` to specify the categorical features.
|
||||
Refer to the parameter ``categorical_feature`` in `Parameters <./Parameters.rst#categorical_feature>`__.
|
||||
|
||||
- Categorical features will be cast to ``int32`` (integer codes will be extracted from pandas categoricals in the Python-package) so they must be encoded as non-negative integers (negative values will be treated as missing)
|
||||
less than ``Int32.MaxValue`` (2147483647).
|
||||
It is best to use a contiguous range of integers started from zero.
|
||||
Floating point numbers in categorical features will be rounded towards 0.
|
||||
|
||||
- When using ``pandas.DataFrame`` inputs with columns of dtype ``category``, LightGBM will
|
||||
align categories to those observed during training before converting them to integer values.
|
||||
This ensures consistent encoding between training and prediction without additional preprocessing.
|
||||
|
||||
- At ``predict()`` time, categories not seen during training will be treated as missing values.
|
||||
|
||||
- Use ``min_data_per_group``, ``cat_smooth`` to deal with over-fitting (when ``#data`` is small or ``#category`` is large).
|
||||
|
||||
- For a categorical feature with high cardinality (``#category`` is large), it often works best to
|
||||
treat the feature as numeric, either by simply ignoring the categorical interpretation of the integers or
|
||||
by embedding the categories in a low-dimensional numeric space.
|
||||
|
||||
LambdaRank
|
||||
----------
|
||||
|
||||
- The label should be of type ``int``, such that larger numbers correspond to higher relevance (e.g. 0:bad, 1:fair, 2:good, 3:perfect).
|
||||
|
||||
- Use ``label_gain`` to set the gain(weight) of ``int`` label.
|
||||
|
||||
- Use ``lambdarank_truncation_level`` to truncate the max DCG.
|
||||
|
||||
Cost Efficient Gradient Boosting
|
||||
--------------------------------
|
||||
|
||||
`Cost Efficient Gradient Boosting <https://proceedings.neurips.cc/paper/2017/hash/4fac9ba115140ac4f1c22da82aa0bc7f-Abstract.html>`_ (CEGB) makes it possible to penalise boosting based on the cost of obtaining feature values.
|
||||
CEGB penalises learning in the following ways:
|
||||
|
||||
- Each time a tree is split, a penalty of ``cegb_penalty_split`` is applied.
|
||||
- When a feature is used for the first time, ``cegb_penalty_feature_coupled`` is applied. This penalty can be different for each feature and should be specified as one ``double`` per feature.
|
||||
- When a feature is used for the first time for a data row, ``cegb_penalty_feature_lazy`` is applied. Like ``cegb_penalty_feature_coupled``, this penalty is specified as one ``double`` per feature.
|
||||
|
||||
Each of the penalties above is scaled by ``cegb_tradeoff``.
|
||||
Using this parameter, it is possible to change the overall strength of the CEGB penalties by changing only one parameter.
|
||||
|
||||
Parameters Tuning
|
||||
-----------------
|
||||
|
||||
- Refer to `Parameters Tuning <./Parameters-Tuning.rst>`__.
|
||||
|
||||
.. _Parallel Learning:
|
||||
|
||||
Distributed Learning
|
||||
--------------------
|
||||
|
||||
- Refer to `Distributed Learning Guide <./Parallel-Learning-Guide.rst>`__.
|
||||
|
||||
GPU Support
|
||||
-----------
|
||||
|
||||
- Refer to `GPU Tutorial <./GPU-Tutorial.rst>`__ and `GPU Targets <./GPU-Targets.rst>`__.
|
||||
|
||||
Support for Position Bias Treatment
|
||||
------------------------------------
|
||||
|
||||
Often the relevance labels provided in Learning-to-Rank tasks might be derived from implicit user feedback (e.g., clicks) and therefore might be biased due to their position/location on the screen when having been presented to a user.
|
||||
LightGBM can make use of positional data.
|
||||
|
||||
For example, consider the case where you expect that the first 3 results from a search engine will be visible in users' browsers without scrolling, and all other results for a query would require scrolling.
|
||||
|
||||
LightGBM could be told to account for the position bias from results being "above the fold" by providing a ``positions`` array encoded as follows:
|
||||
|
||||
::
|
||||
|
||||
0
|
||||
0
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
0
|
||||
0
|
||||
1
|
||||
...
|
||||
|
||||
Where ``0 = "above the fold"`` and ``1 = "requires scrolling"``.
|
||||
The specific values are not important, as long as they are consistent across all observations in the training data.
|
||||
An encoding like ``100 = "above the fold"`` and ``17 = "requires scrolling"`` would result in exactly the same trained model.
|
||||
|
||||
In that way, ``positions`` in LightGBM's API are similar to a categorical feature.
|
||||
Just as with non-ordinal categorical features, an integer representation is just used for memory and computational efficiency... LightGBM does not care about the absolute or relative magnitude of the values.
|
||||
|
||||
Unlike a categorical feature, however, ``positions`` are used to adjust the target to reduce the bias in predictions made by the trained model.
|
||||
|
||||
The position file corresponds with training data file line by line, and has one position per line. And if the name of training data file is ``train.txt``, the position file should be named as ``train.txt.position`` and placed in the same folder as the data file.
|
||||
In this case, LightGBM will load the position file automatically if it exists. The positions can also be specified through the ``Dataset`` constructor when using Python API. If the positions are specified in both approaches, the ``.position`` file will be ignored.
|
||||
|
||||
Currently, implemented is an approach to model position bias by using an idea of Generalized Additive Models (`GAM <https://en.wikipedia.org/wiki/Generalized_additive_model>`_) to linearly decompose the document score ``s`` into the sum of a relevance component ``f`` and a positional component ``g``: ``s(x, pos) = f(x) + g(pos)`` where the former component depends on the original query-document features and the latter depends on the position of an item.
|
||||
During the training, the compound scoring function ``s(x, pos)`` is fit with a standard ranking algorithm (e.g., LambdaMART) which boils down to jointly learning the relevance component ``f(x)`` (it is later returned as an unbiased model) and the position factors ``g(pos)`` that help better explain the observed (biased) labels.
|
||||
Similar score decomposition ideas have previously been applied for classification & pointwise ranking tasks with assumptions of binary labels and binary relevance (a.k.a. "two-tower" models, refer to the papers: `Towards Disentangling Relevance and Bias in Unbiased Learning to Rank <https://arxiv.org/abs/2212.13937>`_, `PAL: a position-bias aware learning framework for CTR prediction in live recommender systems <https://dl.acm.org/doi/10.1145/3298689.3347033>`_, `A General Framework for Debiasing in CTR Prediction <https://arxiv.org/abs/2112.02767>`_).
|
||||
In LightGBM, we adapt this idea to general pairwise Lerarning-to-Rank with arbitrary ordinal relevance labels.
|
||||
Besides, GAMs have been used in the context of explainable ML (`Accurate Intelligible Models with Pairwise Interactions <https://www.cs.cornell.edu/~yinlou/projects/gam/>`_) to linearly decompose the contribution of each feature (and possibly their pairwise interactions) to the overall score, for subsequent analysis and interpretation of their effects in the trained models.
|
||||
@@ -0,0 +1,4 @@
|
||||
C API
|
||||
=====
|
||||
|
||||
.. doxygenfile:: c_api.h
|
||||
@@ -0,0 +1,95 @@
|
||||
Development Guide
|
||||
=================
|
||||
|
||||
Algorithms
|
||||
----------
|
||||
|
||||
Refer to `Features <./Features.rst>`__ for understanding of important algorithms used in LightGBM.
|
||||
|
||||
Classes and Code Structure
|
||||
--------------------------
|
||||
|
||||
Important Classes
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| Class | Description |
|
||||
+=========================+========================================================================================+
|
||||
| ``Application`` | The entrance of application, including training and prediction logic |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``Bin`` | Data structure used for storing feature discrete values (converted from float values) |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``Boosting`` | Boosting interface (GBDT, DART, etc.) |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``Config`` | Stores parameters and configurations |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``Dataset`` | Stores information of dataset |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``DatasetLoader`` | Used to construct dataset |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``FeatureGroup`` | Stores the data of feature, could be multiple features |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``Metric`` | Evaluation metrics |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``Network`` | Network interfaces and communication algorithms |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``ObjectiveFunction`` | Objective functions used to train |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``Tree`` | Stores information of tree model |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
| ``TreeLearner`` | Used to learn trees |
|
||||
+-------------------------+----------------------------------------------------------------------------------------+
|
||||
|
||||
Code Structure
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| Path | Description |
|
||||
+=====================+====================================================================================================================================+
|
||||
| ./include | Header files |
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| ./include/utils | Some common functions |
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| ./src/application | Implementations of training and prediction logic |
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| ./src/boosting | Implementations of Boosting |
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| ./src/io | Implementations of IO related classes, including ``Bin``, ``Config``, ``Dataset``, ``DatasetLoader``, ``Feature`` and ``Tree`` |
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| ./src/metric | Implementations of metrics |
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| ./src/network | Implementations of network functions |
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| ./src/objective | Implementations of objective functions |
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
| ./src/treelearner | Implementations of tree learners |
|
||||
+---------------------+------------------------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
Documents API
|
||||
-------------
|
||||
|
||||
Refer to `docs README <./README.rst>`__.
|
||||
|
||||
C API
|
||||
-----
|
||||
|
||||
Refer to `C API <./C-API.rst>`__ or the comments in `c\_api.h <https://github.com/lightgbm-org/LightGBM/blob/main/include/LightGBM/c_api.h>`__ file, from which the documentation is generated.
|
||||
|
||||
Tests
|
||||
-----
|
||||
|
||||
C++ unit tests are located in the ``./tests/cpp_tests`` folder and written with the help of Google Test framework.
|
||||
To run tests locally first refer to the `Installation Guide <./Installation-Guide.rst#build-c-unit-tests>`__ for how to build tests and then simply run compiled executable file.
|
||||
It is highly recommended to build tests with `sanitizers <./Installation-Guide.rst#sanitizers>`__.
|
||||
|
||||
High Level Language Package
|
||||
---------------------------
|
||||
|
||||
See the implementations at `Python-package <https://github.com/lightgbm-org/LightGBM/tree/main/python-package>`__ and `R-package <https://github.com/lightgbm-org/LightGBM/tree/main/R-package>`__.
|
||||
|
||||
Questions
|
||||
---------
|
||||
|
||||
Refer to `FAQ <./FAQ.rst>`__.
|
||||
|
||||
Also feel free to open `issues <https://github.com/lightgbm-org/LightGBM/issues>`__ if you met problems.
|
||||
@@ -0,0 +1,253 @@
|
||||
Experiments
|
||||
===========
|
||||
|
||||
Comparison Experiment
|
||||
---------------------
|
||||
|
||||
For the detailed experiment scripts and output logs, please refer to this `repo`_.
|
||||
|
||||
History
|
||||
^^^^^^^
|
||||
|
||||
08 Mar, 2020: update according to the latest master branch (`1b97eaf <https://github.com/dmlc/xgboost/commit/1b97eaf7a74315bfa2c132d59f937a35408bcfd1>`__ for XGBoost, `bcad692 <https://github.com/lightgbm-org/LightGBM/commit/bcad692e263e0317cab11032dd017c78f9e58e5f>`__ for LightGBM). (``xgboost_exact`` is not updated for it is too slow.)
|
||||
|
||||
27 Feb, 2017: first version.
|
||||
|
||||
Data
|
||||
^^^^
|
||||
|
||||
We used 5 datasets to conduct our comparison experiments. Details of data are listed in the following table:
|
||||
|
||||
+-----------+-----------------------+---------------------------------------------------------------------------------+-------------+----------+----------------------------------------------+
|
||||
| Data | Task | Link | #Train\_Set | #Feature | Comments |
|
||||
+===========+=======================+=================================================================================+=============+==========+==============================================+
|
||||
| Higgs | Binary classification | `link <https://archive.ics.uci.edu/dataset/280/higgs>`__ | 10,500,000 | 28 | last 500,000 samples were used as test set |
|
||||
+-----------+-----------------------+---------------------------------------------------------------------------------+-------------+----------+----------------------------------------------+
|
||||
| Yahoo LTR | Learning to rank | `link <https://proceedings.mlr.press/v14/chapelle11a.html>`__ | 473,134 | 700 | set1.train as train, set1.test as test |
|
||||
+-----------+-----------------------+---------------------------------------------------------------------------------+-------------+----------+----------------------------------------------+
|
||||
| MS LTR | Learning to rank | `link <https://www.microsoft.com/en-us/research/project/mslr/>`__ | 2,270,296 | 137 | {S1,S2,S3} as train set, {S5} as test set |
|
||||
+-----------+-----------------------+---------------------------------------------------------------------------------+-------------+----------+----------------------------------------------+
|
||||
| Expo | Binary classification | `link <https://community.amstat.org/jointscsg-section/dataexpo/dataexpo2009>`__ | 11,000,000 | 700 | last 1,000,000 samples were used as test set |
|
||||
+-----------+-----------------------+---------------------------------------------------------------------------------+-------------+----------+----------------------------------------------+
|
||||
| Allstate | Binary classification | `link <https://www.kaggle.com/c/ClaimPredictionChallenge>`__ | 13,184,290 | 4228 | last 1,000,000 samples were used as test set |
|
||||
+-----------+-----------------------+---------------------------------------------------------------------------------+-------------+----------+----------------------------------------------+
|
||||
|
||||
Environment
|
||||
^^^^^^^^^^^
|
||||
|
||||
We ran all experiments on a single Linux server (Azure ND24s) with the following specifications:
|
||||
|
||||
+------------------+-----------------+---------------------+
|
||||
| OS | CPU | Memory |
|
||||
+==================+=================+=====================+
|
||||
| Ubuntu 16.04 LTS | 2 \* E5-2690 v4 | 448GB |
|
||||
+------------------+-----------------+---------------------+
|
||||
|
||||
Baseline
|
||||
^^^^^^^^
|
||||
|
||||
We used `xgboost`_ as a baseline.
|
||||
|
||||
Both xgboost and LightGBM were built with OpenMP support.
|
||||
|
||||
Settings
|
||||
^^^^^^^^
|
||||
|
||||
We set up total 3 settings for experiments. The parameters of these settings are:
|
||||
|
||||
1. xgboost:
|
||||
|
||||
.. code:: text
|
||||
|
||||
eta = 0.1
|
||||
max_depth = 8
|
||||
num_round = 500
|
||||
nthread = 16
|
||||
tree_method = exact
|
||||
min_child_weight = 100
|
||||
|
||||
2. xgboost\_hist (using histogram based algorithm):
|
||||
|
||||
.. code:: text
|
||||
|
||||
eta = 0.1
|
||||
num_round = 500
|
||||
nthread = 16
|
||||
min_child_weight = 100
|
||||
tree_method = hist
|
||||
grow_policy = lossguide
|
||||
max_depth = 0
|
||||
max_leaves = 255
|
||||
|
||||
3. LightGBM:
|
||||
|
||||
.. code:: text
|
||||
|
||||
learning_rate = 0.1
|
||||
num_leaves = 255
|
||||
num_trees = 500
|
||||
num_threads = 16
|
||||
min_data_in_leaf = 0
|
||||
min_sum_hessian_in_leaf = 100
|
||||
|
||||
xgboost grows trees depth-wise and controls model complexity by ``max_depth``.
|
||||
LightGBM uses a leaf-wise algorithm instead and controls model complexity by ``num_leaves``.
|
||||
So we cannot compare them in the exact same model setting. For the tradeoff, we use xgboost with ``max_depth=8``, which will have max number leaves to 255, to compare with LightGBM with ``num_leaves=255``.
|
||||
|
||||
Other parameters are default values.
|
||||
|
||||
Result
|
||||
^^^^^^
|
||||
|
||||
Speed
|
||||
'''''
|
||||
|
||||
We compared speed using only the training task without any test or metric output. We didn't count the time for IO.
|
||||
For the ranking tasks, since XGBoost and LightGBM implement different ranking objective functions, we used ``regression`` objective for speed benchmark, for the fair comparison.
|
||||
|
||||
The following table is the comparison of time cost:
|
||||
|
||||
+-----------+-----------+---------------+---------------+
|
||||
| Data | xgboost | xgboost\_hist | LightGBM |
|
||||
+===========+===========+===============+===============+
|
||||
| Higgs | 3794.34 s | 165.575 s | **130.094 s** |
|
||||
+-----------+-----------+---------------+---------------+
|
||||
| Yahoo LTR | 674.322 s | 131.462 s | **76.229 s** |
|
||||
+-----------+-----------+---------------+---------------+
|
||||
| MS LTR | 1251.27 s | 98.386 s | **70.417 s** |
|
||||
+-----------+-----------+---------------+---------------+
|
||||
| Expo | 1607.35 s | 137.65 s | **62.607 s** |
|
||||
+-----------+-----------+---------------+---------------+
|
||||
| Allstate | 2867.22 s | 315.256 s | **148.231 s** |
|
||||
+-----------+-----------+---------------+---------------+
|
||||
|
||||
LightGBM ran faster than xgboost on all experiment data sets.
|
||||
|
||||
Accuracy
|
||||
''''''''
|
||||
|
||||
We computed all accuracy metrics only on the test data set.
|
||||
|
||||
+-----------+-----------------+----------+-------------------+--------------+
|
||||
| Data | Metric | xgboost | xgboost\_hist | LightGBM |
|
||||
+===========+=================+==========+===================+==============+
|
||||
| Higgs | AUC | 0.839593 | 0.845314 | **0.845724** |
|
||||
+-----------+-----------------+----------+-------------------+--------------+
|
||||
| Yahoo LTR | NDCG\ :sub:`1` | 0.719748 | 0.720049 | **0.732981** |
|
||||
| +-----------------+----------+-------------------+--------------+
|
||||
| | NDCG\ :sub:`3` | 0.717813 | 0.722573 | **0.735689** |
|
||||
| +-----------------+----------+-------------------+--------------+
|
||||
| | NDCG\ :sub:`5` | 0.737849 | 0.740899 | **0.75352** |
|
||||
| +-----------------+----------+-------------------+--------------+
|
||||
| | NDCG\ :sub:`10` | 0.78089 | 0.782957 | **0.793498** |
|
||||
+-----------+-----------------+----------+-------------------+--------------+
|
||||
| MS LTR | NDCG\ :sub:`1` | 0.483956 | 0.485115 | **0.517767** |
|
||||
| +-----------------+----------+-------------------+--------------+
|
||||
| | NDCG\ :sub:`3` | 0.467951 | 0.47313 | **0.501063** |
|
||||
| +-----------------+----------+-------------------+--------------+
|
||||
| | NDCG\ :sub:`5` | 0.472476 | 0.476375 | **0.504648** |
|
||||
| +-----------------+----------+-------------------+--------------+
|
||||
| | NDCG\ :sub:`10` | 0.492429 | 0.496553 | **0.524252** |
|
||||
+-----------+-----------------+----------+-------------------+--------------+
|
||||
| Expo | AUC | 0.756713 | 0.776224 | **0.776935** |
|
||||
+-----------+-----------------+----------+-------------------+--------------+
|
||||
| Allstate | AUC | 0.607201 | **0.609465** | 0.609072 |
|
||||
+-----------+-----------------+----------+-------------------+--------------+
|
||||
|
||||
Memory Consumption
|
||||
''''''''''''''''''
|
||||
|
||||
We monitored RES while running training task. And we set ``two_round=true`` (this will increase data-loading time and
|
||||
reduce peak memory usage but not affect training speed or accuracy) in LightGBM to reduce peak memory usage.
|
||||
|
||||
+-----------+---------+---------------+--------------------+--------------------+
|
||||
| Data | xgboost | xgboost\_hist | LightGBM (col-wise)|LightGBM (row-wise) |
|
||||
+===========+=========+===============+====================+====================+
|
||||
| Higgs | 4.853GB | 7.335GB | **0.897GB** | 1.401GB |
|
||||
+-----------+---------+---------------+--------------------+--------------------+
|
||||
| Yahoo LTR | 1.907GB | 4.023GB | **1.741GB** | 2.161GB |
|
||||
+-----------+---------+---------------+--------------------+--------------------+
|
||||
| MS LTR | 5.469GB | 7.491GB | **0.940GB** | 1.296GB |
|
||||
+-----------+---------+---------------+--------------------+--------------------+
|
||||
| Expo | 1.553GB | 2.606GB | **0.555GB** | 0.711GB |
|
||||
+-----------+---------+---------------+--------------------+--------------------+
|
||||
| Allstate | 6.237GB | 12.090GB | **1.116GB** | 1.755GB |
|
||||
+-----------+---------+---------------+--------------------+--------------------+
|
||||
|
||||
Parallel Experiment
|
||||
-------------------
|
||||
|
||||
History
|
||||
^^^^^^^
|
||||
|
||||
27 Feb, 2017: first version.
|
||||
|
||||
Data
|
||||
^^^^
|
||||
|
||||
We used a terabyte click log dataset to conduct parallel experiments. Details are listed in following table:
|
||||
|
||||
+--------+-----------------------+---------+---------------+----------+
|
||||
| Data | Task | Link | #Data | #Feature |
|
||||
+========+=======================+=========+===============+==========+
|
||||
| Criteo | Binary classification | `link`_ | 1,700,000,000 | 67 |
|
||||
+--------+-----------------------+---------+---------------+----------+
|
||||
|
||||
This data contains 13 integer features and 26 categorical features for 24 days of click logs.
|
||||
We statisticized the click-through rate (CTR) and count for these 26 categorical features from the first ten days.
|
||||
Then we used next ten days' data, after replacing the categorical features by the corresponding CTR and count, as training data.
|
||||
The processed training data have a total of 1.7 billions records and 67 features.
|
||||
|
||||
Environment
|
||||
^^^^^^^^^^^
|
||||
|
||||
We ran our experiments on 16 Windows servers with the following specifications:
|
||||
|
||||
+---------------------+-----------------+---------------------+-------------------------------------------+
|
||||
| OS | CPU | Memory | Network Adapter |
|
||||
+=====================+=================+=====================+===========================================+
|
||||
| Windows Server 2012 | 2 \* E5-2670 v2 | DDR3 1600Mhz, 256GB | Mellanox ConnectX-3, 54Gbps, RDMA support |
|
||||
+---------------------+-----------------+---------------------+-------------------------------------------+
|
||||
|
||||
Settings
|
||||
^^^^^^^^
|
||||
|
||||
.. code:: text
|
||||
|
||||
learning_rate = 0.1
|
||||
num_leaves = 255
|
||||
num_trees = 100
|
||||
num_thread = 16
|
||||
tree_learner = data
|
||||
|
||||
We used data parallel here because this data is large in ``#data`` but small in ``#feature``. Other parameters were default values.
|
||||
|
||||
Results
|
||||
^^^^^^^
|
||||
|
||||
+----------+---------------+---------------------------+
|
||||
| #Machine | Time per Tree | Memory Usage(per Machine) |
|
||||
+==========+===============+===========================+
|
||||
| 1 | 627.8 s | 176GB |
|
||||
+----------+---------------+---------------------------+
|
||||
| 2 | 311 s | 87GB |
|
||||
+----------+---------------+---------------------------+
|
||||
| 4 | 156 s | 43GB |
|
||||
+----------+---------------+---------------------------+
|
||||
| 8 | 80 s | 22GB |
|
||||
+----------+---------------+---------------------------+
|
||||
| 16 | 42 s | 11GB |
|
||||
+----------+---------------+---------------------------+
|
||||
|
||||
The results show that LightGBM achieves a linear speedup with distributed learning.
|
||||
|
||||
GPU Experiments
|
||||
---------------
|
||||
|
||||
Refer to `GPU Performance <./GPU-Performance.rst>`__.
|
||||
|
||||
.. _repo: https://github.com/guolinke/boosting_tree_benchmarks
|
||||
|
||||
.. _xgboost: https://github.com/dmlc/xgboost
|
||||
|
||||
.. _link: https://ailab.criteo.com/download-criteo-1tb-click-logs-dataset/
|
||||
+420
@@ -0,0 +1,420 @@
|
||||
.. role:: raw-html(raw)
|
||||
:format: html
|
||||
|
||||
LightGBM FAQ
|
||||
############
|
||||
|
||||
.. contents:: LightGBM Frequently Asked Questions
|
||||
:depth: 1
|
||||
:local:
|
||||
:backlinks: none
|
||||
|
||||
------
|
||||
|
||||
Please post questions, feature requests, and bug reports at https://github.com/lightgbm-org/LightGBM/issues.
|
||||
|
||||
This project is mostly maintained by volunteers, so please be patient.
|
||||
If your request is time-sensitive or more than a month goes by without a response, please tag the maintainers below for help.
|
||||
|
||||
- `@guolinke <https://github.com/guolinke>`__ **Guolin Ke**
|
||||
- `@shiyu1994 <https://github.com/shiyu1994>`__ **Yu Shi**
|
||||
- `@jameslamb <https://github.com/jameslamb>`__ **James Lamb**
|
||||
- `@jmoralez <https://github.com/jmoralez>`__ **José Morales**
|
||||
- `@borchero <https://github.com/borchero>`__ **Oliver Borchert**
|
||||
- `@mayer79 <https://github.com/mayer79>`__ **Michael Mayer**
|
||||
|
||||
--------------
|
||||
|
||||
General LightGBM Questions
|
||||
==========================
|
||||
|
||||
.. contents::
|
||||
:local:
|
||||
:backlinks: none
|
||||
|
||||
1. Where do I find more details about LightGBM parameters?
|
||||
----------------------------------------------------------
|
||||
|
||||
Take a look at `Parameters <./Parameters.rst>`__.
|
||||
|
||||
2. On datasets with millions of features, training does not start (or starts after a very long time).
|
||||
-----------------------------------------------------------------------------------------------------
|
||||
|
||||
Use a smaller value for ``bin_construct_sample_cnt`` and a larger value for ``min_data``.
|
||||
|
||||
3. When running LightGBM on a large dataset, my computer runs out of RAM.
|
||||
-------------------------------------------------------------------------
|
||||
|
||||
**Multiple Solutions**: set the ``histogram_pool_size`` parameter to the MB you want to use for LightGBM (histogram\_pool\_size + dataset size = approximately RAM used),
|
||||
lower ``num_leaves`` or lower ``max_bin`` (see `lightgbm-org/LightGBM#562 <https://github.com/lightgbm-org/LightGBM/issues/562>`__).
|
||||
|
||||
4. I am using Windows. Should I use Visual Studio or MinGW for compiling LightGBM?
|
||||
----------------------------------------------------------------------------------
|
||||
|
||||
Visual Studio `performs best for LightGBM <https://github.com/lightgbm-org/LightGBM/issues/542>`__.
|
||||
|
||||
5. When using LightGBM GPU, I cannot reproduce results over several runs.
|
||||
-------------------------------------------------------------------------
|
||||
|
||||
This is normal and expected behaviour, but you may try to use ``gpu_use_dp = true`` for reproducibility
|
||||
(see `lightgbm-org/LightGBM#560 <https://github.com/lightgbm-org/LightGBM/pull/560#issuecomment-304561654>`__).
|
||||
You may also use the CPU version.
|
||||
|
||||
6. Bagging is not reproducible when changing the number of threads.
|
||||
-------------------------------------------------------------------
|
||||
|
||||
:raw-html:`<strike>`
|
||||
LightGBM bagging is multithreaded, so its output depends on the number of threads used.
|
||||
There is `no workaround currently <https://github.com/lightgbm-org/LightGBM/issues/632>`__.
|
||||
:raw-html:`</strike>`
|
||||
|
||||
Starting from `#2804 <https://github.com/lightgbm-org/LightGBM/pull/2804>`__ bagging result doesn't depend on the number of threads.
|
||||
So this issue should be solved in the latest version.
|
||||
|
||||
7. I tried to use Random Forest mode, and LightGBM crashes!
|
||||
-----------------------------------------------------------
|
||||
|
||||
This is expected behaviour for arbitrary parameters. To enable Random Forest,
|
||||
you must use ``bagging_fraction`` and ``feature_fraction`` different from 1, along with a ``bagging_freq``.
|
||||
`This thread <https://github.com/lightgbm-org/LightGBM/issues/691>`__ includes an example.
|
||||
|
||||
8. CPU usage is low (like 10%) in Windows when using LightGBM on very large datasets with many-core systems.
|
||||
------------------------------------------------------------------------------------------------------------
|
||||
|
||||
Please use `Visual Studio <https://visualstudio.microsoft.com/downloads/>`__
|
||||
as it may be `10x faster than MinGW <https://github.com/lightgbm-org/LightGBM/issues/749>`__ especially for very large trees.
|
||||
|
||||
9. When I'm trying to specify a categorical column with the ``categorical_feature`` parameter, I get the following sequence of warnings, but there are no negative values in the column.
|
||||
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
[LightGBM] [Warning] Met negative value in categorical features, will convert it to NaN
|
||||
[LightGBM] [Warning] There are no meaningful features, as all feature values are constant.
|
||||
|
||||
The column you're trying to pass via ``categorical_feature`` likely contains very large values.
|
||||
Categorical features in LightGBM are limited by int32 range,
|
||||
so you cannot pass values that are greater than ``Int32.MaxValue`` (2147483647) as categorical features (see `lightgbm-org/LightGBM#1359 <https://github.com/lightgbm-org/LightGBM/issues/1359>`__).
|
||||
You should convert them to integers ranging from zero to the number of categories first.
|
||||
|
||||
10. LightGBM crashes randomly with the error like: ``Initializing libiomp5.dylib, but found libomp.dylib already initialized.``
|
||||
-------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized.
|
||||
OMP: Hint: This means that multiple copies of the OpenMP runtime have been linked into the program. That is dangerous, since it can degrade performance or cause incorrect results. The best thing to do is to ensure that only a single OpenMP runtime is linked into the process, e.g. by avoiding static linking of the OpenMP runtime in any library. As an unsafe, unsupported, undocumented workaround you can set the environment variable KMP_DUPLICATE_LIB_OK=TRUE to allow the program to continue to execute, but that may cause crashes or silently produce incorrect results. For more information, please see http://www.intel.com/software/products/support/.
|
||||
|
||||
**Possible Cause**: This error means that you have multiple OpenMP libraries installed on your machine and they conflict with each other.
|
||||
(File extensions in the error message may differ depending on the operating system).
|
||||
|
||||
If you are using Python distributed by Conda, then it is highly likely that the error is caused by the ``numpy`` package from Conda which includes the ``mkl`` package which in turn conflicts with the system-wide library.
|
||||
In this case you can update the ``numpy`` package in Conda or replace the Conda's OpenMP library instance with system-wide one by creating a symlink to it in Conda environment folder ``$CONDA_PREFIX/lib``.
|
||||
|
||||
**Solution**: Assuming you are using macOS with Homebrew, the command which overwrites OpenMP library files in the current active Conda environment with symlinks to the system-wide library ones installed by Homebrew:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib
|
||||
|
||||
The described above fix worked fine before the release of OpenMP 8.0.0 version.
|
||||
Starting from 8.0.0 version, Homebrew formula for OpenMP includes ``-DLIBOMP_INSTALL_ALIASES=OFF`` option which leads to that the fix doesn't work anymore.
|
||||
However, you can create symlinks to library aliases manually:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
for LIBOMP_ALIAS in libgomp.dylib libiomp5.dylib libomp.dylib; do sudo ln -sf "$(brew --cellar libomp)"/*/lib/libomp.dylib $CONDA_PREFIX/lib/$LIBOMP_ALIAS; done
|
||||
|
||||
Another workaround would be removing MKL optimizations from Conda's packages completely:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
conda install nomkl
|
||||
|
||||
If this is not your case, then you should find conflicting OpenMP library installations on your own and leave only one of them.
|
||||
|
||||
11. LightGBM hangs when multithreading (OpenMP) and using forking in Linux at the same time.
|
||||
--------------------------------------------------------------------------------------------
|
||||
|
||||
Use ``nthreads=1`` to disable multithreading of LightGBM. There is a bug with OpenMP which hangs forked sessions
|
||||
with multithreading activated. A more expensive solution is to use new processes instead of using fork, however,
|
||||
keep in mind it is creating new processes where you have to copy memory and load libraries (example: if you want to
|
||||
fork 16 times your current process, then you will require to make 16 copies of your dataset in memory)
|
||||
(see `lightgbm-org/LightGBM#1789 <https://github.com/lightgbm-org/LightGBM/issues/1789#issuecomment-433713383>`__).
|
||||
|
||||
An alternative, if multithreading is really necessary inside the forked sessions, would be to compile LightGBM with
|
||||
Intel toolchain. Intel compilers are unaffected by this bug.
|
||||
|
||||
For C/C++ users, any OpenMP feature cannot be used before the fork happens. If an OpenMP feature is used before the
|
||||
fork happens (example: using OpenMP for forking), OpenMP will hang inside the forked sessions. Use new processes instead
|
||||
and copy memory as required by creating new processes instead of forking (or, use Intel compilers).
|
||||
|
||||
Cloud platform container services may cause LightGBM to hang, if they use Linux fork to run multiple containers on a
|
||||
single instance. For example, LightGBM hangs in AWS Batch array jobs, which `use the ECS agent
|
||||
<https://aws.amazon.com/batch/faqs>`__ to manage multiple running jobs. Setting ``nthreads=1`` mitigates the issue.
|
||||
|
||||
12. Why is early stopping not enabled by default in LightGBM?
|
||||
-------------------------------------------------------------
|
||||
|
||||
Early stopping involves choosing a validation set, a special type of holdout which is used to evaluate the current state of the model after each iteration to see if training can stop.
|
||||
|
||||
In ``LightGBM``, `we have decided to require that users specify this set directly <./Parameters.rst#valid>`_. Many options exist for splitting training data into training, test, and validation sets.
|
||||
|
||||
The appropriate splitting strategy depends on the task and domain of the data, information that a modeler has but which ``LightGBM`` as a general-purpose tool does not.
|
||||
|
||||
13. Does LightGBM support direct loading data from zero-based or one-based LibSVM format file?
|
||||
----------------------------------------------------------------------------------------------
|
||||
|
||||
LightGBM supports loading data from zero-based LibSVM format file directly.
|
||||
|
||||
14. Why CMake cannot find the compiler when compiling LightGBM with MinGW?
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
CMake Error: CMAKE_C_COMPILER not set, after EnableLanguage
|
||||
CMake Error: CMAKE_CXX_COMPILER not set, after EnableLanguage
|
||||
|
||||
This is a known issue of CMake when using MinGW. The easiest solution is to run again your ``cmake`` command to bypass the one time stopper from CMake. Or you can upgrade your version of CMake to at least version 3.17.0.
|
||||
|
||||
See `lightgbm-org/LightGBM#3060 <https://github.com/lightgbm-org/LightGBM/issues/3060#issuecomment-626338538>`__ for more details.
|
||||
|
||||
15. Where can I find LightGBM's logo to use it in my presentation?
|
||||
------------------------------------------------------------------
|
||||
|
||||
You can find LightGBM's logo in different file formats and resolutions `here <https://github.com/lightgbm-org/LightGBM/tree/main/docs/logo>`__.
|
||||
|
||||
16. LightGBM crashes randomly or operating system hangs during or after running LightGBM.
|
||||
-----------------------------------------------------------------------------------------
|
||||
|
||||
**Possible Cause**: This behavior may indicate that you have multiple OpenMP libraries installed on your machine and they conflict with each other, similarly to the ``FAQ #10``.
|
||||
|
||||
If you are using any Python-package that depends on ``threadpoolctl``, you also may see the following warning in your logs in this case:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
/root/miniconda/envs/test-env/lib/python3.8/site-packages/threadpoolctl.py:546: RuntimeWarning:
|
||||
Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at
|
||||
the same time. Both libraries are known to be incompatible and this
|
||||
can cause random crashes or deadlocks on Linux when loaded in the
|
||||
same Python program.
|
||||
Using threadpoolctl may cause crashes or deadlocks. For more
|
||||
information and possible workarounds, please see
|
||||
https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md
|
||||
|
||||
Detailed description of conflicts between multiple OpenMP instances is provided in the `following document <https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md>`__.
|
||||
|
||||
**Solution**: Assuming you are using LightGBM Python-package and conda as a package manager, we strongly recommend using ``conda-forge`` channel as the only source of all your Python package installations because it contains built-in patches to workaround OpenMP conflicts. Some other workarounds are listed `here <https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md>`__ under the "Workarounds for Intel OpenMP and LLVM OpenMP case" section.
|
||||
|
||||
If this is not your case, then you should find conflicting OpenMP library installations on your own and leave only one of them.
|
||||
|
||||
17. Loading LightGBM fails like: ``cannot allocate memory in static TLS block``
|
||||
-------------------------------------------------------------------------------
|
||||
|
||||
When loading LightGBM, you may encounter errors like the following.
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
lib/libgomp.so.1: cannot allocate memory in static TLS block
|
||||
|
||||
This most commonly happens on aarch64 Linux systems.
|
||||
|
||||
``gcc``'s OpenMP library (``libgomp.so``) tries to allocate a small amount of static thread-local storage ("TLS")
|
||||
when it's dynamically loaded.
|
||||
|
||||
That error can happen when the loader isn't able to find a large enough block of memory.
|
||||
|
||||
On aarch64 Linux, processes and loaded libraries share the same pool of static TLS,
|
||||
which makes such failures more likely. See these discussions:
|
||||
|
||||
* https://bugzilla.redhat.com/show_bug.cgi?id=1722181#c6
|
||||
* https://gcc.gcc.gnu.narkive.com/vOXMQqLA/failure-to-dlopen-libgomp-due-to-static-tls-data
|
||||
|
||||
If you are experiencing this issue when using the ``lightgbm`` Python-package, try upgrading
|
||||
to at least ``v4.6.0``.
|
||||
|
||||
For older versions of the Python-package, or for other LightGBM APIs, this issue can
|
||||
often be avoided by loading ``libgomp.so.1``. That can be done directly by setting environment
|
||||
variable ``LD_PRELOAD``, like this:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
export LD_PRELOAD=/root/miniconda3/envs/test-env/lib/libgomp.so.1
|
||||
|
||||
It can also be done indirectly by changing the order that other libraries are loaded
|
||||
into processes, which varies by programming language and application type.
|
||||
|
||||
For more details, see these discussions:
|
||||
|
||||
* https://github.com/lightgbm-org/LightGBM/pull/6654#issuecomment-2352014275
|
||||
* https://github.com/lightgbm-org/LightGBM/issues/6509
|
||||
* https://maskray.me/blog/2021-02-14-all-about-thread-local-storage
|
||||
* https://bugzilla.redhat.com/show_bug.cgi?id=1722181#c6
|
||||
|
||||
------
|
||||
|
||||
R-package
|
||||
=========
|
||||
|
||||
.. contents::
|
||||
:local:
|
||||
:backlinks: none
|
||||
|
||||
1. Any training command using LightGBM does not work after an error occurred during the training of a previous LightGBM model.
|
||||
------------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
In older versions of the R-package (prior to ``v3.3.0``), this could happen occasionally and the solution was to run ``lgb.unloader(wipe = TRUE)`` to remove all LightGBM-related objects. Some conversation about this could be found in `lightgbm-org/LightGBM#698 <https://github.com/lightgbm-org/LightGBM/issues/698>`__.
|
||||
|
||||
That is no longer necessary as of ``v3.3.0``, and function ``lgb.unloader()`` has since been removed from the R-package.
|
||||
|
||||
2. I used ``setinfo()``, tried to print my ``lgb.Dataset``, and now the R console froze!
|
||||
----------------------------------------------------------------------------------------
|
||||
|
||||
As of at least LightGBM v3.3.0, this issue has been resolved and printing a ``Dataset`` object does not cause the console to freeze.
|
||||
|
||||
In older versions, avoid printing the ``Dataset`` after calling ``setinfo()``.
|
||||
|
||||
As of LightGBM v4.0.0, ``setinfo()`` has been replaced by a new method, ``set_field()``.
|
||||
|
||||
3. ``error in data.table::data.table()...argument 2 is NULL``.
|
||||
--------------------------------------------------------------
|
||||
|
||||
If you are experiencing this error when running ``lightgbm``, you may be facing the same issue reported in `#2715 <https://github.com/lightgbm-org/LightGBM/issues/2715>`_ and later in `#2989 <https://github.com/lightgbm-org/LightGBM/pull/2989#issuecomment-614374151>`_. We have seen that in some situations, using ``data.table`` 1.11.x results in this error. To get around this, you can upgrade your version of ``data.table`` to at least version 1.12.0.
|
||||
|
||||
4. ``package/dependency ‘Matrix’ is not available ...``
|
||||
-------------------------------------------------------
|
||||
|
||||
In April 2024, ``Matrix==1.7-0`` was published to CRAN.
|
||||
That version had a floor of ``R (>=4.4.0)``.
|
||||
``{Matrix}`` is a hard runtime dependency of ``{lightgbm}``, so on any version of R older than ``4.4.0``, running ``install.packages("lightgbm")`` results in something like the following.
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
package ‘Matrix’ is not available for this version of R
|
||||
|
||||
To fix that without upgrading to R 4.4.0 or greater, manually install an older version of ``{Matrix}``.
|
||||
|
||||
.. code-block:: R
|
||||
|
||||
install.packages('https://cran.r-project.org/src/contrib/Archive/Matrix/Matrix_1.6-5.tar.gz', repos = NULL)
|
||||
|
||||
------
|
||||
|
||||
Python-package
|
||||
==============
|
||||
|
||||
.. contents::
|
||||
:local:
|
||||
:backlinks: none
|
||||
|
||||
1. ``Error: setup script specifies an absolute path`` when installing from GitHub using ``python setup.py install``.
|
||||
--------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
.. note::
|
||||
As of v4.0.0, ``lightgbm`` does not support directly invoking ``setup.py``.
|
||||
This answer refers only to versions of ``lightgbm`` prior to v4.0.0.
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
error: Error: setup script specifies an absolute path:
|
||||
/Users/Microsoft/LightGBM/python-package/lightgbm/../../lib_lightgbm.so
|
||||
setup() arguments must *always* be /-separated paths relative to the setup.py directory, *never* absolute paths.
|
||||
|
||||
This error should be solved in latest version.
|
||||
If you still meet this error, try to remove ``lightgbm.egg-info`` folder in your Python-package and reinstall,
|
||||
or check `this thread on stackoverflow <https://stackoverflow.com/questions/18085571/pip-install-error-setup-script-specifies-an-absolute-path>`__.
|
||||
|
||||
2. Error messages: ``Cannot ... before construct dataset``.
|
||||
-----------------------------------------------------------
|
||||
|
||||
I see error messages like...
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
Cannot get/set label/weight/init_score/group/num_data/num_feature before construct dataset
|
||||
|
||||
but I've already constructed a dataset by some code like:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
train = lightgbm.Dataset(X_train, y_train)
|
||||
|
||||
or error messages like
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
Cannot set predictor/reference/categorical feature after freed raw data, set free_raw_data=False when construct Dataset to avoid this.
|
||||
|
||||
**Solution**: Because LightGBM constructs bin mappers to build trees, and train and valid Datasets within one Booster share the same bin mappers,
|
||||
categorical features and feature names etc., the Dataset objects are constructed when constructing a Booster.
|
||||
If you set ``free_raw_data=True`` (default), the raw data (with Python data struct) will be freed.
|
||||
So, if you want to:
|
||||
|
||||
- get label (or weight/init\_score/group/data) before constructing a dataset, it's same as get ``self.label``;
|
||||
|
||||
- set label (or weight/init\_score/group) before constructing a dataset, it's same as ``self.label=some_label_array``;
|
||||
|
||||
- get num\_data (or num\_feature) before constructing a dataset, you can get data with ``self.data``.
|
||||
Then, if your data is ``numpy.ndarray``, use some code like ``self.data.shape``. But do not do this after subsetting the Dataset, because you'll get always ``None``;
|
||||
|
||||
- set predictor (or reference/categorical feature) after constructing a dataset,
|
||||
you should set ``free_raw_data=False`` or init a Dataset object with the same raw data.
|
||||
|
||||
3. I encounter segmentation faults (segfaults) randomly after installing LightGBM from PyPI using ``pip install lightgbm``.
|
||||
---------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
We are doing our best to provide universal wheels which have high running speed and are compatible with any hardware, OS, compiler, etc. at the same time.
|
||||
However, sometimes it's just impossible to guarantee the possibility of usage of LightGBM in any specific environment (see `lightgbm-org/LightGBM#1743 <https://github.com/lightgbm-org/LightGBM/issues/1743>`__).
|
||||
|
||||
Therefore, the first thing you should try in case of segfaults is **compiling from the source** using ``pip install --no-binary lightgbm lightgbm``.
|
||||
For the OS-specific prerequisites see https://github.com/lightgbm-org/LightGBM/blob/main/python-package/README.rst.
|
||||
|
||||
Also, feel free to post a new issue in our GitHub repository. We always look at each case individually and try to find a root cause.
|
||||
|
||||
4. I would like to install LightGBM from conda. What channel should I choose?
|
||||
-----------------------------------------------------------------------------
|
||||
|
||||
We strongly recommend installation from the ``conda-forge`` channel and not from the ``default`` one.
|
||||
|
||||
For some specific examples, see `this comment <https://github.com/lightgbm-org/LightGBM/issues/4948#issuecomment-1013766397>`__.
|
||||
|
||||
In addition, as of ``lightgbm==4.4.0``, the ``conda-forge`` package automatically supports CUDA-based GPU acceleration.
|
||||
|
||||
5. How do I subclass ``scikit-learn`` estimators?
|
||||
-------------------------------------------------
|
||||
|
||||
For ``lightgbm <= 4.5.0``, copy all of the constructor arguments from the corresponding
|
||||
``lightgbm`` class into the constructor of your custom estimator.
|
||||
|
||||
For later versions, just ensure that the constructor of your custom estimator calls ``super().__init__()``.
|
||||
|
||||
Consider the example below, which implements a regressor that allows creation of truncated predictions.
|
||||
This pattern will work with ``lightgbm > 4.5.0``.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import numpy as np
|
||||
from lightgbm import LGBMRegressor
|
||||
from sklearn.datasets import make_regression
|
||||
|
||||
class TruncatedRegressor(LGBMRegressor):
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def predict(self, X, max_score: float = np.inf):
|
||||
preds = super().predict(X)
|
||||
np.clip(preds, a_min=None, a_max=max_score, out=preds)
|
||||
return preds
|
||||
|
||||
X, y = make_regression(n_samples=1_000, n_features=4)
|
||||
|
||||
reg_trunc = TruncatedRegressor().fit(X, y)
|
||||
|
||||
preds = reg_trunc.predict(X)
|
||||
print(f"mean: {preds.mean():.2f}, max: {preds.max():.2f}")
|
||||
# mean: -6.81, max: 345.10
|
||||
|
||||
preds_trunc = reg_trunc.predict(X, max_score=preds.mean())
|
||||
print(f"mean: {preds_trunc.mean():.2f}, max: {preds_trunc.max():.2f}")
|
||||
# mean: -56.50, max: -6.81
|
||||
@@ -0,0 +1,298 @@
|
||||
Features
|
||||
========
|
||||
|
||||
This is a conceptual overview of how LightGBM works\ `[1] <#references>`__. We assume familiarity with decision tree boosting algorithms to focus instead on aspects of LightGBM that may differ from other boosting packages. For detailed algorithms, please refer to the citations or source code.
|
||||
|
||||
Optimization in Speed and Memory Usage
|
||||
--------------------------------------
|
||||
|
||||
Many boosting tools use pre-sort-based algorithms\ `[2, 3] <#references>`__ (e.g. default algorithm in xgboost) for decision tree learning. It is a simple solution, but not easy to optimize.
|
||||
|
||||
LightGBM uses histogram-based algorithms\ `[4, 5, 6] <#references>`__, which bucket continuous feature (attribute) values into discrete bins. This speeds up training and reduces memory usage. Advantages of histogram-based algorithms include the following:
|
||||
|
||||
- **Reduced cost of calculating the gain for each split**
|
||||
|
||||
- Pre-sort-based algorithms have time complexity ``O(#data)``
|
||||
|
||||
- Computing the histogram has time complexity ``O(#data)``, but this involves only a fast sum-up operation. Once the histogram is constructed, a histogram-based algorithm has time complexity ``O(#bins)``, and ``#bins`` is far smaller than ``#data``.
|
||||
|
||||
- **Use histogram subtraction for further speedup**
|
||||
|
||||
- To get one leaf's histograms in a binary tree, use the histogram subtraction of its parent and its neighbor
|
||||
|
||||
- So it needs to construct histograms for only one leaf (with smaller ``#data`` than its neighbor). It then can get histograms of its neighbor by histogram subtraction with small cost (``O(#bins)``)
|
||||
|
||||
- **Reduce memory usage**
|
||||
|
||||
- Replaces continuous values with discrete bins. If ``#bins`` is small, can use small data type, e.g. uint8\_t, to store training data
|
||||
|
||||
- No need to store additional information for pre-sorting feature values
|
||||
|
||||
- **Reduce communication cost for distributed learning**
|
||||
|
||||
Sparse Optimization
|
||||
-------------------
|
||||
|
||||
- Need only ``O(2 * #non_zero_data)`` to construct histogram for sparse features
|
||||
|
||||
Optimization in Accuracy
|
||||
------------------------
|
||||
|
||||
Leaf-wise (Best-first) Tree Growth
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Most decision tree learning algorithms grow trees by level (depth)-wise, like the following image:
|
||||
|
||||
.. image:: ./_static/images/level-wise.png
|
||||
:align: center
|
||||
:alt: A diagram depicting level wise tree growth in which the best possible node is split one level down. The strategy results in a symmetric tree, where every node in a level has child nodes resulting in an additional layer of depth.
|
||||
|
||||
LightGBM grows trees leaf-wise (best-first)\ `[7] <#references>`__. It will choose the leaf with max delta loss to grow.
|
||||
Holding ``#leaf`` fixed, leaf-wise algorithms tend to achieve lower loss than level-wise algorithms.
|
||||
|
||||
Leaf-wise may cause over-fitting when ``#data`` is small, so LightGBM includes the ``max_depth`` parameter to limit tree depth. However, trees still grow leaf-wise even when ``max_depth`` is specified.
|
||||
|
||||
.. image:: ./_static/images/leaf-wise.png
|
||||
:align: center
|
||||
:alt: A diagram depicting leaf wise tree growth in which only the node with the highest loss change is split and not bother with the rest of the nodes in the same level. This results in an asymmetrical tree where subsequent splitting is happening only on one side of the tree.
|
||||
|
||||
Optimal Split for Categorical Features
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
It is common to represent categorical features with one-hot encoding, but this approach is suboptimal for tree learners. Particularly for high-cardinality categorical features, a tree built on one-hot features tends to be unbalanced and needs to grow very deep to achieve good accuracy.
|
||||
|
||||
Instead of one-hot encoding, the optimal solution is to split on a categorical feature by partitioning its categories into 2 subsets. If the feature has ``k`` categories, there are ``2^(k-1) - 1`` possible partitions.
|
||||
But there is an efficient solution for regression trees\ `[8] <#references>`__. It needs about ``O(k * log(k))`` to find the optimal partition.
|
||||
|
||||
The basic idea is to sort the categories according to the training objective at each split.
|
||||
More specifically, LightGBM sorts the histogram (for a categorical feature) according to its accumulated values (``sum_gradient / sum_hessian``) and then finds the best split on the sorted histogram.
|
||||
|
||||
Optimization in Network Communication
|
||||
-------------------------------------
|
||||
|
||||
It only needs to use some collective communication algorithms, like "All reduce", "All gather" and "Reduce scatter", in distributed learning of LightGBM.
|
||||
LightGBM implements state-of-the-art algorithms\ `[9] <#references>`__.
|
||||
These collective communication algorithms can provide much better performance than point-to-point communication.
|
||||
|
||||
.. _Optimization in Parallel Learning:
|
||||
|
||||
Optimization in Distributed Learning
|
||||
------------------------------------
|
||||
|
||||
LightGBM provides the following distributed learning algorithms.
|
||||
|
||||
Feature Parallel
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Traditional Algorithm
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Feature parallel aims to parallelize the "Find Best Split" in the decision tree. The procedure of traditional feature parallel is:
|
||||
|
||||
1. Partition data vertically (different machines have different feature set).
|
||||
|
||||
2. Workers find local best split point {feature, threshold} on local feature set.
|
||||
|
||||
3. Communicate local best splits with each other and get the best one.
|
||||
|
||||
4. Worker with best split to perform split, then send the split result of data to other workers.
|
||||
|
||||
5. Other workers split data according to received data.
|
||||
|
||||
The shortcomings of traditional feature parallel:
|
||||
|
||||
- Has computation overhead, since it cannot speed up "split", whose time complexity is ``O(#data)``.
|
||||
Thus, feature parallel cannot speed up well when ``#data`` is large.
|
||||
|
||||
- Need communication of split result, which costs about ``O(#data / 8)`` (one bit for one data).
|
||||
|
||||
Feature Parallel in LightGBM
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Since feature parallel cannot speed up well when ``#data`` is large, we make a little change: instead of partitioning data vertically, every worker holds the full data.
|
||||
Thus, LightGBM doesn't need to communicate for split result of data since every worker knows how to split data.
|
||||
And ``#data`` won't be larger, so it is reasonable to hold the full data in every machine.
|
||||
|
||||
The procedure of feature parallel in LightGBM:
|
||||
|
||||
1. Workers find local best split point {feature, threshold} on local feature set.
|
||||
|
||||
2. Communicate local best splits with each other and get the best one.
|
||||
|
||||
3. Perform best split.
|
||||
|
||||
However, this feature parallel algorithm still suffers from computation overhead for "split" when ``#data`` is large.
|
||||
So it will be better to use data parallel when ``#data`` is large.
|
||||
|
||||
Data Parallel
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
Traditional Algorithm
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Data parallel aims to parallelize the whole decision learning. The procedure of data parallel is:
|
||||
|
||||
1. Partition data horizontally.
|
||||
|
||||
2. Workers use local data to construct local histograms.
|
||||
|
||||
3. Merge global histograms from all local histograms.
|
||||
|
||||
4. Find best split from merged global histograms, then perform splits.
|
||||
|
||||
The shortcomings of traditional data parallel:
|
||||
|
||||
- High communication cost.
|
||||
If using point-to-point communication algorithm, communication cost for one machine is about ``O(#machine * #feature * #bin)``.
|
||||
If using collective communication algorithm (e.g. "All Reduce"), communication cost is about ``O(2 * #feature * #bin)`` (check cost of "All Reduce" in chapter 4.5 at `[9] <#references>`__).
|
||||
|
||||
Data Parallel in LightGBM
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
We reduce communication cost of data parallel in LightGBM:
|
||||
|
||||
1. Instead of "Merge global histograms from all local histograms", LightGBM uses "Reduce Scatter" to merge histograms of different (non-overlapping) features for different workers.
|
||||
Then workers find the local best split on local merged histograms and sync up the global best split.
|
||||
|
||||
2. As aforementioned, LightGBM uses histogram subtraction to speed up training.
|
||||
Based on this, we can communicate histograms only for one leaf, and get its neighbor's histograms by subtraction as well.
|
||||
|
||||
All things considered, data parallel in LightGBM has time complexity ``O(0.5 * #feature * #bin)``.
|
||||
|
||||
Voting Parallel
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Voting parallel further reduces the communication cost in `Data Parallel <#data-parallel>`__ to constant cost.
|
||||
It uses two-stage voting to reduce the communication cost of feature histograms\ `[10] <#references>`__.
|
||||
|
||||
GPU Support
|
||||
-----------
|
||||
|
||||
Thanks `@huanzhang12 <https://github.com/huanzhang12>`__ for contributing this feature. Please read `[11] <#references>`__ to get more details.
|
||||
|
||||
- `GPU Installation <./Installation-Guide.rst#build-gpu-version>`__
|
||||
|
||||
- `GPU Tutorial <./GPU-Tutorial.rst>`__
|
||||
|
||||
Applications and Metrics
|
||||
------------------------
|
||||
|
||||
LightGBM supports the following applications:
|
||||
|
||||
- regression, the objective function is L2 loss
|
||||
|
||||
- binary classification, the objective function is logloss
|
||||
|
||||
- multi classification
|
||||
|
||||
- cross-entropy, the objective function is logloss and supports training on non-binary labels
|
||||
|
||||
- LambdaRank, the objective function is LambdaRank with NDCG
|
||||
|
||||
LightGBM supports the following metrics:
|
||||
|
||||
- L1 loss
|
||||
|
||||
- L2 loss
|
||||
|
||||
- Log loss
|
||||
|
||||
- Classification error rate
|
||||
|
||||
- AUC
|
||||
|
||||
- NDCG
|
||||
|
||||
- MAP
|
||||
|
||||
- Multi-class log loss
|
||||
|
||||
- Multi-class error rate
|
||||
|
||||
- AUC-mu ``(new in v3.0.0)``
|
||||
|
||||
- Average precision ``(new in v3.1.0)``
|
||||
|
||||
- Fair
|
||||
|
||||
- Huber
|
||||
|
||||
- Poisson
|
||||
|
||||
- Quantile
|
||||
|
||||
- MAPE
|
||||
|
||||
- Kullback-Leibler
|
||||
|
||||
- Gamma
|
||||
|
||||
- Tweedie
|
||||
|
||||
For more details, please refer to `Parameters <./Parameters.rst#metric-parameters>`__.
|
||||
|
||||
Other Features
|
||||
--------------
|
||||
|
||||
- Limit ``max_depth`` of tree while grows tree leaf-wise
|
||||
|
||||
- `DART <https://arxiv.org/abs/1505.01866>`__
|
||||
|
||||
- L1/L2 regularization
|
||||
|
||||
- Bagging
|
||||
|
||||
- Column (feature) sub-sample
|
||||
|
||||
- Continued train with input GBDT model
|
||||
|
||||
- Continued train with the input score file
|
||||
|
||||
- Weighted training
|
||||
|
||||
- Validation metric output during training
|
||||
|
||||
- Multiple validation data
|
||||
|
||||
- Multiple metrics
|
||||
|
||||
- Early stopping (both training and prediction)
|
||||
|
||||
- Prediction for leaf index
|
||||
|
||||
For more details, please refer to `Parameters <./Parameters.rst>`__.
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
[1] Guolin Ke, Qi Meng, Thomas Finley, Taifeng Wang, Wei Chen, Weidong Ma, Qiwei Ye, Tie-Yan Liu. "`LightGBM\: A Highly Efficient Gradient Boosting Decision Tree`_." Advances in Neural Information Processing Systems 30 (NIPS 2017), pp. 3149-3157.
|
||||
|
||||
[2] Mehta, Manish, Rakesh Agrawal, and Jorma Rissanen. "SLIQ: A fast scalable classifier for data mining." International Conference on Extending Database Technology. Springer Berlin Heidelberg, 1996.
|
||||
|
||||
[3] Shafer, John, Rakesh Agrawal, and Manish Mehta. "SPRINT: A scalable parallel classifier for data mining." Proc. 1996 Int. Conf. Very Large Data Bases. 1996.
|
||||
|
||||
[4] Ranka, Sanjay, and V. Singh. "CLOUDS: A decision tree classifier for large datasets." Proceedings of the 4th Knowledge Discovery and Data Mining Conference. 1998.
|
||||
|
||||
[5] Machado, F. P. "Communication and memory efficient parallel decision tree construction." (2003).
|
||||
|
||||
[6] Li, Ping, Qiang Wu, and Christopher J. Burges. "Mcrank: Learning to rank using multiple classification and gradient boosting." Advances in Neural Information Processing Systems 20 (NIPS 2007).
|
||||
|
||||
[7] Shi, Haijian. "Best-first decision tree learning." Diss. The University of Waikato, 2007.
|
||||
|
||||
[8] Walter D. Fisher. "`On Grouping for Maximum Homogeneity`_." Journal of the American Statistical Association. Vol. 53, No. 284 (Dec., 1958), pp. 789-798.
|
||||
|
||||
[9] Thakur, Rajeev, Rolf Rabenseifner, and William Gropp. "`Optimization of collective communication operations in MPICH`_." International Journal of High Performance Computing Applications 19.1 (2005), pp. 49-66.
|
||||
|
||||
[10] Qi Meng, Guolin Ke, Taifeng Wang, Wei Chen, Qiwei Ye, Zhi-Ming Ma, Tie-Yan Liu. "`A Communication-Efficient Parallel Algorithm for Decision Tree`_." Advances in Neural Information Processing Systems 29 (NIPS 2016), pp. 1279-1287.
|
||||
|
||||
[11] Huan Zhang, Si Si and Cho-Jui Hsieh. "`GPU Acceleration for Large-scale Tree Boosting`_." SysML Conference, 2018.
|
||||
|
||||
.. _LightGBM\: A Highly Efficient Gradient Boosting Decision Tree: https://proceedings.neurips.cc/paper/2017/hash/6449f44a102fde848669bdd9eb6b76fa-Abstract.html
|
||||
|
||||
.. _On Grouping for Maximum Homogeneity: https://www.jstor.org/stable/2281952
|
||||
|
||||
.. _Optimization of collective communication operations in MPICH: https://www.mpich.org/2012/10/24/optimization-of-collective-communication-operations-in-mpich/
|
||||
|
||||
.. _A Communication-Efficient Parallel Algorithm for Decision Tree: https://proceedings.neurips.cc/paper/2016/hash/10a5ab2db37feedfdeaab192ead4ac0e-Abstract.html
|
||||
|
||||
.. _GPU Acceleration for Large-scale Tree Boosting: https://arxiv.org/abs/1706.08359
|
||||
@@ -0,0 +1,211 @@
|
||||
GPU Tuning Guide and Performance Comparison
|
||||
===========================================
|
||||
|
||||
How It Works?
|
||||
-------------
|
||||
|
||||
In LightGBM, the main computation cost during training is building the feature histograms. We use an efficient algorithm on GPU to accelerate this process.
|
||||
The implementation is highly modular, and works for all learning tasks (classification, ranking, regression, etc). GPU acceleration also works in distributed learning settings.
|
||||
GPU algorithm implementation is based on OpenCL and can work with a wide range of GPUs.
|
||||
|
||||
Supported Hardware
|
||||
------------------
|
||||
|
||||
We target AMD Graphics Core Next (GCN) architecture and NVIDIA Maxwell and Pascal architectures.
|
||||
Most AMD GPUs released after 2012 and NVIDIA GPUs released after 2014 should be supported. We have tested the GPU implementation on the following GPUs:
|
||||
|
||||
- AMD RX 480 with AMDGPU-pro driver 16.60 on Ubuntu 16.10
|
||||
|
||||
- AMD R9 280X (aka Radeon HD 7970) with fglrx driver 15.302.2301 on Ubuntu 16.10
|
||||
|
||||
- NVIDIA GTX 1080 with driver 375.39 and CUDA 8.0 on Ubuntu 16.10
|
||||
|
||||
- NVIDIA Titan X (Pascal) with driver 367.48 and CUDA 8.0 on Ubuntu 16.04
|
||||
|
||||
- NVIDIA Tesla M40 with driver 375.39 and CUDA 7.5 on Ubuntu 16.04
|
||||
|
||||
Using the following hardware is discouraged:
|
||||
|
||||
- NVIDIA Kepler (K80, K40, K20, most GeForce GTX 700 series GPUs) or earlier NVIDIA GPUs. They don't support hardware atomic operations in local memory space and thus histogram construction will be slow.
|
||||
|
||||
- AMD VLIW4-based GPUs, including Radeon HD 6xxx series and earlier GPUs. These GPUs have been discontinued for years and are rarely seen nowadays.
|
||||
|
||||
How to Achieve Good Speedup on GPU
|
||||
----------------------------------
|
||||
|
||||
#. You want to run a few datasets that we have verified with good speedup (including Higgs, epsilon, Bosch, etc) to ensure your setup is correct.
|
||||
If you have multiple GPUs, make sure to set ``gpu_platform_id`` and ``gpu_device_id`` to use the desired GPU.
|
||||
Also make sure your system is idle (especially when using a shared computer) to get accuracy performance measurements.
|
||||
|
||||
#. GPU works best on large scale and dense datasets. If dataset is too small, computing it on GPU is inefficient as the data transfer overhead can be significant.
|
||||
If you have categorical features, use the ``categorical_column`` option and input them into LightGBM directly; do not convert them into one-hot variables.
|
||||
|
||||
#. To get good speedup with GPU, it is suggested to use a smaller number of bins.
|
||||
Setting ``max_bin=63`` is recommended, as it usually does not noticeably affect training accuracy on large datasets, but GPU training can be significantly faster than using the default bin size of 255.
|
||||
For some dataset, even using 15 bins is enough (``max_bin=15``); using 15 bins will maximize GPU performance. Make sure to check the run log and verify that the desired number of bins is used.
|
||||
|
||||
#. Try to use single precision training (``gpu_use_dp=false``) when possible, because most GPUs (especially NVIDIA consumer GPUs) have poor double-precision performance.
|
||||
|
||||
Performance Comparison
|
||||
----------------------
|
||||
|
||||
We evaluate the training performance of GPU acceleration on the following datasets:
|
||||
|
||||
+-----------+----------------+----------+------------+-----------+------------+
|
||||
| Data | Task | Link | #Examples | #Features | Comments |
|
||||
+===========+================+==========+============+===========+============+
|
||||
| Higgs | Binary | `link1`_ | 10,500,000 | 28 | use last |
|
||||
| | classification | | | | 500,000 |
|
||||
| | | | | | samples |
|
||||
| | | | | | as test |
|
||||
| | | | | | set |
|
||||
+-----------+----------------+----------+------------+-----------+------------+
|
||||
| Epsilon | Binary | `link2`_ | 400,000 | 2,000 | use the |
|
||||
| | classification | | | | provided |
|
||||
| | | | | | test set |
|
||||
+-----------+----------------+----------+------------+-----------+------------+
|
||||
| Bosch | Binary | `link3`_ | 1,000,000 | 968 | use the |
|
||||
| | classification | | | | provided |
|
||||
| | | | | | test set |
|
||||
+-----------+----------------+----------+------------+-----------+------------+
|
||||
| Yahoo LTR | Learning to | `link4`_ | 473,134 | 700 | set1.train |
|
||||
| | rank | | | | as train, |
|
||||
| | | | | | set1.test |
|
||||
| | | | | | as test |
|
||||
+-----------+----------------+----------+------------+-----------+------------+
|
||||
| MS LTR | Learning to | `link5`_ | 2,270,296 | 137 | {S1,S2,S3} |
|
||||
| | rank | | | | as train |
|
||||
| | | | | | set, {S5} |
|
||||
| | | | | | as test |
|
||||
| | | | | | set |
|
||||
+-----------+----------------+----------+------------+-----------+------------+
|
||||
| Expo | Binary | `link6`_ | 11,000,000 | 700 | use last |
|
||||
| | classification | | | | 1,000,000 |
|
||||
| | (Categorical) | | | | as test |
|
||||
| | | | | | set |
|
||||
+-----------+----------------+----------+------------+-----------+------------+
|
||||
|
||||
We used the following hardware to evaluate the performance of LightGBM GPU training.
|
||||
Our CPU reference is **a high-end dual socket Haswell-EP Xeon server with 28 cores**;
|
||||
GPUs include a budget GPU (RX 480) and a mainstream (GTX 1080) GPU installed on the same server.
|
||||
It is worth mentioning that **the GPUs used are not the best GPUs in the market**;
|
||||
if you are using a better GPU (like AMD RX 580, NVIDIA GTX 1080 Ti, Titan X Pascal, Titan Xp, Tesla P100, etc), you are likely to get a better speedup.
|
||||
|
||||
+--------------------------------+----------------+------------------+---------------+
|
||||
| Hardware | Peak FLOPS | Peak Memory BW | Cost (MSRP) |
|
||||
+================================+================+==================+===============+
|
||||
| AMD Radeon RX 480 | 5,161 GFLOPS | 256 GB/s | $199 |
|
||||
+--------------------------------+----------------+------------------+---------------+
|
||||
| NVIDIA GTX 1080 | 8,228 GFLOPS | 320 GB/s | $499 |
|
||||
+--------------------------------+----------------+------------------+---------------+
|
||||
| 2x Xeon E5-2683v3 (28 cores) | 1,792 GFLOPS | 133 GB/s | $3,692 |
|
||||
+--------------------------------+----------------+------------------+---------------+
|
||||
|
||||
During benchmarking on CPU we used only 28 physical cores of the CPU, and did not use hyper-threading cores,
|
||||
because we found that using too many threads actually makes performance worse.
|
||||
The following shows the training configuration we used:
|
||||
|
||||
::
|
||||
|
||||
max_bin = 63
|
||||
num_leaves = 255
|
||||
num_iterations = 500
|
||||
learning_rate = 0.1
|
||||
tree_learner = serial
|
||||
task = train
|
||||
is_training_metric = false
|
||||
min_data_in_leaf = 1
|
||||
min_sum_hessian_in_leaf = 100
|
||||
ndcg_eval_at = 1,3,5,10
|
||||
device = gpu
|
||||
gpu_platform_id = 0
|
||||
gpu_device_id = 0
|
||||
num_thread = 28
|
||||
|
||||
We use the configuration shown above, except for the Bosch dataset, we use a smaller ``learning_rate=0.015`` and set ``min_sum_hessian_in_leaf=5``.
|
||||
For all GPU training we vary the max number of bins (255, 63 and 15).
|
||||
The GPU implementation is from commit `0bb4a82`_ of LightGBM, when the GPU support was just merged in.
|
||||
|
||||
The following table lists the accuracy on test set that CPU and GPU learner can achieve after 500 iterations.
|
||||
GPU with the same number of bins can achieve a similar level of accuracy as on the CPU, despite using single precision arithmetic.
|
||||
For most datasets, using 63 bins is sufficient.
|
||||
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| | CPU 255 bins | CPU 63 bins | CPU 15 bins | GPU 255 bins | GPU 63 bins | GPU 15 bins |
|
||||
+===========================+================+===============+===============+================+===============+===============+
|
||||
| Higgs AUC | 0.845612 | 0.845239 | 0.841066 | 0.845612 | 0.845209 | 0.840748 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| Epsilon AUC | 0.950243 | 0.949952 | 0.948365 | 0.950057 | 0.949876 | 0.948365 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| Yahoo-LTR NDCG\ :sub:`1` | 0.730824 | 0.730165 | 0.729647 | 0.730936 | 0.732257 | 0.73114 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| Yahoo-LTR NDCG\ :sub:`3` | 0.738687 | 0.737243 | 0.736445 | 0.73698 | 0.739474 | 0.735868 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| Yahoo-LTR NDCG\ :sub:`5` | 0.756609 | 0.755729 | 0.754607 | 0.756206 | 0.757007 | 0.754203 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| Yahoo-LTR NDCG\ :sub:`10` | 0.79655 | 0.795827 | 0.795273 | 0.795894 | 0.797302 | 0.795584 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| Expo AUC | 0.776217 | 0.771566 | 0.743329 | 0.776285 | 0.77098 | 0.744078 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| MS-LTR NDCG\ :sub:`1` | 0.521265 | 0.521392 | 0.518653 | 0.521789 | 0.522163 | 0.516388 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| MS-LTR NDCG\ :sub:`3` | 0.503153 | 0.505753 | 0.501697 | 0.503886 | 0.504089 | 0.501691 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| MS-LTR NDCG\ :sub:`5` | 0.509236 | 0.510391 | 0.507193 | 0.509861 | 0.510095 | 0.50663 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| MS-LTR NDCG\ :sub:`10` | 0.527835 | 0.527304 | 0.524603 | 0.528009 | 0.527059 | 0.524722 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
| Bosch AUC | 0.718115 | 0.721791 | 0.716677 | 0.717184 | 0.724761 | 0.717005 |
|
||||
+---------------------------+----------------+---------------+---------------+----------------+---------------+---------------+
|
||||
|
||||
We record the wall clock time after 500 iterations, as shown in the figure below:
|
||||
|
||||
.. image:: ./_static/images/gpu-performance-comparison.png
|
||||
:align: center
|
||||
:target: ./_static/images/gpu-performance-comparison.png
|
||||
:alt: A performance chart which is a record of the wall clock time after 500 iterations on G P U for Higgs, epsilon, Bosch, Microsoft L T R, Expo and Yahoo L T R and bin size of 63 performs comparatively better.
|
||||
|
||||
When using a GPU, it is advisable to use a bin size of 63 rather than 255, because it can speed up training significantly without noticeably affecting accuracy.
|
||||
On CPU, using a smaller bin size only marginally improves performance, sometimes even slows down training,
|
||||
like in Higgs (we can reproduce the same slowdown on two different machines, with different GCC versions).
|
||||
We found that GPU can achieve impressive acceleration on large and dense datasets like Higgs and Epsilon.
|
||||
Even on smaller and sparse datasets, a *budget* GPU can still compete and be faster than a 28-core Haswell server.
|
||||
|
||||
Memory Usage
|
||||
------------
|
||||
|
||||
The next table shows GPU memory usage reported by ``nvidia-smi`` during training with 63 bins.
|
||||
We can see that even the largest dataset just uses about 1 GB of GPU memory,
|
||||
indicating that our GPU implementation can scale to huge datasets over 10x larger than Bosch or Epsilon.
|
||||
Also, we can observe that generally a larger dataset (using more GPU memory, like Epsilon or Bosch) has better speedup,
|
||||
because the overhead of invoking GPU functions becomes significant when the dataset is small.
|
||||
|
||||
+-------------------------+---------+-----------+---------+----------+--------+-------------+
|
||||
| Datasets | Higgs | Epsilon | Bosch | MS-LTR | Expo | Yahoo-LTR |
|
||||
+=========================+=========+===========+=========+==========+========+=============+
|
||||
| GPU Memory Usage (MB) | 611 | 901 | 1067 | 413 | 405 | 291 |
|
||||
+-------------------------+---------+-----------+---------+----------+--------+-------------+
|
||||
|
||||
Further Reading
|
||||
---------------
|
||||
|
||||
You can find more details about the GPU algorithm and benchmarks in the
|
||||
following article:
|
||||
|
||||
Huan Zhang, Si Si and Cho-Jui Hsieh. `GPU Acceleration for Large-scale Tree Boosting`_. SysML Conference, 2018.
|
||||
|
||||
.. _link1: https://archive.ics.uci.edu/dataset/280/higgs
|
||||
|
||||
.. _link2: https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary.html
|
||||
|
||||
.. _link3: https://www.kaggle.com/c/bosch-production-line-performance/data
|
||||
|
||||
.. _link4: https://proceedings.mlr.press/v14/chapelle11a.html
|
||||
|
||||
.. _link5: https://www.microsoft.com/en-us/research/project/mslr/
|
||||
|
||||
.. _link6: https://community.amstat.org/jointscsg-section/dataexpo/dataexpo2009
|
||||
|
||||
.. _0bb4a82: https://github.com/lightgbm-org/LightGBM/commit/0bb4a82
|
||||
|
||||
.. _GPU Acceleration for Large-scale Tree Boosting: https://arxiv.org/abs/1706.08359
|
||||
@@ -0,0 +1,174 @@
|
||||
GPU SDK Correspondence and Device Targeting Table
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
GPU Targets Table
|
||||
=================
|
||||
|
||||
OpenCL is a universal massively parallel programming framework that targets multiple backends (GPU, CPU, FPGA, etc).
|
||||
Basically, to use a device from a vendor, you have to install drivers from that specific vendor.
|
||||
Intel's and AMD's OpenCL runtime also include x86 CPU target support.
|
||||
NVIDIA's OpenCL runtime only supports NVIDIA GPU (no CPU support).
|
||||
In general, OpenCL CPU backends are quite slow, and should be used for testing and debugging only.
|
||||
|
||||
You can find below a table of correspondence:
|
||||
|
||||
+---------------------------+-----------------+-----------------+-----------------+--------------+
|
||||
| SDK | CPU Intel/AMD | GPU Intel | GPU AMD | GPU NVIDIA |
|
||||
+===========================+=================+=================+=================+==============+
|
||||
| `Intel SDK for OpenCL`_ | Supported | Supported | Not Supported | Not Supported|
|
||||
+---------------------------+-----------------+-----------------+-----------------+--------------+
|
||||
| AMD APP SDK \* | Supported | Not Supported | Supported | Not Supported|
|
||||
+---------------------------+-----------------+-----------------+-----------------+--------------+
|
||||
| `PoCL`_ | Supported | Not Supported | Supported | Not Supported|
|
||||
+---------------------------+-----------------+-----------------+-----------------+--------------+
|
||||
| `NVIDIA CUDA Toolkit`_ | Not Supported | Not Supported | Not Supported | Supported |
|
||||
+---------------------------+-----------------+-----------------+-----------------+--------------+
|
||||
|
||||
Legend:
|
||||
|
||||
\* AMD APP SDK is deprecated. On Windows, OpenCL is included in AMD graphics driver. On Linux, newer generation AMD cards are supported by the `ROCm`_ driver. You can download an archived copy of AMD APP SDK from our GitHub repo (`for Linux`_ and `for Windows`_).
|
||||
|
||||
|
||||
--------------
|
||||
|
||||
Query OpenCL Devices in Your System
|
||||
===================================
|
||||
|
||||
Your system might have multiple GPUs from different vendors ("platforms") installed. Setting up LightGBM GPU device requires two parameters: `OpenCL Platform ID <./Parameters.rst#gpu_platform_id>`__ (``gpu_platform_id``) and `OpenCL Device ID <./Parameters.rst#gpu_device_id>`__ (``gpu_device_id``). Generally speaking, each vendor provides an OpenCL platform, and devices from the same vendor have different device IDs under that platform. For example, if your system has an Intel integrated GPU and two discrete GPUs from AMD, you will have two OpenCL platforms (with ``gpu_platform_id=0`` and ``gpu_platform_id=1``). If the platform 0 is Intel, it has one device (``gpu_device_id=0``) representing the Intel GPU; if the platform 1 is AMD, it has two devices (``gpu_device_id=0``, ``gpu_device_id=1``) representing the two AMD GPUs. If you have a discrete GPU by AMD/NVIDIA and an integrated GPU by Intel, make sure to select the correct ``gpu_platform_id`` to use the discrete GPU as it usually provides better performance.
|
||||
|
||||
On Windows, OpenCL devices can be queried using `GPUCapsViewer`_, under the OpenCL tab. Note that the platform and device IDs reported by this utility start from 1. So you should minus the reported IDs by 1.
|
||||
|
||||
On Linux, OpenCL devices can be listed using the ``clinfo`` command. On Ubuntu, you can install ``clinfo`` by executing ``sudo apt-get install clinfo``.
|
||||
|
||||
|
||||
Examples
|
||||
===============
|
||||
|
||||
We provide test R code below, but you can use the language of your choice with the examples of your choices:
|
||||
|
||||
.. code:: r
|
||||
|
||||
library(lightgbm)
|
||||
data(agaricus.train, package = "lightgbm")
|
||||
train <- agaricus.train
|
||||
train$data[, 1] <- 1:6513
|
||||
dtrain <- lgb.Dataset(train$data, label = train$label)
|
||||
data(agaricus.test, package = "lightgbm")
|
||||
test <- agaricus.test
|
||||
dtest <- lgb.Dataset.create.valid(dtrain, test$data, label = test$label)
|
||||
valids <- list(test = dtest)
|
||||
|
||||
params <- list(objective = "regression",
|
||||
metric = "rmse",
|
||||
device = "gpu",
|
||||
gpu_platform_id = 0,
|
||||
gpu_device_id = 0,
|
||||
nthread = 1,
|
||||
boost_from_average = FALSE,
|
||||
num_tree_per_iteration = 10,
|
||||
max_bin = 32)
|
||||
model <- lgb.train(params,
|
||||
dtrain,
|
||||
2,
|
||||
valids,
|
||||
min_data = 1,
|
||||
learning_rate = 1,
|
||||
early_stopping_rounds = 10)
|
||||
|
||||
Make sure you list the OpenCL devices in your system and set ``gpu_platform_id`` and ``gpu_device_id`` correctly. In the following examples, our system has 1 GPU platform (``gpu_platform_id = 0``) from AMD APP SDK. The first device ``gpu_device_id = 0`` is a GPU device (AMD Oland), and the second device ``gpu_device_id = 1`` is the x86 CPU backend.
|
||||
|
||||
Example of using GPU (``gpu_platform_id = 0`` and ``gpu_device_id = 0`` in our system):
|
||||
|
||||
.. code:: r
|
||||
|
||||
> params <- list(objective = "regression",
|
||||
+ metric = "rmse",
|
||||
+ device = "gpu",
|
||||
+ gpu_platform_id = 0,
|
||||
+ gpu_device_id = 0,
|
||||
+ nthread = 1,
|
||||
+ boost_from_average = FALSE,
|
||||
+ num_tree_per_iteration = 10,
|
||||
+ max_bin = 32)
|
||||
> model <- lgb.train(params,
|
||||
+ dtrain,
|
||||
+ 2,
|
||||
+ valids,
|
||||
+ min_data = 1,
|
||||
+ learning_rate = 1,
|
||||
+ early_stopping_rounds = 10)
|
||||
[LightGBM] [Info] This is the GPU trainer!!
|
||||
[LightGBM] [Info] Total Bins 232
|
||||
[LightGBM] [Info] Number of data: 6513, number of used features: 116
|
||||
[LightGBM] [Info] Using GPU Device: Oland, Vendor: Advanced Micro Devices, Inc.
|
||||
[LightGBM] [Info] Compiling OpenCL Kernel with 16 bins...
|
||||
[LightGBM] [Info] GPU programs have been built
|
||||
[LightGBM] [Info] Size of histogram bin entry: 12
|
||||
[LightGBM] [Info] 40 dense feature groups (0.12 MB) transferred to GPU in 0.004211 secs. 76 sparse feature groups.
|
||||
[LightGBM] [Info] No further splits with positive gain, best gain: -inf
|
||||
[LightGBM] [Info] Trained a tree with leaves=16 and depth=8
|
||||
[1]: test's rmse:1.10643e-17
|
||||
[LightGBM] [Info] No further splits with positive gain, best gain: -inf
|
||||
[LightGBM] [Info] Trained a tree with leaves=7 and depth=5
|
||||
[2]: test's rmse:0
|
||||
|
||||
Running on OpenCL CPU backend devices is in generally slow, and we observe crashes on some Windows and macOS systems. Make sure you check the ``Using GPU Device`` line in the log and it is not using a CPU. The above log shows that we are using ``Oland`` GPU from AMD and not CPU.
|
||||
|
||||
Example of using CPU (``gpu_platform_id = 0``, ``gpu_device_id = 1``). The GPU device reported is ``Intel(R) Core(TM) i7-4600U CPU``, so it is using the CPU backend rather than a real GPU.
|
||||
|
||||
.. code:: r
|
||||
|
||||
> params <- list(objective = "regression",
|
||||
+ metric = "rmse",
|
||||
+ device = "gpu",
|
||||
+ gpu_platform_id = 0,
|
||||
+ gpu_device_id = 1,
|
||||
+ nthread = 1,
|
||||
+ boost_from_average = FALSE,
|
||||
+ num_tree_per_iteration = 10,
|
||||
+ max_bin = 32)
|
||||
> model <- lgb.train(params,
|
||||
+ dtrain,
|
||||
+ 2,
|
||||
+ valids,
|
||||
+ min_data = 1,
|
||||
+ learning_rate = 1,
|
||||
+ early_stopping_rounds = 10)
|
||||
[LightGBM] [Info] This is the GPU trainer!!
|
||||
[LightGBM] [Info] Total Bins 232
|
||||
[LightGBM] [Info] Number of data: 6513, number of used features: 116
|
||||
[LightGBM] [Info] Using requested OpenCL platform 0 device 1
|
||||
[LightGBM] [Info] Using GPU Device: Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz, Vendor: GenuineIntel
|
||||
[LightGBM] [Info] Compiling OpenCL Kernel with 16 bins...
|
||||
[LightGBM] [Info] GPU programs have been built
|
||||
[LightGBM] [Info] Size of histogram bin entry: 12
|
||||
[LightGBM] [Info] 40 dense feature groups (0.12 MB) transferred to GPU in 0.004540 secs. 76 sparse feature groups.
|
||||
[LightGBM] [Info] No further splits with positive gain, best gain: -inf
|
||||
[LightGBM] [Info] Trained a tree with leaves=16 and depth=8
|
||||
[1]: test's rmse:1.10643e-17
|
||||
[LightGBM] [Info] No further splits with positive gain, best gain: -inf
|
||||
[LightGBM] [Info] Trained a tree with leaves=7 and depth=5
|
||||
[2]: test's rmse:0
|
||||
|
||||
|
||||
Known issues:
|
||||
|
||||
- Using a bad combination of ``gpu_platform_id`` and ``gpu_device_id`` can potentially lead to a **crash** due to OpenCL driver issues on some machines (you will lose your entire session content). Beware of it.
|
||||
|
||||
- On some systems, if you have integrated graphics card (Intel HD Graphics) and a dedicated graphics card (AMD, NVIDIA), the dedicated graphics card will automatically override the integrated graphics card. The workaround is to disable your dedicated graphics card to be able to use your integrated graphics card.
|
||||
|
||||
.. _Intel SDK for OpenCL: https://software.intel.com/en-us/articles/opencl-drivers
|
||||
|
||||
.. _ROCm: https://rocmdocs.amd.com/en/latest/
|
||||
|
||||
.. _for Linux: https://github.com/lightgbm-org/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.136-GA-linux64.tar.bz2
|
||||
|
||||
.. _for Windows: https://github.com/lightgbm-org/LightGBM/releases/download/v2.0.12/AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe
|
||||
|
||||
.. _NVIDIA CUDA Toolkit: https://developer.nvidia.com/cuda-downloads
|
||||
|
||||
.. _clinfo: https://github.com/Oblomov/clinfo
|
||||
|
||||
.. _GPUCapsViewer: https://www.ozone3d.net/gpu_caps_viewer/
|
||||
|
||||
.. _PoCL: https://portablecl.org/
|
||||
@@ -0,0 +1,192 @@
|
||||
LightGBM GPU Tutorial
|
||||
=====================
|
||||
|
||||
The purpose of this document is to give you a quick step-by-step tutorial on GPU training.
|
||||
|
||||
We will use the GPU instance on `Microsoft Azure cloud computing platform`_ for demonstration,
|
||||
but you can use any machine with modern AMD or NVIDIA GPUs.
|
||||
|
||||
GPU Setup
|
||||
---------
|
||||
|
||||
You need to launch a ``NV`` type instance on Azure (available in East US, North Central US, South Central US, West Europe and Southeast Asia zones)
|
||||
and select Ubuntu 16.04 LTS as the operating system.
|
||||
|
||||
For testing, the smallest ``NV6`` type virtual machine is sufficient, which includes 1/2 M60 GPU, with 8 GB memory, 180 GB/s memory bandwidth and 4,825 GFLOPS peak computation power.
|
||||
Don't use the ``NC`` type instance as the GPUs (K80) are based on an older architecture (Kepler).
|
||||
|
||||
First we need to install minimal NVIDIA drivers and OpenCL development environment:
|
||||
|
||||
::
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends nvidia-375
|
||||
sudo apt-get install --no-install-recommends nvidia-opencl-icd-375 nvidia-opencl-dev opencl-headers
|
||||
|
||||
After installing the drivers you need to restart the server.
|
||||
|
||||
::
|
||||
|
||||
sudo init 6
|
||||
|
||||
After about 30 seconds, the server should be up again.
|
||||
|
||||
If you are using an AMD GPU, you should download and install the `AMDGPU-Pro`_ driver and also install packages ``ocl-icd-libopencl1`` and ``ocl-icd-opencl-dev``.
|
||||
|
||||
Build LightGBM
|
||||
--------------
|
||||
|
||||
Now install necessary building tools and dependencies:
|
||||
|
||||
::
|
||||
|
||||
sudo apt-get install --no-install-recommends git cmake build-essential libboost-dev libboost-system-dev libboost-filesystem-dev
|
||||
|
||||
The ``NV6`` GPU instance has a 320 GB ultra-fast SSD mounted at ``/mnt``.
|
||||
Let's use it as our workspace (skip this if you are using your own machine):
|
||||
|
||||
::
|
||||
|
||||
sudo mkdir -p /mnt/workspace
|
||||
sudo chown $(whoami):$(whoami) /mnt/workspace
|
||||
cd /mnt/workspace
|
||||
|
||||
Now we are ready to checkout LightGBM and compile it with GPU support:
|
||||
|
||||
::
|
||||
|
||||
git clone --recursive https://github.com/lightgbm-org/LightGBM
|
||||
cd LightGBM
|
||||
cmake -B build -S . -DUSE_GPU=1
|
||||
# if you have installed NVIDIA CUDA to a customized location, you should specify paths to OpenCL headers and library like the following:
|
||||
# cmake -B build -S . -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/
|
||||
cmake --build build -j$(nproc)
|
||||
|
||||
You will see two binaries are generated, ``lightgbm`` and ``lib_lightgbm.so``.
|
||||
|
||||
If you are building on macOS, you probably need to remove macro ``BOOST_COMPUTE_USE_OFFLINE_CACHE`` in ``src/treelearner/gpu_tree_learner.h`` to avoid a known crash bug in Boost.Compute.
|
||||
|
||||
Install Python Interface (optional)
|
||||
-----------------------------------
|
||||
|
||||
If you want to use the Python interface of LightGBM, you can install it now (along with some necessary Python-package dependencies):
|
||||
|
||||
::
|
||||
|
||||
sudo apt-get -y install python3-pip python3-venv
|
||||
sudo -H pip install numpy scipy scikit-learn -U
|
||||
sudo sh ./build-python.sh install --precompile
|
||||
|
||||
You need to set an additional parameter ``"device" : "gpu"`` (along with your other options like ``learning_rate``, ``num_leaves``, etc) to use GPU in Python.
|
||||
|
||||
You can read our `Python-package Examples`_ for more information on how to use the Python interface.
|
||||
|
||||
Dataset Preparation
|
||||
-------------------
|
||||
|
||||
Using the following commands to prepare the Higgs dataset:
|
||||
|
||||
::
|
||||
|
||||
git clone https://github.com/guolinke/boosting_tree_benchmarks.git
|
||||
cd boosting_tree_benchmarks/data
|
||||
wget "https://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz"
|
||||
gunzip HIGGS.csv.gz
|
||||
python higgs2libsvm.py
|
||||
cd ../..
|
||||
ln -s boosting_tree_benchmarks/data/higgs.train
|
||||
ln -s boosting_tree_benchmarks/data/higgs.test
|
||||
|
||||
Now we create a configuration file for LightGBM by running the following commands (please copy the entire block and run it as a whole):
|
||||
|
||||
::
|
||||
|
||||
cat > lightgbm_gpu.conf <<EOF
|
||||
max_bin = 63
|
||||
num_leaves = 255
|
||||
num_iterations = 50
|
||||
learning_rate = 0.1
|
||||
tree_learner = serial
|
||||
task = train
|
||||
is_training_metric = false
|
||||
min_data_in_leaf = 1
|
||||
min_sum_hessian_in_leaf = 100
|
||||
ndcg_eval_at = 1,3,5,10
|
||||
device = gpu
|
||||
gpu_platform_id = 0
|
||||
gpu_device_id = 0
|
||||
EOF
|
||||
echo "num_threads=$(nproc)" >> lightgbm_gpu.conf
|
||||
|
||||
GPU is enabled in the configuration file we just created by setting ``device=gpu``.
|
||||
In this configuration we use the first GPU installed on the system (``gpu_platform_id=0`` and ``gpu_device_id=0``). If ``gpu_platform_id`` or ``gpu_device_id`` is not set, the default platform and GPU will be selected.
|
||||
You might have multiple platforms (AMD/Intel/NVIDIA) or GPUs. You can use the `clinfo`_ utility to identify the GPUs on each platform. On Ubuntu, you can install ``clinfo`` by executing ``sudo apt-get install clinfo``. If you have a discrete GPU by AMD/NVIDIA and an integrated GPU by Intel, make sure to select the correct ``gpu_platform_id`` to use the discrete GPU.
|
||||
|
||||
Run Your First Learning Task on GPU
|
||||
-----------------------------------
|
||||
|
||||
Now we are ready to start GPU training!
|
||||
|
||||
First we want to verify the GPU works correctly.
|
||||
Run the following command to train on GPU, and take a note of the AUC after 50 iterations:
|
||||
|
||||
::
|
||||
|
||||
./lightgbm config=lightgbm_gpu.conf data=higgs.train valid=higgs.test objective=binary metric=auc
|
||||
|
||||
Now train the same dataset on CPU using the following command. You should observe a similar AUC:
|
||||
|
||||
::
|
||||
|
||||
./lightgbm config=lightgbm_gpu.conf data=higgs.train valid=higgs.test objective=binary metric=auc device=cpu
|
||||
|
||||
Now we can make a speed test on GPU without calculating AUC after each iteration.
|
||||
|
||||
::
|
||||
|
||||
./lightgbm config=lightgbm_gpu.conf data=higgs.train objective=binary metric=auc
|
||||
|
||||
Speed test on CPU:
|
||||
|
||||
::
|
||||
|
||||
./lightgbm config=lightgbm_gpu.conf data=higgs.train objective=binary metric=auc device=cpu
|
||||
|
||||
You should observe over three times speedup on this GPU.
|
||||
|
||||
The GPU acceleration can be used on other tasks/metrics (regression, multi-class classification, ranking, etc) as well.
|
||||
For example, we can train the Higgs dataset on GPU as a regression task:
|
||||
|
||||
::
|
||||
|
||||
./lightgbm config=lightgbm_gpu.conf data=higgs.train objective=regression_l2 metric=l2
|
||||
|
||||
Also, you can compare the training speed with CPU:
|
||||
|
||||
::
|
||||
|
||||
./lightgbm config=lightgbm_gpu.conf data=higgs.train objective=regression_l2 metric=l2 device=cpu
|
||||
|
||||
Further Reading
|
||||
---------------
|
||||
|
||||
- `GPU Tuning Guide and Performance Comparison <./GPU-Performance.rst>`__
|
||||
|
||||
- `GPU SDK Correspondence and Device Targeting Table <./GPU-Targets.rst>`__
|
||||
|
||||
Reference
|
||||
---------
|
||||
|
||||
Please kindly cite the following article in your publications if you find the GPU acceleration useful:
|
||||
|
||||
Huan Zhang, Si Si and Cho-Jui Hsieh. "`GPU Acceleration for Large-scale Tree Boosting`_." SysML Conference, 2018.
|
||||
|
||||
.. _Microsoft Azure cloud computing platform: https://azure.microsoft.com/
|
||||
|
||||
.. _AMDGPU-Pro: https://www.amd.com/en/support.html
|
||||
|
||||
.. _Python-package Examples: https://github.com/lightgbm-org/LightGBM/tree/main/examples/python-guide
|
||||
|
||||
.. _GPU Acceleration for Large-scale Tree Boosting: https://arxiv.org/abs/1706.08359
|
||||
|
||||
.. _clinfo: https://github.com/Oblomov/clinfo
|
||||
@@ -0,0 +1,3 @@
|
||||
The content of this document was very outdated and is no longer available to avoid any misleadings.
|
||||
|
||||
Starting from the ``3.2.0`` version LightGBM Python packages have been having built-in support of training on GPU devices.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
The content of this document was very outdated and is no longer available to avoid any misleadings.
|
||||
@@ -0,0 +1,20 @@
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line.
|
||||
SPHINXOPTS = -W
|
||||
SPHINXBUILD = sphinx-build
|
||||
SPHINXPROJ = LightGBM
|
||||
SOURCEDIR = .
|
||||
BUILDDIR = _build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
@@ -0,0 +1,549 @@
|
||||
Distributed Learning Guide
|
||||
==========================
|
||||
|
||||
.. _Parallel Learning Guide:
|
||||
|
||||
This guide describes distributed learning in LightGBM. Distributed learning allows the use of multiple machines to produce a single model.
|
||||
|
||||
Follow the `Quick Start <./Quick-Start.rst>`__ to know how to use LightGBM first.
|
||||
|
||||
How Distributed LightGBM Works
|
||||
------------------------------
|
||||
|
||||
This section describes how distributed learning in LightGBM works. To learn how to do this in various programming languages and frameworks, please see `Integrations <#integrations>`__.
|
||||
|
||||
Choose Appropriate Parallel Algorithm
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
LightGBM provides 3 distributed learning algorithms now.
|
||||
|
||||
+--------------------+---------------------------+
|
||||
| Parallel Algorithm | How to Use |
|
||||
+====================+===========================+
|
||||
| Data parallel | ``tree_learner=data`` |
|
||||
+--------------------+---------------------------+
|
||||
| Feature parallel | ``tree_learner=feature`` |
|
||||
+--------------------+---------------------------+
|
||||
| Voting parallel | ``tree_learner=voting`` |
|
||||
+--------------------+---------------------------+
|
||||
|
||||
These algorithms are suited for different scenarios, which is listed in the following table:
|
||||
|
||||
+-------------------------+-------------------+-----------------+
|
||||
| | #data is small | #data is large |
|
||||
+=========================+===================+=================+
|
||||
| **#feature is small** | Feature Parallel | Data Parallel |
|
||||
+-------------------------+-------------------+-----------------+
|
||||
| **#feature is large** | Feature Parallel | Voting Parallel |
|
||||
+-------------------------+-------------------+-----------------+
|
||||
|
||||
More details about these parallel algorithms can be found in `optimization in distributed learning <./Features.rst#optimization-in-distributed-learning>`__.
|
||||
|
||||
Integrations
|
||||
------------
|
||||
|
||||
This section describes how to run distributed LightGBM training in various programming languages and frameworks. To learn how distributed learning in LightGBM works generally, please see `How Distributed LightGBM Works <#how-distributed-lightgbm-works>`__.
|
||||
|
||||
Apache Spark
|
||||
^^^^^^^^^^^^
|
||||
|
||||
Apache Spark users can use `SynapseML`_ for machine learning workflows with LightGBM. This project is not maintained by LightGBM's maintainers.
|
||||
|
||||
See `this SynapseML example`_ for additional information on using LightGBM on Spark.
|
||||
|
||||
.. note::
|
||||
|
||||
``SynapseML`` is not maintained by LightGBM's maintainers. Bug reports or feature requests should be directed to https://github.com/microsoft/SynapseML/issues.
|
||||
|
||||
Dask
|
||||
^^^^
|
||||
|
||||
.. versionadded:: 3.2.0
|
||||
|
||||
LightGBM's Python-package supports distributed learning via `Dask`_. This integration is maintained by LightGBM's maintainers.
|
||||
|
||||
.. warning::
|
||||
|
||||
Dask integration is only tested on macOS and Linux.
|
||||
|
||||
Dask Examples
|
||||
'''''''''''''
|
||||
|
||||
For sample code using ``lightgbm.dask``, see `these Dask examples`_.
|
||||
|
||||
Training with Dask
|
||||
''''''''''''''''''
|
||||
|
||||
This section contains detailed information on performing LightGBM distributed training using Dask.
|
||||
|
||||
Configuring the Dask Cluster
|
||||
****************************
|
||||
|
||||
**Allocating Threads**
|
||||
|
||||
When setting up a Dask cluster for training, give each Dask worker process at least two threads. If you do not do this, training might be substantially slower because communication work and training work will block each other.
|
||||
|
||||
If you do not have other significant processes competing with Dask for resources, just accept the default ``nthreads`` from your chosen ``dask.distributed`` cluster.
|
||||
|
||||
.. code:: python
|
||||
|
||||
from distributed import Client, LocalCluster
|
||||
|
||||
cluster = LocalCluster(n_workers=3)
|
||||
client = Client(cluster)
|
||||
|
||||
**Managing Memory**
|
||||
|
||||
Use the Dask diagnostic dashboard or your preferred monitoring tool to monitor Dask workers' memory consumption during training. As described in `the Dask worker documentation`_, Dask workers will automatically start spilling data to disk if memory consumption gets too high. This can substantially slow down computations, since disk I/O is usually much slower than reading the same data from memory.
|
||||
|
||||
`At 60% of memory load, [Dask will] spill least recently used data to disk`
|
||||
|
||||
To reduce the risk of hitting memory limits, consider restarting each worker process before running any data loading or training code.
|
||||
|
||||
.. code:: python
|
||||
|
||||
client.restart()
|
||||
|
||||
Setting Up Training Data
|
||||
*************************
|
||||
|
||||
The estimators in ``lightgbm.dask`` expect that matrix-like or array-like data are provided in Dask DataFrame, Dask Array, or (in some cases) Dask Series format. See `the Dask DataFrame documentation`_ and `the Dask Array documentation`_ for more information on how to create such data structures.
|
||||
|
||||
.. image:: ./_static/images/dask-initial-setup.svg
|
||||
:align: center
|
||||
:width: 600px
|
||||
:alt: On the left, rectangles showing a 5 by 5 grid for a local dataset. On the right, two circles representing Dask workers, one with a 3 by 5 grid and one with a 2 by 5 grid.
|
||||
:target: ./_static/images/dask-initial-setup.svg
|
||||
|
||||
While setting up for training, ``lightgbm`` will concatenate all of the partitions on a worker into a single dataset. Distributed training then proceeds with one LightGBM worker process per Dask worker.
|
||||
|
||||
.. image:: ./_static/images/dask-concat.svg
|
||||
:align: center
|
||||
:width: 600px
|
||||
:alt: A section labeled "before" showing two grids and a section labeled "after" showing a single grid that looks like the two from "before" stacked one on top of the other.
|
||||
:target: ./_static/images/dask-concat.svg
|
||||
|
||||
When setting up data partitioning for LightGBM training with Dask, try to follow these suggestions:
|
||||
|
||||
* ensure that each worker in the cluster has some of the training data
|
||||
* try to give each worker roughly the same amount of data, especially if your dataset is small
|
||||
* if you plan to train multiple models (for example, to tune hyperparameters) on the same data, use ``client.persist()`` before training to materialize the data one time
|
||||
|
||||
Using a Specific Dask Client
|
||||
****************************
|
||||
|
||||
In most situations, you should not need to tell ``lightgbm.dask`` to use a specific Dask client. By default, the client returned by ``distributed.default_client()`` will be used.
|
||||
|
||||
However, you might want to explicitly control the Dask client used by LightGBM if you have multiple active clients in the same session. This is useful in more complex workflows like running multiple training jobs on different Dask clusters.
|
||||
|
||||
LightGBM's Dask estimators support setting an attribute ``client`` to control the client that is used.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import lightgbm as lgb
|
||||
from distributed import Client, LocalCluster
|
||||
|
||||
cluster = LocalCluster()
|
||||
client = Client(cluster)
|
||||
|
||||
# option 1: keyword argument in constructor
|
||||
dask_model = lgb.DaskLGBMClassifier(client=client)
|
||||
|
||||
# option 2: set_params() after construction
|
||||
dask_model = lgb.DaskLGBMClassifier()
|
||||
dask_model.set_params(client=client)
|
||||
|
||||
Using Specific Ports
|
||||
********************
|
||||
|
||||
At the beginning of training, ``lightgbm.dask`` sets up a LightGBM network where each Dask worker runs one long-running task that acts as a LightGBM worker. During training, LightGBM workers communicate with each other over TCP sockets. By default, random open ports are used when creating these sockets.
|
||||
|
||||
If the communication between Dask workers in the cluster used for training is restricted by firewall rules, you must tell LightGBM exactly what ports to use.
|
||||
|
||||
**Option 1: provide a specific list of addresses and ports**
|
||||
|
||||
LightGBM supports a parameter ``machines``, a comma-delimited string where each entry refers to one worker (host name or IP) and a port that that worker will accept connections on. If you provide this parameter to the estimators in ``lightgbm.dask``, LightGBM will not search randomly for ports.
|
||||
|
||||
For example, consider the case where you are running one Dask worker process on each of the following IP addresses:
|
||||
|
||||
.. code:: text
|
||||
|
||||
10.0.1.0
|
||||
10.0.2.0
|
||||
10.0.3.0
|
||||
|
||||
You could edit your firewall rules to allow traffic on one additional port on each of these hosts, then provide ``machines`` directly.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import lightgbm as lgb
|
||||
|
||||
machines = "10.0.1.0:12401,10.0.2.0:12402,10.0.3.0:15000"
|
||||
dask_model = lgb.DaskLGBMRegressor(machines=machines)
|
||||
|
||||
If you are running multiple Dask worker processes on physical host in the cluster, be sure that there are multiple entries for that IP address, with different ports. For example, if you were running a cluster with ``nprocs=2`` (2 Dask worker processes per machine), you might open two additional ports on each of these hosts, then provide ``machines`` as follows.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import lightgbm as lgb
|
||||
|
||||
machines = ",".join([
|
||||
"10.0.1.0:16000",
|
||||
"10.0.1.0:16001",
|
||||
"10.0.2.0:16000",
|
||||
"10.0.2.0:16001",
|
||||
])
|
||||
dask_model = lgb.DaskLGBMRegressor(machines=machines)
|
||||
|
||||
.. warning::
|
||||
|
||||
Providing ``machines`` gives you complete control over the networking details of training, but it also makes the training process fragile. Training will fail if you use ``machines`` and any of the following are true:
|
||||
|
||||
* any of the ports mentioned in ``machines`` are not open when training begins
|
||||
* some partitions of the training data are held by machines that that are not present in ``machines``
|
||||
* some machines mentioned in ``machines`` do not hold any of the training data
|
||||
|
||||
**Option 2: specify one port to use on every worker**
|
||||
|
||||
If you are only running one Dask worker process on each host, and if you can reliably identify a port that is open on every host, using ``machines`` is unnecessarily complicated. If ``local_listen_port`` is given and ``machines`` is not, LightGBM will not search for ports randomly, but it will limit the list of addresses in the LightGBM network to those Dask workers that have a piece of the training data.
|
||||
|
||||
For example, consider the case where you are running one Dask worker process on each of the following IP addresses:
|
||||
|
||||
.. code:: text
|
||||
|
||||
10.0.1.0
|
||||
10.0.2.0
|
||||
10.0.3.0
|
||||
|
||||
You could edit your firewall rules to allow communication between any of the workers over one port, then provide that port via parameter ``local_listen_port``.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import lightgbm as lgb
|
||||
|
||||
dask_model = lgb.DaskLGBMRegressor(local_listen_port=12400)
|
||||
|
||||
.. warning::
|
||||
|
||||
Providing ``local_listen_port`` is slightly less fragile than ``machines`` because LightGBM will automatically figure out which workers have pieces of the training data. However, using this method, training can fail if any of the following are true:
|
||||
|
||||
* the port ``local_listen_port`` is not open on any of the worker hosts
|
||||
* any machine has multiple Dask worker processes running on it
|
||||
|
||||
Using Custom Objective Functions with Dask
|
||||
******************************************
|
||||
|
||||
.. versionadded:: 4.0.0
|
||||
|
||||
It is possible to customize the boosting process by providing a custom objective function written in Python.
|
||||
See the Dask API's documentation for details on how to implement such functions.
|
||||
|
||||
.. warning::
|
||||
|
||||
Custom objective functions used with ``lightgbm.dask`` will be called by each worker process on only that worker's local data.
|
||||
|
||||
Follow the example below to use a custom implementation of the ``regression_l2`` objective.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import dask.array as da
|
||||
import lightgbm as lgb
|
||||
import numpy as np
|
||||
from distributed import Client, LocalCluster
|
||||
|
||||
cluster = LocalCluster(n_workers=2)
|
||||
client = Client(cluster)
|
||||
|
||||
X = da.random.random((1000, 10), (500, 10))
|
||||
y = da.random.random((1000,), (500,))
|
||||
|
||||
def custom_l2_obj(y_true, y_pred):
|
||||
grad = y_pred - y_true
|
||||
hess = np.ones(len(y_true))
|
||||
return grad, hess
|
||||
|
||||
dask_model = lgb.DaskLGBMRegressor(
|
||||
objective=custom_l2_obj
|
||||
)
|
||||
dask_model.fit(X, y)
|
||||
|
||||
Prediction with Dask
|
||||
''''''''''''''''''''
|
||||
|
||||
The estimators from ``lightgbm.dask`` can be used to create predictions based on data stored in Dask collections. In that interface, ``.predict()`` expects a Dask Array or Dask DataFrame, and returns a Dask Array of predictions.
|
||||
|
||||
See `the Dask prediction example`_ for some sample code that shows how to perform Dask-based prediction.
|
||||
|
||||
For model evaluation, consider using `the metrics functions from dask-ml`_. Those functions are intended to provide the same API as equivalent functions in ``sklearn.metrics``, but they use distributed computation powered by Dask to compute metrics without all of the input data ever needing to be on a single machine.
|
||||
|
||||
Saving Dask Models
|
||||
''''''''''''''''''
|
||||
|
||||
After training with Dask, you have several options for saving a fitted model.
|
||||
|
||||
**Option 1: pickle the Dask estimator**
|
||||
|
||||
LightGBM's Dask estimators can be pickled directly with ``cloudpickle``, ``joblib``, or ``pickle``.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import dask.array as da
|
||||
import pickle
|
||||
import lightgbm as lgb
|
||||
from distributed import Client, LocalCluster
|
||||
|
||||
cluster = LocalCluster(n_workers=2)
|
||||
client = Client(cluster)
|
||||
|
||||
X = da.random.random((1000, 10), (500, 10))
|
||||
y = da.random.random((1000,), (500,))
|
||||
|
||||
dask_model = lgb.DaskLGBMRegressor()
|
||||
dask_model.fit(X, y)
|
||||
|
||||
with open("dask-model.pkl", "wb") as f:
|
||||
pickle.dump(dask_model, f)
|
||||
|
||||
A model saved this way can then later be loaded with whichever serialization library you used to save it.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import pickle
|
||||
with open("dask-model.pkl", "rb") as f:
|
||||
dask_model = pickle.load(f)
|
||||
|
||||
.. note::
|
||||
|
||||
If you explicitly set a Dask client (see `Using a Specific Dask Client <#using-a-specific-dask-client>`__), it will not be saved when pickling the estimator. When loading a Dask estimator from disk, if you need to use a specific client you can add it after loading with ``dask_model.set_params(client=client)``.
|
||||
|
||||
**Option 2: pickle the sklearn estimator**
|
||||
|
||||
The estimators available from ``lightgbm.dask`` can be converted to an instance of the equivalent class from ``lightgbm.sklearn``. Choosing this option allows you to use Dask for training but avoid depending on any Dask libraries at scoring time.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import dask.array as da
|
||||
import joblib
|
||||
import lightgbm as lgb
|
||||
from distributed import Client, LocalCluster
|
||||
|
||||
cluster = LocalCluster(n_workers=2)
|
||||
client = Client(cluster)
|
||||
|
||||
X = da.random.random((1000, 10), (500, 10))
|
||||
y = da.random.random((1000,), (500,))
|
||||
|
||||
dask_model = lgb.DaskLGBMRegressor()
|
||||
dask_model.fit(X, y)
|
||||
|
||||
# convert to sklearn equivalent
|
||||
sklearn_model = dask_model.to_local()
|
||||
|
||||
print(type(sklearn_model))
|
||||
#> lightgbm.sklearn.LGBMRegressor
|
||||
|
||||
joblib.dump(sklearn_model, "sklearn-model.joblib")
|
||||
|
||||
A model saved this way can then later be loaded with whichever serialization library you used to save it.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import joblib
|
||||
|
||||
sklearn_model = joblib.load("sklearn-model.joblib")
|
||||
|
||||
**Option 3: save the LightGBM Booster**
|
||||
|
||||
The lowest-level model object in LightGBM is the ``lightgbm.Booster``. After training, you can extract a Booster from the Dask estimator.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import dask.array as da
|
||||
import lightgbm as lgb
|
||||
from distributed import Client, LocalCluster
|
||||
|
||||
cluster = LocalCluster(n_workers=2)
|
||||
client = Client(cluster)
|
||||
|
||||
X = da.random.random((1000, 10), (500, 10))
|
||||
y = da.random.random((1000,), (500,))
|
||||
|
||||
dask_model = lgb.DaskLGBMRegressor()
|
||||
dask_model.fit(X, y)
|
||||
|
||||
# get underlying Booster object
|
||||
bst = dask_model.booster_
|
||||
|
||||
From the point forward, you can use any of the following methods to save the Booster:
|
||||
|
||||
* serialize with ``cloudpickle``, ``joblib``, or ``pickle``
|
||||
* ``bst.dump_model()``: dump the model to a dictionary which could be written out as JSON
|
||||
* ``bst.model_to_string()``: dump the model to a string in memory
|
||||
* ``bst.save_model()``: write the output of ``bst.model_to_string()`` to a text file
|
||||
|
||||
Kubeflow
|
||||
^^^^^^^^
|
||||
|
||||
Kubeflow users can also use the `Kubeflow XGBoost Operator`_ for machine learning workflows with LightGBM. You can see `this example`_ for more details.
|
||||
|
||||
Kubeflow integrations for LightGBM are not maintained by LightGBM's maintainers.
|
||||
|
||||
.. note::
|
||||
|
||||
The Kubeflow integrations for LightGBM are not maintained by LightGBM's maintainers. Bug reports or feature requests should be directed to https://github.com/kubeflow/fairing/issues or https://github.com/kubeflow/xgboost-operator/issues.
|
||||
|
||||
LightGBM CLI
|
||||
^^^^^^^^^^^^
|
||||
|
||||
.. _Build Parallel Version:
|
||||
|
||||
Preparation
|
||||
'''''''''''
|
||||
|
||||
By default, distributed learning with LightGBM uses socket-based communication.
|
||||
|
||||
If you need to build distributed version with MPI support, please refer to `Installation Guide <./Installation-Guide.rst#build-mpi-version>`__.
|
||||
|
||||
Socket Version
|
||||
**************
|
||||
|
||||
It needs to collect IP of all machines that want to run distributed learning in and allocate one TCP port (assume 12345 here) for all machines,
|
||||
and change firewall rules to allow income of this port (12345). Then write these IP and ports in one file (assume ``mlist.txt``), like following:
|
||||
|
||||
.. code:: text
|
||||
|
||||
machine1_ip 12345
|
||||
machine2_ip 12345
|
||||
|
||||
MPI Version
|
||||
***********
|
||||
|
||||
It needs to collect IP (or hostname) of all machines that want to run distributed learning in.
|
||||
Then write these IP in one file (assume ``mlist.txt``) like following:
|
||||
|
||||
.. code:: text
|
||||
|
||||
machine1_ip
|
||||
machine2_ip
|
||||
|
||||
**Note**: For Windows users, need to start "smpd" to start MPI service. More details can be found `here`_.
|
||||
|
||||
Run Distributed Learning
|
||||
''''''''''''''''''''''''
|
||||
|
||||
.. _Run Parallel Learning:
|
||||
|
||||
Socket Version
|
||||
**************
|
||||
|
||||
1. Edit following parameters in config file:
|
||||
|
||||
``tree_learner=your_parallel_algorithm``, edit ``your_parallel_algorithm`` (e.g. feature/data) here.
|
||||
|
||||
``num_machines=your_num_machines``, edit ``your_num_machines`` (e.g. 4) here.
|
||||
|
||||
``machine_list_file=mlist.txt``, ``mlist.txt`` is created in `Preparation section <#preparation>`__.
|
||||
|
||||
``local_listen_port=12345``, ``12345`` is allocated in `Preparation section <#preparation>`__.
|
||||
|
||||
2. Copy data file, executable file, config file and ``mlist.txt`` to all machines.
|
||||
|
||||
3. Run following command on all machines, you need to change ``your_config_file`` to real config file.
|
||||
|
||||
For Windows: ``lightgbm.exe config=your_config_file``
|
||||
|
||||
For Linux: ``./lightgbm config=your_config_file``
|
||||
|
||||
MPI Version
|
||||
***********
|
||||
|
||||
1. Edit following parameters in config file:
|
||||
|
||||
``tree_learner=your_parallel_algorithm``, edit ``your_parallel_algorithm`` (e.g. feature/data) here.
|
||||
|
||||
``num_machines=your_num_machines``, edit ``your_num_machines`` (e.g. 4) here.
|
||||
|
||||
2. Copy data file, executable file, config file and ``mlist.txt`` to all machines.
|
||||
|
||||
**Note**: MPI needs to be run in the **same path on all machines**.
|
||||
|
||||
3. Run following command on one machine (not need to run on all machines), need to change ``your_config_file`` to real config file.
|
||||
|
||||
For Windows:
|
||||
|
||||
.. code:: console
|
||||
|
||||
mpiexec.exe /machinefile mlist.txt lightgbm.exe config=your_config_file
|
||||
|
||||
For Linux:
|
||||
|
||||
.. code:: console
|
||||
|
||||
mpiexec --machinefile mlist.txt ./lightgbm config=your_config_file
|
||||
|
||||
Example
|
||||
'''''''
|
||||
|
||||
- `A simple distributed learning example`_
|
||||
|
||||
Ray
|
||||
^^^
|
||||
|
||||
`Ray`_ is a Python-based framework for distributed computing. Ray provides LightGBM support through the Ray Train API with ``LightGBMTrainer`` and the `lightgbm_ray`_ project maintained within the official Ray GitHub organization.
|
||||
|
||||
For the Ray Train API, see `the Ray documentation`_ for usage examples.
|
||||
|
||||
For the lightgbm_ray project, see `the lightgbm_ray documentation`_ for usage examples.
|
||||
|
||||
.. note::
|
||||
|
||||
``lightgbm_ray`` and ``ray`` are not maintained by LightGBM's maintainers. Bug reports or feature requests should be directed to https://github.com/ray-project/lightgbm_ray/issues and https://github.com/ray-project/ray/issues respectively.
|
||||
|
||||
Mars
|
||||
^^^^
|
||||
|
||||
`Mars`_ is a tensor-based framework for large-scale data computation. LightGBM integration, maintained within the Mars GitHub repository, can be used to perform distributed LightGBM training using ``pymars``.
|
||||
|
||||
See `the mars documentation`_ for usage examples.
|
||||
|
||||
.. note::
|
||||
|
||||
``Mars`` is not maintained by LightGBM's maintainers. Bug reports or feature requests should be directed to https://github.com/mars-project/mars/issues.
|
||||
|
||||
.. _Dask: https://docs.dask.org/en/latest/
|
||||
|
||||
.. _SynapseML: https://aka.ms/spark
|
||||
|
||||
.. _this SynapseML example: https://github.com/microsoft/SynapseML/tree/master/docs/Explore%20Algorithms/LightGBM
|
||||
|
||||
.. _the Dask Array documentation: https://docs.dask.org/en/latest/array.html
|
||||
|
||||
.. _the Dask DataFrame documentation: https://docs.dask.org/en/latest/dataframe.html
|
||||
|
||||
.. _the Dask prediction example: https://github.com/lightgbm-org/LightGBM/blob/main/examples/python-guide/dask/prediction.py
|
||||
|
||||
.. _the Dask worker documentation: https://distributed.dask.org/en/stable/worker-memory.html
|
||||
|
||||
.. _the metrics functions from dask-ml: https://ml.dask.org/modules/api.html#dask-ml-metrics-metrics
|
||||
|
||||
.. _these Dask examples: https://github.com/lightgbm-org/LightGBM/tree/main/examples/python-guide/dask
|
||||
|
||||
.. _Kubeflow XGBoost Operator: https://github.com/kubeflow/xgboost-operator
|
||||
|
||||
.. _this example: https://github.com/kubeflow/xgboost-operator/tree/master/config/samples/lightgbm-dist
|
||||
|
||||
.. _here: https://www.youtube.com/watch?v=iqzXhp5TxUY
|
||||
|
||||
.. _A simple distributed learning example: https://github.com/lightgbm-org/LightGBM/tree/main/examples/parallel_learning
|
||||
|
||||
.. _lightgbm_ray: https://github.com/ray-project/lightgbm_ray
|
||||
|
||||
.. _Ray: https://www.ray.io/
|
||||
|
||||
.. _the lightgbm_ray documentation: https://docs.ray.io/en/latest/tune/api_docs/integration.html#lightgbm-tune-integration-lightgbm
|
||||
|
||||
.. _the Ray documentation: https://docs.ray.io/en/latest/train/api/api.html#lightgbm
|
||||
|
||||
.. _Mars: https://mars-project.readthedocs.io/en/latest/
|
||||
|
||||
.. _the mars documentation: https://mars-project.readthedocs.io/en/latest/user_guide/learn/lightgbm.html
|
||||
@@ -0,0 +1,221 @@
|
||||
Parameters Tuning
|
||||
=================
|
||||
|
||||
This page contains parameters tuning guides for different scenarios.
|
||||
|
||||
**List of other helpful links**
|
||||
|
||||
- `Parameters <./Parameters.rst>`__
|
||||
- `Python API <./Python-API.rst>`__
|
||||
- `FLAML`_ for automated hyperparameter tuning
|
||||
- `Optuna`_ for automated hyperparameter tuning
|
||||
|
||||
Tune Parameters for the Leaf-wise (Best-first) Tree
|
||||
---------------------------------------------------
|
||||
|
||||
LightGBM uses the `leaf-wise <./Features.rst#leaf-wise-best-first-tree-growth>`__ tree growth algorithm, while many other popular tools use depth-wise tree growth.
|
||||
Compared with depth-wise growth, the leaf-wise algorithm can converge much faster.
|
||||
However, the leaf-wise growth may be over-fitting if not used with the appropriate parameters.
|
||||
|
||||
To get good results using a leaf-wise tree, these are some important parameters:
|
||||
|
||||
1. ``num_leaves``. This is the main parameter to control the complexity of the tree model.
|
||||
Theoretically, we can set ``num_leaves = 2^(max_depth)`` to obtain the same number of leaves as depth-wise tree.
|
||||
However, this simple conversion is not good in practice.
|
||||
A leaf-wise tree is typically much deeper than a depth-wise tree for a fixed number of leaves. Unconstrained depth can induce over-fitting.
|
||||
Thus, when trying to tune the ``num_leaves``, we should let it be smaller than ``2^(max_depth)``.
|
||||
For example, when the ``max_depth=7`` the depth-wise tree can get good accuracy,
|
||||
but setting ``num_leaves`` to ``127`` may cause over-fitting, and setting it to ``70`` or ``80`` may get better accuracy than depth-wise.
|
||||
|
||||
2. ``min_data_in_leaf``. This is a very important parameter to prevent over-fitting in a leaf-wise tree.
|
||||
Its optimal value depends on the number of training samples and ``num_leaves``.
|
||||
Setting it to a large value can avoid growing too deep a tree, but may cause under-fitting.
|
||||
In practice, setting it to hundreds or thousands is enough for a large dataset.
|
||||
|
||||
3. ``max_depth``. You also can use ``max_depth`` to limit the tree depth explicitly.
|
||||
If you set ``max_depth``, also explicitly set ``num_leaves`` to some value ``<= 2^max_depth``.
|
||||
|
||||
For Faster Speed
|
||||
----------------
|
||||
|
||||
Add More Computational Resources
|
||||
''''''''''''''''''''''''''''''''
|
||||
|
||||
On systems where it is available, LightGBM uses OpenMP to parallelize many operations. The maximum number of threads used by LightGBM is controlled by the parameter ``num_threads``. By default, this will defer to the default behavior of OpenMP (one thread per real CPU core or the value in environment variable ``OMP_NUM_THREADS``, if it is set). For best performance, set this to the number of **real** CPU cores available.
|
||||
|
||||
You might be able to achieve faster training by moving to a machine with more available CPU cores.
|
||||
|
||||
Using distributed (multi-machine) training might also reduce training time. See the `Distributed Learning Guide <./Parallel-Learning-Guide.rst>`_ for details.
|
||||
|
||||
Use a GPU-enabled version of LightGBM
|
||||
'''''''''''''''''''''''''''''''''''''
|
||||
|
||||
You might find that training is faster using a GPU-enabled build of LightGBM. See the `GPU Tutorial <./GPU-Tutorial.rst>`__ for details.
|
||||
|
||||
Grow Shallower Trees
|
||||
''''''''''''''''''''
|
||||
|
||||
The total training time for LightGBM increases with the total number of tree nodes added. LightGBM comes with several parameters that can be used to control the number of nodes per tree.
|
||||
|
||||
The suggestions below will speed up training, but might hurt training accuracy.
|
||||
|
||||
Decrease ``max_depth``
|
||||
**********************
|
||||
|
||||
This parameter is an integer that controls the maximum distance between the root node of each tree and a leaf node. Decrease ``max_depth`` to reduce training time.
|
||||
|
||||
Decrease ``num_leaves``
|
||||
***********************
|
||||
|
||||
LightGBM adds nodes to trees based on the gain from adding that node, regardless of depth. This figure from `the feature documentation <./Features.rst#leaf-wise-best-first-tree-growth>`__ illustrates the process.
|
||||
|
||||
.. image:: ./_static/images/leaf-wise.png
|
||||
:align: center
|
||||
:alt: Three consecutive images of decision trees, where each shows the tree with an additional two leaf nodes added. Shows that leaf-wise growth can result in trees that have some branches which are longer than others.
|
||||
|
||||
Because of this growth strategy, it isn't straightforward to use ``max_depth`` alone to limit the complexity of trees. The ``num_leaves`` parameter sets the maximum number of nodes per tree. Decrease ``num_leaves`` to reduce training time.
|
||||
|
||||
Increase ``min_gain_to_split``
|
||||
******************************
|
||||
|
||||
When adding a new tree node, LightGBM chooses the split point that has the largest gain. Gain is basically the reduction in training loss that results from adding a split point. By default, LightGBM sets ``min_gain_to_split`` to 0.0, which means "there is no improvement that is too small". However, in practice you might find that very small improvements in the training loss don't have a meaningful impact on the generalization error of the model. Increase ``min_gain_to_split`` to reduce training time.
|
||||
|
||||
Increase ``min_data_in_leaf`` and ``min_sum_hessian_in_leaf``
|
||||
*************************************************************
|
||||
|
||||
Depending on the size of the training data and the distribution of features, it's possible for LightGBM to add tree nodes that only describe a small number of observations. In the most extreme case, consider the addition of a tree node that only a single observation from the training data falls into. This is very unlikely to generalize well, and probably is a sign of overfitting.
|
||||
|
||||
This can be prevented indirectly with parameters like ``max_depth`` and ``num_leaves``, but LightGBM also offers parameters to help you directly avoid adding these overly-specific tree nodes.
|
||||
|
||||
- ``min_data_in_leaf``: Minimum number of observations that must fall into a tree node for it to be added.
|
||||
- ``min_sum_hessian_in_leaf``: Minimum sum of the Hessian (second derivative of the objective function evaluated for each observation) for observations in a leaf. For some regression objectives, this is just the minimum number of records that have to fall into each node. For classification objectives, it represents a sum over a distribution of probabilities. See `this Stack Overflow answer <https://stats.stackexchange.com/questions/317073/explanation-of-min-child-weight-in-xgboost-algorithm>`_ for a good description of how to reason about values of this parameter.
|
||||
|
||||
Grow Less Trees
|
||||
'''''''''''''''
|
||||
|
||||
Decrease ``num_iterations``
|
||||
***************************
|
||||
|
||||
The ``num_iterations`` parameter controls the number of boosting rounds that will be performed. Since LightGBM uses decision trees as the learners, this can also be thought of as "number of trees".
|
||||
|
||||
If you try changing ``num_iterations``, change the ``learning_rate`` as well. ``learning_rate`` will not have any impact on training time, but it will impact the training accuracy. As a general rule, if you reduce ``num_iterations``, you should increase ``learning_rate``.
|
||||
|
||||
Choosing the right value of ``num_iterations`` and ``learning_rate`` is highly dependent on the data and objective, so these parameters are often chosen from a set of possible values through hyperparameter tuning.
|
||||
|
||||
Decrease ``num_iterations`` to reduce training time.
|
||||
|
||||
Use Early Stopping
|
||||
******************
|
||||
|
||||
If early stopping is enabled, after each boosting round the model's training accuracy is evaluated against a validation set that contains data not available to the training process. That accuracy is then compared to the accuracy as of the previous boosting round. If the model's accuracy fails to improve for some number of consecutive rounds, LightGBM stops the training process.
|
||||
|
||||
That "number of consecutive rounds" is controlled by the parameter ``early_stopping_round``. For example, ``early_stopping_round=1`` says "the first time accuracy on the validation set does not improve, stop training".
|
||||
|
||||
Set ``early_stopping_round`` and provide a validation set to possibly reduce training time.
|
||||
|
||||
Consider Fewer Splits
|
||||
'''''''''''''''''''''
|
||||
|
||||
The parameters described in previous sections control how many trees are constructed and how many nodes are constructed per tree. Training time can be further reduced by reducing the amount of time needed to add a tree node to the model.
|
||||
|
||||
The suggestions below will speed up training, but might hurt training accuracy.
|
||||
|
||||
Enable Feature Pre-Filtering When Creating Dataset
|
||||
**************************************************
|
||||
|
||||
By default, when a LightGBM ``Dataset`` object is constructed, some features will be filtered out based on the value of ``min_data_in_leaf``.
|
||||
|
||||
For a simple example, consider a 1000-observation dataset with a feature called ``feature_1``. ``feature_1`` takes on only two values: 25.0 (995 observations) and 50.0 (5 observations). If ``min_data_in_leaf = 10``, there is no split for this feature which will result in a valid split at least one of the leaf nodes will only have 5 observations.
|
||||
|
||||
Instead of reconsidering this feature and then ignoring it every iteration, LightGBM filters this feature out at before training, when the ``Dataset`` is constructed.
|
||||
|
||||
If this default behavior has been overridden by setting ``feature_pre_filter=False``, set ``feature_pre_filter=True`` to reduce training time.
|
||||
|
||||
Decrease ``max_bin`` or ``max_bin_by_feature`` When Creating Dataset
|
||||
********************************************************************
|
||||
|
||||
LightGBM training `buckets continuous features into discrete bins <./Features.rst#optimization-in-speed-and-memory-usage>`_ to improve training speed and reduce memory requirements for training. This binning is done one time during ``Dataset`` construction. The number of splits considered when adding a node is ``O(#feature * #bin)``, so reducing the number of bins per feature can reduce the number of splits that need to be evaluated.
|
||||
|
||||
``max_bin`` is controls the maximum number of bins that features will bucketed into. It is also possible to set this maximum feature-by-feature, by passing ``max_bin_by_feature``.
|
||||
|
||||
Reduce ``max_bin`` or ``max_bin_by_feature`` to reduce training time.
|
||||
|
||||
Increase ``min_data_in_bin`` When Creating Dataset
|
||||
**************************************************
|
||||
|
||||
Some bins might contain a small number of observations, which might mean that the effort of evaluating that bin's boundaries as possible split points isn't likely to change the final model very much. You can control the granularity of the bins by setting ``min_data_in_bin``.
|
||||
|
||||
Increase ``min_data_in_bin`` to reduce training time.
|
||||
|
||||
Decrease ``feature_fraction``
|
||||
*****************************
|
||||
|
||||
By default, LightGBM considers all features in a ``Dataset`` during the training process. This behavior can be changed by setting ``feature_fraction`` to a value ``> 0`` and ``<= 1.0``. Setting ``feature_fraction`` to ``0.5``, for example, tells LightGBM to randomly select ``50%`` of features at the beginning of constructing each tree. This reduces the total number of splits that have to be evaluated to add each tree node.
|
||||
|
||||
Decrease ``feature_fraction`` to reduce training time.
|
||||
|
||||
Decrease ``max_cat_threshold``
|
||||
******************************
|
||||
|
||||
LightGBM uses a `custom approach for finding optimal splits for categorical features <./Advanced-Topics.html#categorical-feature-support>`_. In this process, LightGBM explores splits that break a categorical feature into two groups. These are sometimes called "k-vs.-rest" splits. Higher ``max_cat_threshold`` values correspond to more split points and larger possible group sizes to search.
|
||||
|
||||
Decrease ``max_cat_threshold`` to reduce training time.
|
||||
|
||||
Use Less Data
|
||||
'''''''''''''
|
||||
|
||||
Use Bagging
|
||||
***********
|
||||
|
||||
By default, LightGBM uses all observations in the training data for each iteration. It is possible to instead tell LightGBM to randomly sample the training data. This process of training over multiple random samples without replacement is called "bagging".
|
||||
|
||||
Set ``bagging_freq`` to an integer greater than 0 to control how often a new sample is drawn. Set ``bagging_fraction`` to a value ``> 0.0`` and ``< 1.0`` to control the size of the sample. For example, ``{"bagging_freq": 5, "bagging_fraction": 0.75}`` tells LightGBM "re-sample without replacement every 5 iterations, and draw samples of 75% of the training data".
|
||||
|
||||
Decrease ``bagging_fraction`` to reduce training time.
|
||||
|
||||
|
||||
Save Constructed Datasets with ``save_binary``
|
||||
''''''''''''''''''''''''''''''''''''''''''''''
|
||||
|
||||
This only applies to the LightGBM CLI. If you pass parameter ``save_binary``, the training dataset and all validations sets will be saved in a binary format understood by LightGBM. This can speed up training next time, because binning and other work done when constructing a ``Dataset`` does not have to be re-done.
|
||||
|
||||
|
||||
For Better Accuracy
|
||||
-------------------
|
||||
|
||||
- Use large ``max_bin`` (may be slower)
|
||||
|
||||
- Use small ``learning_rate`` with large ``num_iterations``
|
||||
|
||||
- Use large ``num_leaves`` (may cause over-fitting)
|
||||
|
||||
- Use bigger training data
|
||||
|
||||
- Try ``dart``
|
||||
|
||||
Deal with Over-fitting
|
||||
----------------------
|
||||
|
||||
- Use small ``max_bin``
|
||||
|
||||
- Use small ``num_leaves``
|
||||
|
||||
- Use ``min_data_in_leaf`` and ``min_sum_hessian_in_leaf``
|
||||
|
||||
- Use bagging by set ``bagging_fraction`` and ``bagging_freq``
|
||||
|
||||
- Use feature sub-sampling by set ``feature_fraction``
|
||||
|
||||
- Use bigger training data
|
||||
|
||||
- Try ``lambda_l1``, ``lambda_l2`` and ``min_gain_to_split`` for regularization
|
||||
|
||||
- Try ``max_depth`` to avoid growing deep tree
|
||||
|
||||
- Try ``extra_trees``
|
||||
|
||||
- Try increasing ``path_smooth``
|
||||
|
||||
.. _Optuna: https://medium.com/optuna/lightgbm-tuner-new-optuna-integration-for-hyperparameter-optimization-8b7095e99258
|
||||
|
||||
.. _FLAML: https://github.com/microsoft/FLAML
|
||||
+1483
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,78 @@
|
||||
Python API
|
||||
==========
|
||||
|
||||
.. currentmodule:: lightgbm
|
||||
|
||||
Data Structure API
|
||||
------------------
|
||||
|
||||
.. autosummary::
|
||||
:toctree: pythonapi/
|
||||
|
||||
Dataset
|
||||
Booster
|
||||
CVBooster
|
||||
Sequence
|
||||
|
||||
Training API
|
||||
------------
|
||||
|
||||
.. autosummary::
|
||||
:toctree: pythonapi/
|
||||
|
||||
train
|
||||
cv
|
||||
|
||||
Scikit-learn API
|
||||
----------------
|
||||
|
||||
.. autosummary::
|
||||
:toctree: pythonapi/
|
||||
|
||||
LGBMModel
|
||||
LGBMClassifier
|
||||
LGBMRegressor
|
||||
LGBMRanker
|
||||
|
||||
Dask API
|
||||
--------
|
||||
|
||||
.. versionadded:: 3.2.0
|
||||
|
||||
.. autosummary::
|
||||
:toctree: pythonapi/
|
||||
|
||||
DaskLGBMClassifier
|
||||
DaskLGBMRegressor
|
||||
DaskLGBMRanker
|
||||
|
||||
Callbacks
|
||||
---------
|
||||
|
||||
.. autosummary::
|
||||
:toctree: pythonapi/
|
||||
|
||||
early_stopping
|
||||
log_evaluation
|
||||
record_evaluation
|
||||
reset_parameter
|
||||
|
||||
Plotting
|
||||
--------
|
||||
|
||||
.. autosummary::
|
||||
:toctree: pythonapi/
|
||||
|
||||
plot_importance
|
||||
plot_split_value_histogram
|
||||
plot_metric
|
||||
plot_tree
|
||||
create_tree_digraph
|
||||
|
||||
Utilities
|
||||
---------
|
||||
|
||||
.. autosummary::
|
||||
:toctree: pythonapi/
|
||||
|
||||
register_logger
|
||||
@@ -0,0 +1,267 @@
|
||||
Python-package Introduction
|
||||
===========================
|
||||
|
||||
This document gives a basic walk-through of LightGBM Python-package.
|
||||
|
||||
**List of other helpful links**
|
||||
|
||||
- `Python Examples <https://github.com/lightgbm-org/LightGBM/tree/main/examples/python-guide>`__
|
||||
|
||||
- `Python API <./Python-API.rst>`__
|
||||
|
||||
- `Parameters Tuning <./Parameters-Tuning.rst>`__
|
||||
|
||||
Install
|
||||
-------
|
||||
|
||||
The preferred way to install LightGBM is via pip:
|
||||
|
||||
::
|
||||
|
||||
pip install lightgbm
|
||||
|
||||
Refer to `Python-package`_ folder for the detailed installation guide.
|
||||
|
||||
To verify your installation, try to ``import lightgbm`` in Python:
|
||||
|
||||
::
|
||||
|
||||
import lightgbm as lgb
|
||||
|
||||
Data Interface
|
||||
--------------
|
||||
|
||||
The LightGBM Python module can load data from:
|
||||
|
||||
- LibSVM (zero-based) / TSV / CSV format text file
|
||||
|
||||
- NumPy 2D array(s), SciPy sparse matrix
|
||||
|
||||
- pandas DataFrame, polars DataFrame, pyarrow Table
|
||||
|
||||
- LightGBM binary file
|
||||
|
||||
- LightGBM ``Sequence`` object(s)
|
||||
|
||||
The data is stored in a ``Dataset`` object.
|
||||
|
||||
Many of the examples in this page use functionality from ``numpy``. To run the examples, be sure to import ``numpy`` in your session.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import numpy as np
|
||||
|
||||
**To load a LibSVM (zero-based) text file or a LightGBM binary file into Dataset:**
|
||||
|
||||
.. code:: python
|
||||
|
||||
train_data = lgb.Dataset('train.svm.bin')
|
||||
|
||||
**To load a numpy array into Dataset:**
|
||||
|
||||
.. code:: python
|
||||
|
||||
rng = np.random.default_rng()
|
||||
data = rng.uniform(size=(500, 10)) # 500 entities, each contains 10 features
|
||||
label = rng.integers(low=0, high=2, size=(500, )) # binary target
|
||||
train_data = lgb.Dataset(data, label=label)
|
||||
|
||||
**To load a scipy.sparse.csr\_matrix array into Dataset:**
|
||||
|
||||
.. code:: python
|
||||
|
||||
import scipy
|
||||
csr = scipy.sparse.csr_matrix((dat, (row, col)))
|
||||
train_data = lgb.Dataset(csr)
|
||||
|
||||
**Load from Sequence objects:**
|
||||
|
||||
We can implement ``Sequence`` interface to read binary files. The following example shows reading HDF5 file with ``h5py``.
|
||||
|
||||
.. code:: python
|
||||
|
||||
import h5py
|
||||
|
||||
class HDFSequence(lgb.Sequence):
|
||||
def __init__(self, hdf_dataset, batch_size):
|
||||
self.data = hdf_dataset
|
||||
self.batch_size = batch_size
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.data[idx]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data)
|
||||
|
||||
f = h5py.File('train.hdf5', 'r')
|
||||
train_data = lgb.Dataset(HDFSequence(f['X'], 8192), label=f['Y'][:])
|
||||
|
||||
Features of using ``Sequence`` interface:
|
||||
|
||||
- Data sampling uses random access, thus does not go through the whole dataset
|
||||
- Reading data in batch, thus saves memory when constructing ``Dataset`` object
|
||||
- Supports creating ``Dataset`` from multiple data files
|
||||
|
||||
Please refer to ``Sequence`` `API doc <./Python-API.rst#data-structure-api>`__.
|
||||
|
||||
`dataset_from_multi_hdf5.py <https://github.com/lightgbm-org/LightGBM/blob/main/examples/python-guide/dataset_from_multi_hdf5.py>`__ is a detailed example.
|
||||
|
||||
**Saving Dataset into a LightGBM binary file will make loading faster:**
|
||||
|
||||
.. code:: python
|
||||
|
||||
train_data = lgb.Dataset('train.svm.txt')
|
||||
train_data.save_binary('train.bin')
|
||||
|
||||
**Create validation data:**
|
||||
|
||||
.. code:: python
|
||||
|
||||
validation_data = train_data.create_valid('validation.svm')
|
||||
|
||||
or
|
||||
|
||||
.. code:: python
|
||||
|
||||
validation_data = lgb.Dataset('validation.svm', reference=train_data)
|
||||
|
||||
In LightGBM, the validation data should be aligned with training data.
|
||||
|
||||
**Specific feature names and categorical features:**
|
||||
|
||||
.. code:: python
|
||||
|
||||
train_data = lgb.Dataset(data, label=label, feature_name=['c1', 'c2', 'c3'], categorical_feature=['c3'])
|
||||
|
||||
LightGBM can use categorical features as input directly.
|
||||
It doesn't need to convert to one-hot encoding, and is much faster than one-hot encoding (about 8x speed-up).
|
||||
|
||||
**Note**: You should convert your categorical features to ``int`` type before you construct ``Dataset``.
|
||||
|
||||
**Weights can be set when needed:**
|
||||
|
||||
.. code:: python
|
||||
|
||||
rng = np.random.default_rng()
|
||||
w = rng.uniform(size=(500, ))
|
||||
train_data = lgb.Dataset(data, label=label, weight=w)
|
||||
|
||||
or
|
||||
|
||||
.. code:: python
|
||||
|
||||
train_data = lgb.Dataset(data, label=label)
|
||||
rng = np.random.default_rng()
|
||||
w = rng.uniform(size=(500, ))
|
||||
train_data.set_weight(w)
|
||||
|
||||
And you can use ``Dataset.set_init_score()`` to set initial score, and ``Dataset.set_group()`` to set group/query data for ranking tasks.
|
||||
|
||||
**Memory efficient usage:**
|
||||
|
||||
The ``Dataset`` object in LightGBM is very memory-efficient, it only needs to save discrete bins.
|
||||
However, Numpy/Array/Pandas object is memory expensive.
|
||||
If you are concerned about your memory consumption, you can save memory by:
|
||||
|
||||
1. Set ``free_raw_data=True`` (default is ``True``) when constructing the ``Dataset``
|
||||
|
||||
2. Explicitly set ``raw_data=None`` after the ``Dataset`` has been constructed
|
||||
|
||||
3. Call ``gc``
|
||||
|
||||
Setting Parameters
|
||||
------------------
|
||||
|
||||
LightGBM can use a dictionary to set `Parameters <./Parameters.rst>`__.
|
||||
For instance:
|
||||
|
||||
- Booster parameters:
|
||||
|
||||
.. code:: python
|
||||
|
||||
param = {'num_leaves': 31, 'objective': 'binary'}
|
||||
param['metric'] = 'auc'
|
||||
|
||||
- You can also specify multiple eval metrics:
|
||||
|
||||
.. code:: python
|
||||
|
||||
param['metric'] = ['auc', 'binary_logloss']
|
||||
|
||||
Training
|
||||
--------
|
||||
|
||||
Training a model requires a parameter list and data set:
|
||||
|
||||
.. code:: python
|
||||
|
||||
num_round = 10
|
||||
bst = lgb.train(param, train_data, num_round, valid_sets=[validation_data])
|
||||
|
||||
After training, the model can be saved:
|
||||
|
||||
.. code:: python
|
||||
|
||||
bst.save_model('model.txt')
|
||||
|
||||
The trained model can also be dumped to JSON format:
|
||||
|
||||
.. code:: python
|
||||
|
||||
json_model = bst.dump_model()
|
||||
|
||||
A saved model can be loaded:
|
||||
|
||||
.. code:: python
|
||||
|
||||
bst = lgb.Booster(model_file='model.txt') # init model
|
||||
|
||||
CV
|
||||
--
|
||||
|
||||
Training with 5-fold CV:
|
||||
|
||||
.. code:: python
|
||||
|
||||
lgb.cv(param, train_data, num_round, nfold=5)
|
||||
|
||||
Early Stopping
|
||||
--------------
|
||||
|
||||
If you have a validation set, you can use early stopping to find the optimal number of boosting rounds.
|
||||
Early stopping requires at least one set in ``valid_sets``. If there is more than one, it will use all of them except the training data:
|
||||
|
||||
.. code:: python
|
||||
|
||||
bst = lgb.train(param, train_data, num_round, valid_sets=valid_sets, callbacks=[lgb.early_stopping(stopping_rounds=5)])
|
||||
bst.save_model('model.txt', num_iteration=bst.best_iteration)
|
||||
|
||||
The model will train until the validation score stops improving.
|
||||
Validation score needs to improve at least every ``stopping_rounds`` to continue training.
|
||||
|
||||
The index of iteration that has the best performance will be saved in the ``best_iteration`` field if early stopping logic is enabled by setting ``early_stopping`` callback.
|
||||
Note that ``train()`` will return a model from the best iteration.
|
||||
|
||||
This works with both metrics to minimize (L2, log loss, etc.) and to maximize (NDCG, AUC, etc.).
|
||||
Note that if you specify more than one evaluation metric, all of them will be used for early stopping.
|
||||
However, you can change this behavior and make LightGBM check only the first metric for early stopping by passing ``first_metric_only=True`` in ``early_stopping`` callback constructor.
|
||||
|
||||
Prediction
|
||||
----------
|
||||
|
||||
A model that has been trained or loaded can perform predictions on datasets:
|
||||
|
||||
.. code:: python
|
||||
|
||||
# 7 entities, each contains 10 features
|
||||
rng = np.random.default_rng()
|
||||
data = rng.uniform(size=(7, 10))
|
||||
ypred = bst.predict(data)
|
||||
|
||||
If early stopping is enabled during training, you can get predictions from the best iteration with ``bst.best_iteration``:
|
||||
|
||||
.. code:: python
|
||||
|
||||
ypred = bst.predict(data, num_iteration=bst.best_iteration)
|
||||
|
||||
.. _Python-package: https://github.com/lightgbm-org/LightGBM/tree/main/python-package
|
||||
@@ -0,0 +1,88 @@
|
||||
Quick Start
|
||||
===========
|
||||
|
||||
This is a quick start guide for LightGBM CLI version.
|
||||
|
||||
Follow the `Installation Guide <./Installation-Guide.rst>`__ to install LightGBM first.
|
||||
|
||||
**List of other helpful links**
|
||||
|
||||
- `Parameters <./Parameters.rst>`__
|
||||
|
||||
- `Parameters Tuning <./Parameters-Tuning.rst>`__
|
||||
|
||||
- `Python-package Quick Start <./Python-Intro.rst>`__
|
||||
|
||||
- `Python API <./Python-API.rst>`__
|
||||
|
||||
Training Data Format
|
||||
--------------------
|
||||
|
||||
LightGBM supports input data files with `CSV`_, `TSV`_ and `LibSVM`_ (zero-based) formats.
|
||||
|
||||
Files could be both with and without `headers <./Parameters.rst#header>`__.
|
||||
|
||||
`Label column <./Parameters.rst#label_column>`__ could be specified both by index and by name.
|
||||
|
||||
Some columns could be `ignored <./Parameters.rst#ignore_column>`__.
|
||||
|
||||
Categorical Feature Support
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
LightGBM can use categorical features directly (without one-hot encoding).
|
||||
The experiment on `Expo data`_ shows about 8x speed-up compared with one-hot encoding.
|
||||
|
||||
For the setting details, please refer to the ``categorical_feature`` `parameter <./Parameters.rst#categorical_feature>`__.
|
||||
|
||||
Weight and Query/Group Data
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
LightGBM also supports weighted training, it needs an additional `weight data <./Parameters.rst#weight-data>`__.
|
||||
And it needs an additional `query data <./Parameters.rst#query-data>`_ for ranking task.
|
||||
|
||||
Also, `weight <./Parameters.rst#weight_column>`__ and `query <./Parameters.rst#group_column>`__ data could be specified as columns in training data in the same manner as label.
|
||||
|
||||
Parameters Quick Look
|
||||
---------------------
|
||||
|
||||
The parameters format is ``key1=value1 key2=value2 ...``.
|
||||
|
||||
Parameters can be set both in config file and command line.
|
||||
If one parameter appears in both command line and config file, LightGBM will use the parameter from the command line.
|
||||
|
||||
The most important parameters which new users should take a look at are located into `Core Parameters <./Parameters.rst#core-parameters>`__
|
||||
and the top of `Learning Control Parameters <./Parameters.rst#learning-control-parameters>`__
|
||||
sections of the full detailed list of `LightGBM's parameters <./Parameters.rst>`__.
|
||||
|
||||
Run LightGBM
|
||||
------------
|
||||
|
||||
::
|
||||
|
||||
lightgbm config=your_config_file other_args ...
|
||||
|
||||
Parameters can be set both in the config file and command line, and the parameters in command line have higher priority than in the config file.
|
||||
For example, the following command line will keep ``num_trees=10`` and ignore the same parameter in the config file.
|
||||
|
||||
::
|
||||
|
||||
lightgbm config=train.conf num_trees=10
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
- `Binary Classification <https://github.com/lightgbm-org/LightGBM/tree/main/examples/binary_classification>`__
|
||||
|
||||
- `Regression <https://github.com/lightgbm-org/LightGBM/tree/main/examples/regression>`__
|
||||
|
||||
- `Lambdarank <https://github.com/lightgbm-org/LightGBM/tree/main/examples/lambdarank>`__
|
||||
|
||||
- `Distributed Learning <https://github.com/lightgbm-org/LightGBM/tree/main/examples/parallel_learning>`__
|
||||
|
||||
.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values
|
||||
|
||||
.. _TSV: https://en.wikipedia.org/wiki/Tab-separated_values
|
||||
|
||||
.. _LibSVM: https://www.csie.ntu.edu.tw/~cjlin/libsvm/
|
||||
|
||||
.. _Expo data: https://community.amstat.org/jointscsg-section/dataexpo/dataexpo2009
|
||||
@@ -0,0 +1,69 @@
|
||||
Documentation
|
||||
=============
|
||||
|
||||
Documentation for LightGBM is generated using `Sphinx <https://www.sphinx-doc.org/>`__
|
||||
and `Breathe <https://breathe.readthedocs.io/>`__, which works on top of `Doxygen <https://www.doxygen.nl/index.html>`__ output.
|
||||
|
||||
List of parameters and their descriptions in `Parameters.rst <./Parameters.rst>`__
|
||||
is generated automatically from comments in `config file <https://github.com/lightgbm-org/LightGBM/blob/main/include/LightGBM/config.h>`__
|
||||
by `this script <https://github.com/lightgbm-org/LightGBM/blob/main/.ci/parameter-generator.py>`__.
|
||||
|
||||
After each commit on ``main``, documentation is updated and published to `Read the Docs <https://lightgbm.readthedocs.io/>`__.
|
||||
|
||||
Build
|
||||
-----
|
||||
|
||||
It is not necessary to re-build this documentation while modifying LightGBM's source code.
|
||||
The HTML files generated using ``Sphinx`` are not checked into source control.
|
||||
However, you may want to build them locally during development to test changes.
|
||||
|
||||
Docker
|
||||
^^^^^^
|
||||
|
||||
The most reliable way to build the documentation locally is with Docker, using `the same images Read the Docs uses <https://hub.docker.com/r/readthedocs/build>`_.
|
||||
|
||||
Run the following from the root of this repository to pull the relevant image and run a container locally.
|
||||
|
||||
.. code:: sh
|
||||
|
||||
docker run \
|
||||
--rm \
|
||||
--user=0 \
|
||||
-v $(pwd):/opt/LightGBM \
|
||||
--env C_API=true \
|
||||
--env CONDA=/opt/miniforge \
|
||||
--env READTHEDOCS=true \
|
||||
--workdir=/opt/LightGBM/docs \
|
||||
--entrypoint="" \
|
||||
readthedocs/build:ubuntu-24.04-2024.06.17 \
|
||||
/bin/bash build-docs.sh
|
||||
|
||||
When that code completes, open ``docs/_build/html/index.html`` in your browser.
|
||||
|
||||
.. note::
|
||||
|
||||
The navigation in these locally-built docs does not link to the local copy of the R documentation. To view the local version of the R docs, open ``docs/_build/html/R/index.html`` in your browser.
|
||||
|
||||
Without Docker
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
You can build the documentation locally without Docker. Just install Doxygen and run in ``docs`` folder
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install breathe sphinx 'sphinx_rtd_theme>=0.5'
|
||||
make html
|
||||
|
||||
Note that this will not build the R documentation.
|
||||
Consider using common R utilities for documentation generation, if you need it.
|
||||
Or use the Docker-based approach described above to build the R documentation locally.
|
||||
|
||||
Optionally, you may also install ``scikit-learn`` and get richer documentation for the classes in ``Scikit-learn API``.
|
||||
|
||||
If you faced any problems with Doxygen installation or you simply do not need documentation for C code, it is possible to build the documentation without it:
|
||||
|
||||
.. code:: sh
|
||||
|
||||
pip install sphinx 'sphinx_rtd_theme>=0.5'
|
||||
export C_API=NO || set C_API=NO
|
||||
make html
|
||||
Vendored
+1
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 87 KiB |
+1
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 59 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 108 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Vendored
+52
@@ -0,0 +1,52 @@
|
||||
$(() => {
|
||||
/* Use wider container for the page content */
|
||||
$(".wy-nav-content").each(function () {
|
||||
this.style.setProperty("max-width", "none", "important");
|
||||
});
|
||||
|
||||
/* List each class property item on a new line
|
||||
https://github.com/lightgbm-org/LightGBM/issues/5073 */
|
||||
if (window.location.pathname.toLocaleLowerCase().indexOf("pythonapi") !== -1) {
|
||||
$(".py.property").each(function () {
|
||||
this.style.setProperty("display", "inline", "important");
|
||||
});
|
||||
}
|
||||
|
||||
/* Collapse specified sections in the installation guide */
|
||||
if (window.location.pathname.toLocaleLowerCase().indexOf("installation-guide") !== -1) {
|
||||
$(
|
||||
'<style>.closed, .opened {cursor: pointer;} .closed:before, .opened:before {font-family: FontAwesome; display: inline-block; padding-right: 6px;} .closed:before {content: "\\f054";} .opened:before {content: "\\f078";}</style>',
|
||||
).appendTo("body");
|
||||
const collapsible = [
|
||||
"#build-threadless-version-not-recommended",
|
||||
"#build-mpi-version",
|
||||
"#build-gpu-version",
|
||||
"#build-cuda-version",
|
||||
"#build-rocm-version",
|
||||
"#build-java-wrapper",
|
||||
"#build-python-package",
|
||||
"#build-r-package",
|
||||
"#build-c-unit-tests",
|
||||
];
|
||||
$.each(collapsible, (_, val) => {
|
||||
const header = `${val} > :header:first`;
|
||||
const content = `${val} :not(:header:first)`;
|
||||
$(header).addClass("closed");
|
||||
$(content).hide();
|
||||
$(header).click(() => {
|
||||
$(header).toggleClass("closed opened");
|
||||
$(content).slideToggle(0);
|
||||
});
|
||||
});
|
||||
/* Uncollapse parent sections when nested section is specified in the URL or before navigate to it from navbar */
|
||||
function uncollapse(section) {
|
||||
section.parents().each((_, val) => {
|
||||
$(val).children(".closed").click();
|
||||
});
|
||||
}
|
||||
uncollapse($(window.location.hash));
|
||||
$(".wy-menu.wy-menu-vertical li a.reference.internal").click(function () {
|
||||
uncollapse($($(this).attr("href")));
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e -E -u -o pipefail
|
||||
|
||||
rm -f ./_FIRST_RUN.flag
|
||||
|
||||
export PATH="${CONDA}/bin:${PATH}"
|
||||
|
||||
curl \
|
||||
-sL \
|
||||
-o "${HOME}/miniforge.sh" \
|
||||
https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh
|
||||
|
||||
/bin/bash "${HOME}/miniforge.sh" -b -p "${CONDA}"
|
||||
|
||||
conda config --set always_yes yes --set changeps1 no
|
||||
conda update -q -y conda
|
||||
|
||||
conda env create \
|
||||
--name docs-env \
|
||||
--file env.yml || exit 1
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source activate docs-env
|
||||
make clean html || exit 1
|
||||
|
||||
echo "Done building docs. Open docs/_build/html/index.html in a web browser to view them."
|
||||
+335
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# LightGBM documentation build configuration file, created by
|
||||
# sphinx-quickstart on Thu May 4 14:30:58 2017.
|
||||
#
|
||||
# This file is execfile()d with the current directory set to its
|
||||
# containing dir.
|
||||
#
|
||||
# Note that not all possible configuration values are present in this
|
||||
# autogenerated file.
|
||||
#
|
||||
# All configuration values have a default; values that are commented out
|
||||
# serve to show the default.
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute.
|
||||
"""Sphinx configuration file."""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from re import compile
|
||||
from shutil import copytree
|
||||
from subprocess import PIPE, Popen
|
||||
from typing import Any, List
|
||||
|
||||
import sphinx
|
||||
from docutils.nodes import reference
|
||||
from docutils.parsers.rst import Directive
|
||||
from docutils.transforms import Transform
|
||||
from sphinx.application import Sphinx
|
||||
from sphinx.errors import VersionRequirementError
|
||||
|
||||
CURR_PATH = Path(__file__).absolute().parent
|
||||
LIB_PATH = CURR_PATH.parent / "python-package"
|
||||
sys.path.insert(0, str(LIB_PATH))
|
||||
|
||||
INTERNAL_REF_REGEX = compile(r"(?P<url>\.\/.+)(?P<extension>\.rst)(?P<anchor>$|#)")
|
||||
RTD_R_REF_REGEX = compile(r"(?P<begin>https://.+/)(?P<rtd_version>latest)(?P<end>/R/reference/)")
|
||||
|
||||
|
||||
class InternalRefTransform(Transform):
|
||||
"""Replaces '.rst' with '.html' in all internal links like './[Something].rst[#anchor]'."""
|
||||
|
||||
default_priority = 210
|
||||
"""Numerical priority of this transform, 0 through 999."""
|
||||
|
||||
def apply(self, **kwargs: Any) -> None:
|
||||
"""Apply the transform to the document tree."""
|
||||
for section in self.document.traverse(reference):
|
||||
if section.get("refuri") is not None:
|
||||
section["refuri"] = INTERNAL_REF_REGEX.sub(r"\g<url>.html\g<anchor>", section["refuri"])
|
||||
|
||||
|
||||
class IgnoredDirective(Directive):
|
||||
"""Stub for unknown directives."""
|
||||
|
||||
has_content = True
|
||||
|
||||
def run(self) -> List:
|
||||
"""Do nothing."""
|
||||
return []
|
||||
|
||||
|
||||
# -- General configuration ------------------------------------------------
|
||||
|
||||
os.environ["LIGHTGBM_BUILD_DOC"] = "True"
|
||||
C_API = os.environ.get("C_API", "").lower().strip() != "no"
|
||||
RTD = bool(os.environ.get("READTHEDOCS", ""))
|
||||
RTD_VERSION = os.environ.get("READTHEDOCS_VERSION", "stable")
|
||||
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
needs_sphinx = "2.1.0" # Due to sphinx.ext.napoleon, autodoc_typehints
|
||||
if needs_sphinx > sphinx.__version__:
|
||||
message = f"This project needs at least Sphinx v{needs_sphinx}"
|
||||
raise VersionRequirementError(message)
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = [
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.autosummary",
|
||||
"sphinx.ext.todo",
|
||||
"sphinx.ext.viewcode",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.intersphinx",
|
||||
]
|
||||
|
||||
autodoc_default_flags = ["members", "inherited-members", "show-inheritance"]
|
||||
autodoc_default_options = {
|
||||
"members": True,
|
||||
"inherited-members": True,
|
||||
"show-inheritance": True,
|
||||
}
|
||||
# mock out modules
|
||||
autodoc_mock_imports = [
|
||||
"dask",
|
||||
"dask.distributed",
|
||||
"graphviz",
|
||||
"matplotlib",
|
||||
"narwhals",
|
||||
"numpy",
|
||||
"pandas",
|
||||
"scipy",
|
||||
"scipy.sparse",
|
||||
]
|
||||
try:
|
||||
import sklearn # noqa: F401
|
||||
except ImportError:
|
||||
autodoc_mock_imports.append("sklearn")
|
||||
# hide type hints in API docs
|
||||
autodoc_typehints = "none"
|
||||
|
||||
# Generate autosummary pages. Output should be set with: `:toctree: pythonapi/`
|
||||
autosummary_generate = ["Python-API.rst"]
|
||||
|
||||
# Only the class' docstring is inserted.
|
||||
autoclass_content = "class"
|
||||
|
||||
# If true, `todo` and `todoList` produce output, else they produce nothing.
|
||||
todo_include_todos = False
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = "index"
|
||||
|
||||
# General information about the project.
|
||||
project = "LightGBM"
|
||||
copyright = f"{datetime.datetime.now().year}, Microsoft Corporation"
|
||||
author = "Microsoft Corporation"
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top
|
||||
# of the sidebar.
|
||||
html_logo = str(CURR_PATH / "logo" / "LightGBM_logo_grey_text.svg")
|
||||
|
||||
# The name of an image file (relative to this directory) to use as a favicon of
|
||||
# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
||||
# pixels large.
|
||||
html_favicon = str(CURR_PATH / "_static" / "images" / "favicon.ico")
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
# The short X.Y version.
|
||||
version = (CURR_PATH.parent / "VERSION.txt").read_text(encoding="utf-8").strip().replace("rc", "-rc")
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = version
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#
|
||||
# This is also used if you do content translation via gettext catalogs.
|
||||
# Usually you set "language" from the command line for these cases.
|
||||
language = "en"
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This patterns also effect to html_static_path and html_extra_path
|
||||
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = "default"
|
||||
|
||||
# -- Configuration for C API docs generation ------------------------------
|
||||
|
||||
if C_API:
|
||||
extensions.extend(
|
||||
[
|
||||
"breathe",
|
||||
]
|
||||
)
|
||||
breathe_projects = {"LightGBM": str(CURR_PATH / "doxyoutput" / "xml")}
|
||||
breathe_default_project = "LightGBM"
|
||||
breathe_domain_by_extension = {
|
||||
"h": "c",
|
||||
}
|
||||
breathe_show_define_initializer = True
|
||||
c_id_attributes = ["LIGHTGBM_C_EXPORT"]
|
||||
|
||||
# -- Options for HTML output ----------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
html_theme_options = {
|
||||
"includehidden": False,
|
||||
"logo_only": True,
|
||||
}
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ["_static"]
|
||||
|
||||
# -- Options for HTMLHelp output ------------------------------------------
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = "LightGBMdoc"
|
||||
|
||||
# -- Options for LaTeX output ---------------------------------------------
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
# the title page.
|
||||
latex_logo = str(CURR_PATH / "logo" / "LightGBM_logo_black_text_small.png")
|
||||
|
||||
# intersphinx configuration
|
||||
intersphinx_mapping = {
|
||||
"sklearn": ("https://scikit-learn.org/stable/", None),
|
||||
}
|
||||
|
||||
|
||||
def generate_doxygen_xml(app: Sphinx) -> None:
|
||||
"""Generate XML documentation for C API by Doxygen.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
app : sphinx.application.Sphinx
|
||||
The application object representing the Sphinx process.
|
||||
"""
|
||||
doxygen_args = [
|
||||
f"INPUT={CURR_PATH.parent / 'include' / 'LightGBM' / 'c_api.h'}",
|
||||
f"OUTPUT_DIRECTORY={CURR_PATH / 'doxyoutput'}",
|
||||
"GENERATE_HTML=NO",
|
||||
"GENERATE_LATEX=NO",
|
||||
"GENERATE_XML=YES",
|
||||
"XML_OUTPUT=xml",
|
||||
"XML_PROGRAMLISTING=YES",
|
||||
r'ALIASES="rst=\verbatim embed:rst:leading-asterisk"',
|
||||
r'ALIASES+="endrst=\endverbatim"',
|
||||
"ENABLE_PREPROCESSING=YES",
|
||||
"MACRO_EXPANSION=YES",
|
||||
"EXPAND_ONLY_PREDEF=NO",
|
||||
"SKIP_FUNCTION_MACROS=NO",
|
||||
"PREDEFINED=__cplusplus",
|
||||
"SORT_BRIEF_DOCS=YES",
|
||||
"WARN_AS_ERROR=YES",
|
||||
]
|
||||
doxygen_input = "\n".join(doxygen_args)
|
||||
doxygen_input = bytes(doxygen_input, "utf-8")
|
||||
(CURR_PATH / "doxyoutput").mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
# Warning! The following code can cause buffer overflows on RTD.
|
||||
# Consider suppressing output completely if RTD project silently fails.
|
||||
# Refer to https://github.com/svenevs/exhale
|
||||
# /blob/fe7644829057af622e467bb529db6c03a830da99/exhale/deploy.py#L99-L111
|
||||
process = Popen(["doxygen", "-"], stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||
stdout, stderr = process.communicate(doxygen_input)
|
||||
output = "\n".join([i.decode("utf-8") for i in (stdout, stderr) if i is not None])
|
||||
if process.returncode != 0:
|
||||
raise RuntimeError(output)
|
||||
print(output)
|
||||
except BaseException as e:
|
||||
raise Exception(f"An error has occurred while executing Doxygen\n{e}")
|
||||
|
||||
|
||||
def generate_r_docs(app: Sphinx) -> None:
|
||||
"""Generate documentation for R-package.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
app : sphinx.application.Sphinx
|
||||
The application object representing the Sphinx process.
|
||||
"""
|
||||
commands = f"""
|
||||
export TAR=/bin/tar
|
||||
cd {CURR_PATH.parent}
|
||||
export R_LIBS="$CONDA_PREFIX/lib/R/library"
|
||||
sh build-cran-package.sh || exit 1
|
||||
R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit 1
|
||||
Rscript .ci/build-docs.R || exit 1
|
||||
"""
|
||||
try:
|
||||
print("Building R-package documentation")
|
||||
# Warning! The following code can cause buffer overflows on RTD.
|
||||
# Consider suppressing output completely if RTD project silently fails.
|
||||
# Refer to https://github.com/svenevs/exhale
|
||||
# /blob/fe7644829057af622e467bb529db6c03a830da99/exhale/deploy.py#L99-L111
|
||||
process = Popen(["/bin/bash"], stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True)
|
||||
stdout, stderr = process.communicate(commands)
|
||||
output = "\n".join([i for i in (stdout, stderr) if i is not None])
|
||||
if process.returncode != 0:
|
||||
raise RuntimeError(output)
|
||||
print(output)
|
||||
print("Done building R-package documentation")
|
||||
except BaseException as e:
|
||||
raise Exception(f"An error has occurred while generating documentation for R-package\n{e}")
|
||||
|
||||
|
||||
def replace_reference_to_r_docs(app: Sphinx) -> None:
|
||||
"""Make reference to R-package documentation point to the actual version.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
app : sphinx.application.Sphinx
|
||||
The application object representing the Sphinx process.
|
||||
"""
|
||||
index_doc_path = CURR_PATH / "index.rst"
|
||||
with open(index_doc_path, "r+t", encoding="utf-8") as index_doc:
|
||||
content = index_doc.read()
|
||||
content = RTD_R_REF_REGEX.sub(rf"\g<begin>{RTD_VERSION}\g<end>", content)
|
||||
index_doc.seek(0)
|
||||
index_doc.write(content)
|
||||
|
||||
|
||||
def setup(app: Sphinx) -> None:
|
||||
"""Add new elements at Sphinx initialization time.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
app : sphinx.application.Sphinx
|
||||
The application object representing the Sphinx process.
|
||||
"""
|
||||
first_run = not (CURR_PATH / "_FIRST_RUN.flag").exists()
|
||||
if first_run and RTD:
|
||||
(CURR_PATH / "_FIRST_RUN.flag").touch()
|
||||
if C_API:
|
||||
app.connect("builder-inited", generate_doxygen_xml)
|
||||
else:
|
||||
app.add_directive("doxygenfile", IgnoredDirective)
|
||||
if first_run:
|
||||
app.connect("builder-inited", generate_r_docs)
|
||||
app.connect(
|
||||
"build-finished", lambda app, _: copytree(CURR_PATH.parent / "R-package" / "docs", Path(app.outdir) / "R")
|
||||
)
|
||||
app.connect("builder-inited", replace_reference_to_r_docs)
|
||||
app.add_transform(InternalRefTransform)
|
||||
add_js_file = getattr(app, "add_js_file", False) or app.add_javascript
|
||||
add_js_file("js/script.js")
|
||||
@@ -0,0 +1,19 @@
|
||||
name: docs-env
|
||||
channels:
|
||||
- nodefaults
|
||||
- conda-forge
|
||||
dependencies:
|
||||
- breathe>=4.36
|
||||
- doxygen>=1.13.2
|
||||
- python=3.14
|
||||
- r-base>=4.4
|
||||
- r-data.table=1.17.8
|
||||
- r-jsonlite=2.0.0
|
||||
- r-knitr=1.51
|
||||
- r-markdown=2.0
|
||||
- r-matrix=1.7_4
|
||||
- r-pkgdown=2.2.0
|
||||
- r-roxygen2=8.0.0
|
||||
- scikit-learn>=1.8.0
|
||||
- sphinx>=8.1.3
|
||||
- sphinx_rtd_theme>=3.0.1
|
||||
@@ -0,0 +1 @@
|
||||
The content of this document was very outdated and is no longer available to avoid any misleadings.
|
||||
@@ -0,0 +1,58 @@
|
||||
.. LightGBM documentation master file, created by
|
||||
sphinx-quickstart on Thu May 4 14:30:58 2017.
|
||||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
.. image:: ./logo/LightGBM_logo_black_text.svg
|
||||
:align: center
|
||||
:width: 600
|
||||
:alt: Light Gradient Boosting Machine logo.
|
||||
|
||||
|
|
||||
|
||||
Welcome to LightGBM's documentation!
|
||||
====================================
|
||||
|
||||
**LightGBM** is a gradient boosting framework that uses tree based learning algorithms. It is designed to be distributed and efficient with the following advantages:
|
||||
|
||||
- Faster training speed and higher efficiency.
|
||||
- Lower memory usage.
|
||||
- Better accuracy.
|
||||
- Support of parallel, distributed, and GPU learning.
|
||||
- Capable of handling large-scale data.
|
||||
|
||||
For more details, please refer to `Features <./Features.rst>`__.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:caption: Contents:
|
||||
|
||||
Installation Guide <Installation-Guide>
|
||||
Quick Start <Quick-Start>
|
||||
Python Quick Start <Python-Intro>
|
||||
Features <Features>
|
||||
Experiments <Experiments>
|
||||
Parameters <Parameters>
|
||||
Parameters Tuning <Parameters-Tuning>
|
||||
C API <C-API>
|
||||
Python API <Python-API>
|
||||
R API <https://lightgbm.readthedocs.io/en/latest/R/reference/>
|
||||
Distributed Learning Guide <Parallel-Learning-Guide>
|
||||
GPU Tutorial <GPU-Tutorial>
|
||||
Advanced Topics <Advanced-Topics>
|
||||
FAQ <FAQ>
|
||||
Development Guide <Development-Guide>
|
||||
|
||||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
GPU-Performance
|
||||
GPU-Targets
|
||||
GPU-Windows
|
||||
gcc-Tips
|
||||
README
|
||||
|
||||
Indices and Tables
|
||||
==================
|
||||
|
||||
* :ref:`genindex`
|
||||
Binary file not shown.
@@ -0,0 +1,41 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Creator: CorelDRAW 2020 (64-Bit Evaluation Version) -->
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="8442px" height="9748px" version="1.1" style="shape-rendering:geometricPrecision; text-rendering:geometricPrecision; image-rendering:optimizeQuality; fill-rule:evenodd; clip-rule:evenodd"
|
||||
viewBox="0 0 8438.53 9743.98"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns:xodm="http://www.corel.com/coreldraw/odm/2003">
|
||||
<defs>
|
||||
<style type="text/css">
|
||||
<![CDATA[
|
||||
.str0 {stroke:#373435;stroke-width:2.36;stroke-miterlimit:22.9256}
|
||||
.fil1 {fill:#373435}
|
||||
.fil5 {fill:#EF4A37}
|
||||
.fil3 {fill:#1B9AD7}
|
||||
.fil2 {fill:#76B644}
|
||||
.fil0 {fill:#FCB518}
|
||||
.fil4 {fill:#E6E7E8;fill-rule:nonzero}
|
||||
]]>
|
||||
</style>
|
||||
</defs>
|
||||
<g id="图层_x0020_1">
|
||||
<metadata id="CorelCorpID_0Corel-Layer"/>
|
||||
<polygon class="fil0" points="4219.26,0 6328.9,1218 8438.53,2435.99 8438.53,4871.99 8438.53,7307.98 6328.9,8525.98 4219.26,9743.98 2109.63,8525.98 0,7307.98 0,4871.99 0,2435.99 2109.63,1218 "/>
|
||||
<polygon class="fil1 str0" points="4219.26,287 6204.62,1433.25 8189.98,2579.5 8189.98,4871.99 8189.98,7164.48 6204.62,8310.72 4219.26,9456.97 2233.91,8310.72 248.55,7164.48 248.55,4871.99 248.55,2579.5 2233.91,1433.25 "/>
|
||||
<polygon class="fil2" points="3906.02,3183.13 5436.73,2190.94 5436.73,3483.55 "/>
|
||||
<polygon class="fil0" points="2815.23,5749.77 5436.74,5749.77 5436.74,4071.86 "/>
|
||||
<polygon class="fil3" points="4345.88,3965.45 2815.16,4957.64 2815.16,3665.03 "/>
|
||||
<g id="_1713672554880">
|
||||
<path class="fil4" d="M7626.3 7313.8l-194.4 0 0 -467.44c0,-89.77 -12.58,-154.96 -38.02,-194.98 -25.45,-40.31 -68.33,-60.32 -128.66,-60.32 -50.89,0 -94.05,25.44 -129.51,76.05 -35.45,50.88 -53.17,111.78 -53.17,182.68l0 464.01 -194.98 0 0 -483.17c0,-159.81 -56.32,-239.57 -169.25,-239.57 -52.61,0 -95.78,24.01 -129.52,72.04 -33.73,48.03 -50.6,110.07 -50.6,186.69l0 464.01 -194.4 0 0 -857.68 194.4 0 0 135.79 3.44 0c62.03,-104.06 152.09,-155.81 270.45,-155.81 59.18,0 110.93,16.3 155.53,48.89 44.31,32.59 74.61,75.48 90.63,128.65 63.75,-118.36 158.67,-177.54 285.03,-177.54 188.69,0 283.03,116.36 283.03,349.08l0 528.62z"/>
|
||||
<path class="fil4" d="M5535 7189.72l-3.43 0 0 124.08 -194.41 0 0 -1269.66 194.41 0 0 562.64 3.43 0c66.33,-113.78 163.53,-170.68 291.33,-170.68 108.35,0 193.26,38.31 254.15,115.22 61.19,76.62 91.78,179.54 91.78,308.47 0,143.53 -34.31,258.45 -102.92,344.8 -68.62,86.05 -162.39,129.22 -281.32,129.22 -111.79,0 -196.12,-48.03 -253.02,-144.09zm-5.14 -340.79l0 106.35c0,62.62 20.01,115.51 60.03,159.25 39.74,43.45 90.63,65.19 151.81,65.19 72.05,0 128.65,-28.03 169.82,-84.06 40.89,-56.04 61.47,-134.37 61.47,-235 0,-84.34 -19.15,-150.39 -57.46,-198.13 -38.03,-47.74 -90.06,-71.47 -155.24,-71.47 -69.19,0 -124.94,24.58 -167.25,73.47 -42.03,49.17 -63.18,110.64 -63.18,184.4z"/>
|
||||
<path class="fil4" d="M4150.5 7304.65c-37.74,18.87 -87.77,28.3 -149.8,28.3 -166.4,0 -249.59,-79.76 -249.59,-239.58l0 -484.88 -143.24 0 0 -152.37 143.24 0 0 -198.42 194.41 -55.46 0 253.88 204.98 0 0 152.37 -204.98 0 0 428.85c0,50.89 9.14,87.2 27.44,108.92 18.58,21.73 49.17,32.59 92.34,32.59 32.88,0 61.19,-9.43 85.2,-28.3l0 154.1z"/>
|
||||
<path class="fil4" d="M3435.32 7313.8l-194.12 0 0 -469.16c0,-169.53 -56.89,-254.44 -170.11,-254.44 -56.89,0 -104.92,24.59 -144.09,73.76 -39.16,48.89 -58.6,111.78 -58.6,188.41l0 461.43 -195.27 0 0 -1269.66 195.27 0 0 554.35 3.43 0c64.61,-108.35 157.52,-162.39 277.88,-162.39 190.41,0 285.61,116.36 285.61,349.08l0 528.62z"/>
|
||||
<polygon class="fil4" points="1482.64,7313.8 1288.23,7313.8 1288.23,6456.12 1482.64,6456.12 "/>
|
||||
<path class="fil4" d="M1386.29 6276c-31.73,0 -58.89,-10.29 -81.48,-30.88 -22.87,-20.87 -34.02,-46.88 -34.02,-78.9 0,-31.74 11.15,-58.32 34.02,-79.48 22.59,-21.16 49.75,-31.73 81.48,-31.73 32.88,0 60.9,10.57 83.77,31.73 22.87,21.16 34.31,47.74 34.31,79.48 0,30.3 -11.44,56.04 -34.31,77.47 -22.87,21.45 -50.89,32.31 -83.77,32.31z"/>
|
||||
<polygon class="fil4" points="1112.75,7313.8 917.48,7313.8 917.48,6044.14 1112.75,6044.14 "/>
|
||||
<path class="fil4" d="M2496.58 7245.18c0,314.77 -158.39,472.3 -474.87,472.3 -111.79,0 -209.28,-18.58 -292.48,-56.04l0 -177.54c93.78,53.47 182.98,80.34 267.32,80.34 203.84,0 305.62,-100.35 305.62,-300.76l0 -93.77 -3.43 0c-64.05,109.5 -160.68,164.1 -289.61,164.1 -104.36,0 -188.7,-38.02 -252.73,-114.36 -63.76,-76.05 -95.78,-178.4 -95.78,-306.76 0,-145.81 34.31,-261.6 102.92,-347.65 68.9,-86.05 163.25,-128.94 283.32,-128.94 113.22,0 197.27,46.32 251.88,138.94l3.43 0 0 -118.92 194.41 0 0 789.06zm-192.7 -324.2l0 -111.5c0,-60.32 -20.01,-111.78 -59.75,-154.38 -40.02,-42.89 -89.77,-64.04 -149.52,-64.04 -73.76,0 -131.51,27.16 -172.97,81.48 -41.74,54.6 -62.61,130.65 -62.61,228.43 0,84.33 20.02,151.52 60.04,202.12 39.74,50.61 92.91,75.76 158.67,75.76 66.9,0 121.51,-24.01 163.25,-72.32 42.02,-48.32 62.89,-110.08 62.89,-185.55z"/>
|
||||
<path class="fil4" d="M5165.09 7245.18c0,314.77 -158.38,472.3 -474.87,472.3 -111.78,0 -209.27,-18.58 -292.47,-56.04l0 -177.54c93.77,53.47 182.97,80.34 267.31,80.34 203.84,0 305.62,-100.35 305.62,-300.76l0 -93.77 -3.43 0c-64.04,109.5 -160.67,164.1 -289.61,164.1 -104.35,0 -188.69,-38.02 -252.73,-114.36 -63.76,-76.05 -95.78,-178.4 -95.78,-306.76 0,-145.81 34.31,-261.6 102.93,-347.65 68.9,-86.05 163.24,-128.94 283.32,-128.94 113.21,0 197.27,46.32 251.87,138.94l3.43 0 0 -118.92 194.41 0 0 789.06zm-192.69 -324.2l0 -111.5c0,-60.32 -20.02,-111.78 -59.75,-154.38 -40.03,-42.89 -89.77,-64.04 -149.53,-64.04 -73.76,0 -131.51,27.16 -172.96,81.48 -41.74,54.6 -62.61,130.65 -62.61,228.43 0,84.33 20.01,151.52 60.03,202.12 39.74,50.61 92.92,75.76 158.67,75.76 66.9,0 121.51,-24.01 163.25,-72.32 42.03,-48.32 62.9,-110.08 62.9,-185.55z"/>
|
||||
</g>
|
||||
<polygon class="fil5" points="2814.64,3068.81 2814.64,1388.62 5436.7,1388.62 "/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 77 KiB |
Binary file not shown.
@@ -0,0 +1,39 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Creator: CorelDRAW 2020 (64-Bit Evaluation Version) -->
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="4672px" height="1058px" version="1.1" style="shape-rendering:geometricPrecision; text-rendering:geometricPrecision; image-rendering:optimizeQuality; fill-rule:evenodd; clip-rule:evenodd"
|
||||
viewBox="0 0 4645.44 1052.32"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns:xodm="http://www.corel.com/coreldraw/odm/2003">
|
||||
<defs>
|
||||
<style type="text/css">
|
||||
<![CDATA[
|
||||
.fil0 {fill:none}
|
||||
.fil5 {fill:#1B9AD7}
|
||||
.fil2 {fill:#76B644}
|
||||
.fil1 {fill:#EF4927}
|
||||
.fil4 {fill:#FCB518}
|
||||
.fil3 {fill:#4B4B4D;fill-rule:nonzero}
|
||||
]]>
|
||||
</style>
|
||||
</defs>
|
||||
<g id="图层_x0020_1">
|
||||
<metadata id="CorelCorpID_0Corel-Layer"/>
|
||||
<rect class="fil0" width="4645.44" height="1052.32"/>
|
||||
<polygon class="fil1" points="629.73,0.27 0,0.27 0,403.33 "/>
|
||||
<polygon class="fil2" points="262.04,431.06 629.75,192.72 629.75,503.23 "/>
|
||||
<g>
|
||||
<path class="fil3" d="M4645.44 837.58l-103.46 0 4.34 -529.43c-6.44,27.4 -12.13,47.17 -17.07,59.14l-187.75 470.29 -71.87 0 -188.21 -466.69c-5.24,-13.78 -10.93,-34.74 -16.77,-62.74l-1.65 0c2.25,25.16 3.45,66.18 3.45,123.23l0 406.2 -96.42 0 0 -628.99 146.88 0 165.29 418.93c12.58,32.19 20.82,56.15 24.71,71.87l2.1 0c10.93,-33.09 19.61,-57.65 26.34,-73.67l168.45 -417.13 141.64 0 0 628.99zm-375.81 0l-253.66 -628.99 100.94 0 192.1 490.8 -39.38 138.19zm322.98 -628.99l-88.81 0 88.81 0zm0 0l-251.11 628.99 -33.01 -139.52 195.31 -489.47 88.81 0z"/>
|
||||
</g>
|
||||
<path class="fil3" d="M3449.37 834.75l0 -625.48 198.47 0c60.45,0 108.4,13.25 143.68,39.6 35.29,26.51 53.01,61.05 53.01,103.48 0,35.44 -10.13,66.26 -30.08,92.46 -20.1,26.21 -47.94,44.82 -83.38,55.84l0 1.78c43.03,4.92 77.42,20.85 103.18,47.65 25.76,26.95 38.57,61.94 38.57,104.96 0,53.46 -21,96.78 -63.13,129.99 -42.29,33.2 -95.44,49.72 -159.76,49.72l-200.56 0zm103.39 -534.58l0.2 136.51c0.04,24.18 -2.1,34.43 28.69,34.19l38.2 -0.3c36.03,-0.28 64.32,-8.49 84.87,-25.61 20.55,-16.97 30.67,-41.09 30.67,-72.21 0,-53.45 -35.73,-80.25 -107.2,-80.25l-66.2 0c-6.79,0 -9.25,0.65 -9.23,7.67zm-0.06 264.61l0 154.26c0,26.53 -0.42,32.78 28.85,32.78l59.74 0c38.56,0 68.49,-8.93 89.63,-26.8 21,-17.86 31.57,-42.43 31.57,-73.85 0,-64.91 -44.97,-97.37 -134.75,-97.37l-63.23 0c-8.81,0 -11.81,2.15 -11.81,10.98z"/>
|
||||
<path class="fil3" d="M3351.47 783.93c-63.06,34.27 -133.19,51.4 -210.51,51.4 -89.42,0 -161.85,-27.79 -217.29,-83.51 -55.43,-55.58 -83.08,-129.31 -83.08,-221.03 0,-93.59 30.24,-170.48 90.86,-230.53 60.62,-60.04 137.66,-90.14 230.96,-90.14 63.51,0 117.52,8.86 161.89,26.47 8.49,3.37 7.73,1.48 7.73,10.2l0 96.24c-46.36,-30.82 -101.65,-46.37 -165.73,-46.37 -64.51,0 -117.21,21.31 -158.39,63.65 -41.18,42.47 -61.92,97.48 -61.92,165.01 0,69.4 17.86,124.12 53.28,163.86 35.42,39.89 83.52,59.76 144.28,59.76 36.83,0 69.27,-6.32 97.3,-18.85 13.55,-6.06 10.69,-4.56 10.69,-19.42l0 -118.76c0,-9.58 -4.78,-10.86 -13.69,-10.86l-113.74 0 0 -66.84c0,-14.7 3.11,-17.54 18.89,-17.54l208.47 0 0 287.26z"/>
|
||||
<path class="fil3" d="M2773.73 821.77c-19.51,9.22 -45.03,13.83 -76.43,13.83 -87.14,0 -130.71,-41.77 -130.71,-125.47l0 -253.93 -37.51 0 0 -79.81 37.51 0 -0.08 -113.52c0,-7.1 2.14,-11.35 9.03,-14.4l92.74 -41.07 0.12 168.99 107.36 0 0 79.81 -107.36 0 0 224.59c0,26.65 4.79,45.66 14.37,57.04 9.74,11.38 25.76,17.07 48.37,17.07 17.22,0 32.04,-4.94 44.62,-14.82l0 78.4c0,2.65 0.39,2.15 -2.03,3.29z"/>
|
||||
<path class="fil3" d="M2464.86 837.58l-101.66 0 0 -245.7c0,-88.78 -29.8,-133.25 -89.09,-133.25 -29.79,0 -54.95,12.87 -75.46,38.63 -20.51,25.6 -30.69,58.54 -30.69,98.67l0 241.65 -102.27 0 0 -624.07c0,-5.64 3.31,-4.95 8.1,-4.95l90.21 0c2.37,0 3.96,-0.37 3.96,2.55l0 251.86 1.79 0c33.84,-56.75 82.5,-85.05 145.54,-85.05 99.71,0 149.57,60.94 149.57,182.82l0 276.84z"/>
|
||||
<path class="fil3" d="M2002.55 789.04c0,164.85 -82.95,247.35 -248.69,247.35 -58.55,0 -109.6,-9.73 -153.17,-29.35l83.52 -58.21c18.34,7.63 38.08,7.31 56.47,7.31 106.76,0 160.06,-52.56 160.06,-157.52l0 -49.11 -1.8 0c-33.54,57.35 -84.14,85.95 -151.67,85.95 -54.65,0 -98.82,-19.92 -132.36,-59.89 -33.39,-39.83 -50.16,-93.43 -50.16,-160.66 0,-76.36 17.97,-137 53.9,-182.07 36.09,-45.06 85.5,-67.52 148.38,-67.52 59.29,0 103.31,24.25 131.91,72.76l1.8 0 0 -49.59c0,-10.62 1.64,-12.69 12.25,-12.69l76.07 0c12.33,0 13.49,-0.94 13.49,11.4l0 401.84zm-100.91 -169.79l0 -58.39c0,-31.59 -10.48,-58.54 -31.3,-80.85 -20.96,-22.46 -47.01,-33.54 -78.3,-33.54 -38.63,0 -68.88,14.22 -90.59,42.67 -21.86,28.6 -32.79,68.43 -32.79,119.63 0,44.17 10.48,79.36 31.45,105.86 20.81,26.5 48.66,39.67 83.09,39.67 35.04,0 63.64,-12.57 85.5,-37.88 22.01,-25.3 32.94,-57.64 32.94,-97.17z"/>
|
||||
<path class="fil3" d="M1499.16 837.58l-101.81 0 0 -441.66c0,-5.84 1.24,-7.51 7.73,-7.51l84.14 0c8.97,0 9.94,-0.28 9.94,8.05l0 441.12z"/>
|
||||
<path class="fil3" d="M1447.36 304.51c-13.77,0 -25.56,-4.47 -35.36,-13.4 -9.93,-9.06 -14.77,-20.35 -14.77,-34.25 0,-13.77 4.84,-25.31 14.77,-34.49 9.8,-9.18 21.59,-13.77 35.36,-13.77 14.27,0 26.43,4.59 36.35,13.77 9.93,9.18 14.89,20.72 14.89,34.49 0,13.16 -4.96,24.32 -14.89,33.63 -9.92,9.31 -22.08,14.02 -36.35,14.02z"/>
|
||||
<path class="fil3" d="M984.92 837.58l0 -622.86c0,-6.28 2.75,-6.13 8.19,-6.13l88.48 0c5.21,0 7.69,-0.29 7.69,5.73l0 531.65c0,4.06 1.5,3.42 4.8,3.42l230.79 0c4.41,0 8.31,-0.51 8.31,5.47l0 82.64 -348.26 0.08z"/>
|
||||
<polygon class="fil4" points="0.02,1047.61 629.75,1047.61 629.75,644.55 "/>
|
||||
<polygon class="fil5" points="367.7,618.99 0,857.33 0,546.82 "/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 137 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.4 KiB |
@@ -0,0 +1,37 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Creator: CorelDRAW 2020 (64-Bit Evaluation Version) -->
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="4672px" height="1053px" version="1.1" style="shape-rendering:geometricPrecision; text-rendering:geometricPrecision; image-rendering:optimizeQuality; fill-rule:evenodd; clip-rule:evenodd"
|
||||
viewBox="0 0 4645.44 1047.35"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns:xodm="http://www.corel.com/coreldraw/odm/2003">
|
||||
<defs>
|
||||
<style type="text/css">
|
||||
<![CDATA[
|
||||
.fil4 {fill:#1B9AD7}
|
||||
.fil1 {fill:#76B644}
|
||||
.fil0 {fill:#EF4927}
|
||||
.fil3 {fill:#FCB518}
|
||||
.fil2 {fill:#E6E7E8;fill-rule:nonzero}
|
||||
]]>
|
||||
</style>
|
||||
</defs>
|
||||
<g id="图层_x0020_1">
|
||||
<metadata id="CorelCorpID_0Corel-Layer"/>
|
||||
<polygon class="fil0" points="629.73,0 0,0 0,403.06 "/>
|
||||
<polygon class="fil1" points="262.04,430.8 629.75,192.45 629.75,502.96 "/>
|
||||
<g>
|
||||
<path class="fil2" d="M4645.44 837.31l-103.46 0 4.34 -529.43c-6.44,27.4 -12.13,47.17 -17.07,59.15l-187.75 470.28 -71.87 0 -188.21 -466.69c-5.24,-13.78 -10.93,-34.74 -16.77,-62.74l-1.65 0c2.25,25.16 3.45,66.18 3.45,123.23l0 406.2 -96.42 0 0 -628.99 146.88 0 165.29 418.93c12.58,32.19 20.82,56.15 24.71,71.87l2.09 0c10.94,-33.09 19.62,-57.65 26.36,-73.67l168.44 -417.13 141.64 0 0 628.99zm-375.81 0l-253.66 -628.99 100.94 0 192.1 490.8 -39.38 138.19zm322.98 -628.99l-88.81 0 88.81 0zm0 0l-251.11 628.99 -33.01 -139.52 195.31 -489.47 88.81 0z"/>
|
||||
</g>
|
||||
<path class="fil2" d="M3449.37 834.49l0 -625.49 198.47 0c60.45,0 108.4,13.25 143.68,39.6 35.29,26.51 53.01,61.05 53.01,103.48 0,35.44 -10.13,66.26 -30.08,92.46 -20.1,26.21 -47.94,44.82 -83.38,55.84l0 1.78c43.03,4.92 77.42,20.85 103.18,47.65 25.76,26.95 38.57,61.94 38.57,104.97 0,53.45 -21,96.77 -63.13,129.98 -42.29,33.2 -95.44,49.73 -159.76,49.73l-200.56 0zm103.39 -534.59l0.2 136.51c0.04,24.18 -2.1,34.44 28.69,34.19l38.2 -0.3c36.03,-0.28 64.32,-8.48 84.87,-25.61 20.55,-16.97 30.67,-41.09 30.67,-72.21 0,-53.45 -35.73,-80.25 -107.2,-80.25l-66.2 0c-6.79,0 -9.25,0.65 -9.23,7.67zm-0.06 264.61l0 154.26c0,26.53 -0.42,32.78 28.85,32.78l59.74 0c38.56,0 68.49,-8.93 89.63,-26.8 21,-17.86 31.57,-42.43 31.57,-73.84 0,-64.92 -44.97,-97.38 -134.75,-97.38l-63.23 0c-8.81,0 -11.81,2.15 -11.81,10.98z"/>
|
||||
<path class="fil2" d="M3351.47 783.66c-63.06,34.27 -133.19,51.41 -210.51,51.41 -89.42,0 -161.85,-27.8 -217.29,-83.52 -55.43,-55.58 -83.08,-129.3 -83.08,-221.03 0,-93.59 30.24,-170.48 90.86,-230.53 60.62,-60.04 137.66,-90.14 230.96,-90.14 63.51,0 117.52,8.87 161.89,26.47 8.49,3.37 7.73,1.48 7.73,10.2l0 96.24c-46.36,-30.82 -101.65,-46.37 -165.73,-46.37 -64.51,0 -117.21,21.31 -158.39,63.65 -41.18,42.47 -61.92,97.48 -61.92,165.01 0,69.4 17.86,124.12 53.28,163.86 35.42,39.89 83.52,59.76 144.28,59.76 36.83,0 69.27,-6.32 97.3,-18.85 13.55,-6.06 10.69,-4.56 10.69,-19.42l0 -118.76c0,-9.57 -4.78,-10.86 -13.69,-10.86l-113.74 0 0 -66.84c0,-14.7 3.11,-17.54 18.89,-17.54l208.47 0 0 287.26z"/>
|
||||
<path class="fil2" d="M2773.73 821.5c-19.51,9.22 -45.03,13.84 -76.43,13.84 -87.14,0 -130.71,-41.78 -130.71,-125.47l0 -253.94 -37.51 0 0 -79.8 37.51 0 -0.08 -113.53c0,-7.1 2.14,-11.35 9.03,-14.4l92.74 -41.07 0.12 169 107.36 0 0 79.8 -107.36 0 0 224.59c0,26.65 4.79,45.66 14.37,57.04 9.74,11.38 25.76,17.07 48.37,17.07 17.22,0 32.04,-4.94 44.62,-14.82l0 78.4c0,2.65 0.39,2.15 -2.03,3.29z"/>
|
||||
<path class="fil2" d="M2464.86 837.31l-101.66 0 0 -245.7c0,-88.78 -29.8,-133.25 -89.09,-133.25 -29.79,0 -54.95,12.87 -75.46,38.63 -20.51,25.6 -30.69,58.54 -30.69,98.67l0 241.65 -102.27 0 0 -624.07c0,-5.64 3.31,-4.95 8.1,-4.95l90.21 0c2.37,0 3.96,-0.37 3.96,2.55l0 251.86 1.79 0c33.84,-56.74 82.5,-85.04 145.54,-85.04 99.71,0 149.57,60.94 149.57,182.81l0 276.84z"/>
|
||||
<path class="fil2" d="M2002.55 788.77c0,164.85 -82.95,247.35 -248.69,247.35 -58.55,0 -109.6,-9.73 -153.17,-29.35l83.52 -58.21c18.34,7.63 38.08,7.31 56.47,7.31 106.76,0 160.06,-52.56 160.06,-157.51l0 -49.12 -1.8 0c-33.54,57.35 -84.14,85.95 -151.67,85.95 -54.65,0 -98.82,-19.92 -132.36,-59.89 -33.39,-39.83 -50.16,-93.43 -50.16,-160.66 0,-76.36 17.97,-137 53.9,-182.06 36.09,-45.07 85.5,-67.53 148.38,-67.53 59.29,0 103.31,24.25 131.91,72.77l1.8 0 0 -49.6c0,-10.62 1.64,-12.69 12.25,-12.69l76.07 0c12.33,0 13.49,-0.94 13.49,11.4l0 401.84zm-100.91 -169.79l0 -58.39c0,-31.59 -10.48,-58.54 -31.3,-80.85 -20.96,-22.46 -47.01,-33.54 -78.3,-33.54 -38.63,0 -68.88,14.22 -90.59,42.67 -21.86,28.6 -32.79,68.43 -32.79,119.63 0,44.17 10.48,79.36 31.45,105.86 20.81,26.5 48.66,39.68 83.09,39.68 35.04,0 63.64,-12.58 85.5,-37.89 22.01,-25.3 32.94,-57.64 32.94,-97.17z"/>
|
||||
<path class="fil2" d="M1499.16 837.31l-101.81 0 0 -441.66c0,-5.84 1.24,-7.51 7.73,-7.51l84.14 0c8.97,0 9.94,-0.28 9.94,8.05l0 441.12z"/>
|
||||
<path class="fil2" d="M1447.36 304.24c-13.77,0 -25.56,-4.47 -35.36,-13.4 -9.93,-9.06 -14.77,-20.35 -14.77,-34.25 0,-13.77 4.84,-25.31 14.77,-34.49 9.8,-9.18 21.59,-13.77 35.36,-13.77 14.27,0 26.43,4.59 36.35,13.77 9.93,9.18 14.89,20.72 14.89,34.49 0,13.16 -4.96,24.32 -14.89,33.63 -9.92,9.31 -22.08,14.02 -36.35,14.02z"/>
|
||||
<path class="fil2" d="M984.92 837.31l0 -622.86c0,-6.28 2.75,-6.13 8.19,-6.13l88.48 0c5.21,0 7.69,-0.29 7.69,5.73l0 531.65c0,4.06 1.5,3.43 4.8,3.43l230.79 0c4.41,0 8.31,-0.52 8.31,5.46l0 82.64 -348.26 0.08z"/>
|
||||
<polygon class="fil3" points="0.02,1047.35 629.75,1047.35 629.75,644.28 "/>
|
||||
<polygon class="fil4" points="367.7,618.72 0,857.06 0,546.56 "/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 138 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.6 KiB |
@@ -0,0 +1,27 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Creator: CorelDRAW 2020 (64-Bit Evaluation Version) -->
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" width="1800px" height="2907px" version="1.1" style="shape-rendering:geometricPrecision; text-rendering:geometricPrecision; image-rendering:optimizeQuality; fill-rule:evenodd; clip-rule:evenodd"
|
||||
viewBox="0 0 1808.08 2919.85"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns:xodm="http://www.corel.com/coreldraw/odm/2003">
|
||||
<defs>
|
||||
<style type="text/css">
|
||||
<![CDATA[
|
||||
.fil4 {fill:none}
|
||||
.fil3 {fill:#1B9AD7}
|
||||
.fil1 {fill:#76B644}
|
||||
.fil0 {fill:#EF4927}
|
||||
.fil2 {fill:#FCB518}
|
||||
]]>
|
||||
</style>
|
||||
</defs>
|
||||
<g id="图层_x0020_1">
|
||||
<metadata id="CorelCorpID_0Corel-Layer"/>
|
||||
<polygon class="fil0" points="1732.73,84.18 71.14,84.18 71.14,1147.68 "/>
|
||||
<polygon class="fil1" points="762.56,1220.86 1732.77,591.98 1732.77,1411.27 "/>
|
||||
<polygon class="fil2" points="71.19,2847.67 1732.78,2847.67 1732.78,1784.16 "/>
|
||||
<polygon class="fil3" points="1041.36,1716.72 71.15,2345.59 71.15,1526.3 "/>
|
||||
<rect class="fil4" width="1808.08" height="2919.85"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 88 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 41 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.9 KiB |
@@ -0,0 +1,37 @@
|
||||
@ECHO OFF
|
||||
|
||||
pushd %~dp0
|
||||
|
||||
REM Command file for Sphinx documentation
|
||||
|
||||
if "%SPHINXBUILD%" == "" (
|
||||
set SPHINXBUILD=sphinx-build
|
||||
)
|
||||
set SOURCEDIR=.
|
||||
set BUILDDIR=_build
|
||||
set SPHINXPROJ=LightGBM
|
||||
set SPHINXOPTS=-W
|
||||
|
||||
if "%1" == "" goto help
|
||||
|
||||
%SPHINXBUILD% >NUL 2>NUL
|
||||
if errorlevel 9009 (
|
||||
echo.
|
||||
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
|
||||
echo.installed, then set the SPHINXBUILD environment variable to point
|
||||
echo.to the full path of the 'sphinx-build' executable. Alternatively you
|
||||
echo.may add the Sphinx directory to PATH.
|
||||
echo.
|
||||
echo.If you don't have Sphinx installed, grab it from
|
||||
echo.https://www.sphinx-doc.org/
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
|
||||
goto end
|
||||
|
||||
:help
|
||||
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
|
||||
|
||||
:end
|
||||
popd
|
||||
Reference in New Issue
Block a user