chore: import upstream snapshot with attribution

examples/python-guide/README.md

@@ -0,0 +1,66 @@
+Python-package Examples
+=======================
+
+Here is an example for LightGBM to use Python-package.
+
+You should install LightGBM [Python-package](https://github.com/lightgbm-org/LightGBM/tree/main/python-package) first.
+
+You also need scikit-learn, pandas, matplotlib (only for plot example), and scipy (only for logistic regression example) to run the examples, but they are not required for the package itself. You can install them with pip:
+
+```
+pip install scikit-learn pandas matplotlib scipy -U
+```
+
+Now you can run examples in this folder, for example:
+
+```
+python simple_example.py
+```
+
+Examples include:
+
+- [`dask/`](./dask): examples using Dask for distributed training
+- [simple_example.py](https://github.com/lightgbm-org/LightGBM/blob/main/examples/python-guide/simple_example.py)
+    - Construct Dataset
+    - Basic train and predict
+    - Eval during training
+    - Early stopping
+    - Save model to file
+- [sklearn_example.py](https://github.com/lightgbm-org/LightGBM/blob/main/examples/python-guide/sklearn_example.py)
+    - Create data for learning with sklearn interface
+    - Basic train and predict with sklearn interface
+    - Feature importances with sklearn interface
+    - Self-defined eval metric with sklearn interface
+    - Find best parameters for the model with sklearn's GridSearchCV
+- [advanced_example.py](https://github.com/lightgbm-org/LightGBM/blob/main/examples/python-guide/advanced_example.py)
+    - Construct Dataset
+    - Set feature names
+    - Directly use categorical features without one-hot encoding
+    - Save model to file
+    - Dump model to JSON format
+    - Get feature names
+    - Get feature importances
+    - Load model to predict
+    - Dump and load model with pickle
+    - Load model file to continue training
+    - Change learning rates during training
+    - Change any parameters during training
+    - Self-defined objective function
+    - Self-defined eval metric
+    - Callback function
+- [logistic_regression.py](https://github.com/lightgbm-org/LightGBM/blob/main/examples/python-guide/logistic_regression.py)
+    - Use objective `xentropy` or `binary`
+    - Use `xentropy` with binary labels or probability labels
+    - Use `binary` only with binary labels
+    - Compare speed of `xentropy` versus `binary`
+- [plot_example.py](https://github.com/lightgbm-org/LightGBM/blob/main/examples/python-guide/plot_example.py)
+    - Construct Dataset
+    - Train and record eval results for further plotting
+    - Plot metrics recorded during training
+    - Plot feature importances
+    - Plot split value histogram
+    - Plot one specified tree
+    - Plot one specified tree with Graphviz
+- [dataset_from_multi_hdf5.py](https://github.com/lightgbm-org/LightGBM/blob/main/examples/python-guide/dataset_from_multi_hdf5.py)
+  - Construct Dataset from multiple HDF5 files
+  - Avoid loading all data into memory

examples/python-guide/advanced_example.py

@@ -0,0 +1,220 @@
+# coding: utf-8
+import copy
+import json
+import pickle
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import roc_auc_score
+
+import lightgbm as lgb
+
+print("Loading data...")
+# load or create your dataset
+binary_example_dir = Path(__file__).absolute().parents[1] / "binary_classification"
+df_train = pd.read_csv(str(binary_example_dir / "binary.train"), header=None, sep="\t")
+df_test = pd.read_csv(str(binary_example_dir / "binary.test"), header=None, sep="\t")
+W_train = pd.read_csv(str(binary_example_dir / "binary.train.weight"), header=None)[0]
+W_test = pd.read_csv(str(binary_example_dir / "binary.test.weight"), header=None)[0]
+
+y_train = df_train[0]
+y_test = df_test[0]
+X_train = df_train.drop(0, axis=1)
+X_test = df_test.drop(0, axis=1)
+
+num_train, num_feature = X_train.shape
+
+# generate feature names
+feature_name = [f"feature_{col}" for col in range(num_feature)]
+
+# create dataset for lightgbm
+# if you want to re-use data, remember to set free_raw_data=False
+lgb_train = lgb.Dataset(
+    X_train, y_train, weight=W_train, feature_name=feature_name, categorical_feature=[21], free_raw_data=False
+)
+lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, weight=W_test, free_raw_data=False)
+
+# specify your configurations as a dict
+params = {
+    "boosting_type": "gbdt",
+    "objective": "binary",
+    "metric": "binary_logloss",
+    "num_leaves": 31,
+    "learning_rate": 0.05,
+    "feature_fraction": 0.9,
+    "bagging_fraction": 0.8,
+    "bagging_freq": 5,
+    "verbose": 0,
+}
+
+print("Starting training...")
+# feature_name and categorical_feature
+gbm = lgb.train(
+    params,
+    lgb_train,
+    num_boost_round=10,
+    valid_sets=lgb_train,  # eval training data
+)
+
+print("Finished first 10 rounds...")
+# check feature name
+print(f"7th feature name is: {lgb_train.feature_name[6]}")
+
+print("Saving model...")
+# save model to file
+gbm.save_model("model.txt")
+
+print("Dumping model to JSON...")
+# dump model to JSON (and save to file)
+model_json = gbm.dump_model()
+
+with open("model.json", "w+") as f:
+    json.dump(model_json, f, indent=4)
+
+# feature names
+print(f"Feature names: {gbm.feature_name()}")
+
+# feature importances
+print(f"Feature importances: {list(gbm.feature_importance())}")
+
+print("Loading model to predict...")
+# load model to predict
+bst = lgb.Booster(model_file="model.txt")
+# can only predict with the best iteration (or the saving iteration)
+y_pred = bst.predict(X_test)
+# eval with loaded model
+auc_loaded_model = roc_auc_score(y_test, y_pred)
+print(f"The ROC AUC of loaded model's prediction is: {auc_loaded_model}")
+
+print("Dumping and loading model with pickle...")
+# dump model with pickle
+with open("model.pkl", "wb") as fout:
+    pickle.dump(gbm, fout)
+# load model with pickle to predict
+with open("model.pkl", "rb") as fin:
+    pkl_bst = pickle.load(fin)
+# can predict with any iteration when loaded in pickle way
+y_pred = pkl_bst.predict(X_test, num_iteration=7)
+# eval with loaded model
+auc_pickled_model = roc_auc_score(y_test, y_pred)
+print(f"The ROC AUC of pickled model's prediction is: {auc_pickled_model}")
+
+# continue training
+# init_model accepts:
+# 1. model file name
+# 2. Booster()
+gbm = lgb.train(params, lgb_train, num_boost_round=10, init_model="model.txt", valid_sets=lgb_eval)
+
+print("Finished 10 - 20 rounds with model file...")
+
+# decay learning rates
+# reset_parameter callback accepts:
+# 1. list with length = num_boost_round
+# 2. function(curr_iter)
+gbm = lgb.train(
+    params,
+    lgb_train,
+    num_boost_round=10,
+    init_model=gbm,
+    valid_sets=lgb_eval,
+    callbacks=[lgb.reset_parameter(learning_rate=lambda iter: 0.05 * (0.99**iter))],
+)
+
+print("Finished 20 - 30 rounds with decay learning rates...")
+
+# change other parameters during training
+gbm = lgb.train(
+    params,
+    lgb_train,
+    num_boost_round=10,
+    init_model=gbm,
+    valid_sets=lgb_eval,
+    callbacks=[lgb.reset_parameter(bagging_fraction=[0.7] * 5 + [0.6] * 5)],
+)
+
+print("Finished 30 - 40 rounds with changing bagging_fraction...")
+
+
+# self-defined objective function
+# f(preds: array, train_data: Dataset) -> grad: array, hess: array
+# log likelihood loss
+def loglikelihood(preds, train_data):
+    labels = train_data.get_label()
+    preds = 1.0 / (1.0 + np.exp(-preds))
+    grad = preds - labels
+    hess = preds * (1.0 - preds)
+    return grad, hess
+
+
+# self-defined eval metric
+# f(preds: array, train_data: Dataset) -> metric_name: str, metric_value: float, maximize: bool
+# binary error
+# NOTE: when you do customized loss function, the default prediction value is margin
+# This may make built-in evaluation metric calculate wrong results
+# For example, we are doing log likelihood loss, the prediction is score before logistic transformation
+# Keep this in mind when you use the customization
+def binary_error(preds, train_data):
+    labels = train_data.get_label()
+    preds = 1.0 / (1.0 + np.exp(-preds))
+    return "error", np.mean(labels != (preds > 0.5)), False
+
+
+# Pass custom objective function through params
+params_custom_obj = copy.deepcopy(params)
+params_custom_obj["objective"] = loglikelihood
+
+gbm = lgb.train(
+    params_custom_obj, lgb_train, num_boost_round=10, init_model=gbm, feval=binary_error, valid_sets=lgb_eval
+)
+
+print("Finished 40 - 50 rounds with self-defined objective function and eval metric...")
+
+
+# another self-defined eval metric
+# f(preds: array, train_data: Dataset) -> metric_name: str, metric_value: float, maximize: bool
+# accuracy
+# NOTE: when you do customized loss function, the default prediction value is margin
+# This may make built-in evaluation metric calculate wrong results
+# For example, we are doing log likelihood loss, the prediction is score before logistic transformation
+# Keep this in mind when you use the customization
+def accuracy(preds, train_data):
+    labels = train_data.get_label()
+    preds = 1.0 / (1.0 + np.exp(-preds))
+    return "accuracy", np.mean(labels == (preds > 0.5)), True
+
+
+# Pass custom objective function through params
+params_custom_obj = copy.deepcopy(params)
+params_custom_obj["objective"] = loglikelihood
+
+gbm = lgb.train(
+    params_custom_obj,
+    lgb_train,
+    num_boost_round=10,
+    init_model=gbm,
+    feval=[binary_error, accuracy],
+    valid_sets=lgb_eval,
+)
+
+print("Finished 50 - 60 rounds with self-defined objective function and multiple self-defined eval metrics...")
+
+print("Starting a new training job...")
+
+
+# callback
+def reset_metrics():
+    def callback(env):
+        lgb_eval_new = lgb.Dataset(X_test, y_test, reference=lgb_train)
+        if env.iteration - env.begin_iteration == 5:
+            print("Add a new valid dataset at iteration 5...")
+            env.model.add_valid(lgb_eval_new, "new_valid")
+
+    callback.before_iteration = True
+    callback.order = 0
+    return callback
+
+
+gbm = lgb.train(params, lgb_train, num_boost_round=10, valid_sets=lgb_train, callbacks=[reset_metrics()])
+
+print("Finished first 10 rounds with callback function...")

examples/python-guide/dask/README.md

@@ -0,0 +1,25 @@
+Dask Examples
+=============
+
+This directory contains examples of machine learning workflows with LightGBM and [Dask](https://dask.org/).
+
+Before running this code, see [the installation instructions for the Dask-package](https://github.com/lightgbm-org/LightGBM/tree/main/python-package#install-dask-package).
+
+After installing the package and its dependencies, any of the examples here can be run with a command like this:
+
+```shell
+python binary-classification.py
+```
+
+The examples listed below contain minimal code showing how to train LightGBM models using Dask.
+
+**Training**
+
+* [binary-classification.py](./binary-classification.py)
+* [multiclass-classification.py](./multiclass-classification.py)
+* [ranking.py](./ranking.py)
+* [regression.py](./regression.py)
+
+**Prediction**
+
+* [prediction.py](./prediction.py)

examples/python-guide/dask/binary-classification.py

@@ -0,0 +1,30 @@
+import dask.array as da
+from distributed import Client, LocalCluster
+from sklearn.datasets import make_blobs
+
+import lightgbm as lgb
+
+if __name__ == "__main__":
+    print("loading data")
+
+    X, y = make_blobs(n_samples=1000, n_features=50, centers=2)
+
+    print("initializing a Dask cluster")
+
+    cluster = LocalCluster()
+    client = Client(cluster)
+
+    print("created a Dask LocalCluster")
+
+    print("distributing training data on the Dask cluster")
+
+    dX = da.from_array(X, chunks=(100, 50))
+    dy = da.from_array(y, chunks=(100,))
+
+    print("beginning training")
+
+    dask_model = lgb.DaskLGBMClassifier(n_estimators=10)
+    dask_model.fit(dX, dy)
+    assert dask_model.fitted_
+
+    print("done training")

examples/python-guide/dask/multiclass-classification.py

@@ -0,0 +1,30 @@
+import dask.array as da
+from distributed import Client, LocalCluster
+from sklearn.datasets import make_blobs
+
+import lightgbm as lgb
+
+if __name__ == "__main__":
+    print("loading data")
+
+    X, y = make_blobs(n_samples=1000, n_features=50, centers=3)
+
+    print("initializing a Dask cluster")
+
+    cluster = LocalCluster(n_workers=2)
+    client = Client(cluster)
+
+    print("created a Dask LocalCluster")
+
+    print("distributing training data on the Dask cluster")
+
+    dX = da.from_array(X, chunks=(100, 50))
+    dy = da.from_array(y, chunks=(100,))
+
+    print("beginning training")
+
+    dask_model = lgb.DaskLGBMClassifier(n_estimators=10)
+    dask_model.fit(dX, dy)
+    assert dask_model.fitted_
+
+    print("done training")

examples/python-guide/dask/prediction.py

@@ -0,0 +1,48 @@
+import dask.array as da
+from distributed import Client, LocalCluster
+from sklearn.datasets import make_regression
+from sklearn.metrics import mean_squared_error
+
+import lightgbm as lgb
+
+if __name__ == "__main__":
+    print("loading data")
+
+    X, y = make_regression(n_samples=1000, n_features=50)
+
+    print("initializing a Dask cluster")
+
+    cluster = LocalCluster(n_workers=2)
+    client = Client(cluster)
+
+    print("created a Dask LocalCluster")
+
+    print("distributing training data on the Dask cluster")
+
+    dX = da.from_array(X, chunks=(100, 50))
+    dy = da.from_array(y, chunks=(100,))
+
+    print("beginning training")
+
+    dask_model = lgb.DaskLGBMRegressor(n_estimators=10)
+    dask_model.fit(dX, dy)
+    assert dask_model.fitted_
+
+    print("done training")
+
+    print("predicting on the training data")
+
+    preds = dask_model.predict(dX)
+
+    # the code below uses sklearn.metrics, but this requires pulling all of the
+    # predictions and target values back from workers to the client
+    #
+    # for larger datasets, consider the metrics from dask-ml instead
+    # https://ml.dask.org/modules/api.html#dask-ml-metrics-metrics
+    print("computing MSE")
+
+    preds_local = preds.compute()
+    actuals_local = dy.compute()
+    mse = mean_squared_error(actuals_local, preds_local)
+
+    print(f"MSE: {mse}")

examples/python-guide/dask/ranking.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+import dask.array as da
+import numpy as np
+from distributed import Client, LocalCluster
+from sklearn.datasets import load_svmlight_file
+
+import lightgbm as lgb
+
+if __name__ == "__main__":
+    print("loading data")
+
+    rank_example_dir = Path(__file__).absolute().parents[2] / "lambdarank"
+    X, y = load_svmlight_file(str(rank_example_dir / "rank.train"))
+    group = np.loadtxt(str(rank_example_dir / "rank.train.query"))
+
+    print("initializing a Dask cluster")
+
+    cluster = LocalCluster(n_workers=2)
+    client = Client(cluster)
+
+    print("created a Dask LocalCluster")
+
+    print("distributing training data on the Dask cluster")
+
+    # split training data into two partitions
+    rows_in_part1 = int(np.sum(group[:100]))
+    rows_in_part2 = X.shape[0] - rows_in_part1
+    num_features = X.shape[1]
+
+    # make this array dense because we're splitting across
+    # a sparse boundary to partition the data
+    X = X.toarray()
+
+    dX = da.from_array(x=X, chunks=[(rows_in_part1, rows_in_part2), (num_features,)])
+    dy = da.from_array(
+        x=y,
+        chunks=[
+            (rows_in_part1, rows_in_part2),
+        ],
+    )
+    dg = da.from_array(x=group, chunks=[(100, group.size - 100)])
+
+    print("beginning training")
+
+    dask_model = lgb.DaskLGBMRanker(n_estimators=10)
+    dask_model.fit(dX, dy, group=dg)
+    assert dask_model.fitted_
+
+    print("done training")

examples/python-guide/dask/regression.py

@@ -0,0 +1,30 @@
+import dask.array as da
+from distributed import Client, LocalCluster
+from sklearn.datasets import make_regression
+
+import lightgbm as lgb
+
+if __name__ == "__main__":
+    print("loading data")
+
+    X, y = make_regression(n_samples=1000, n_features=50)
+
+    print("initializing a Dask cluster")
+
+    cluster = LocalCluster(n_workers=2)
+    client = Client(cluster)
+
+    print("created a Dask LocalCluster")
+
+    print("distributing training data on the Dask cluster")
+
+    dX = da.from_array(X, chunks=(100, 50))
+    dy = da.from_array(y, chunks=(100,))
+
+    print("beginning training")
+
+    dask_model = lgb.DaskLGBMRegressor(n_estimators=10)
+    dask_model.fit(dX, dy)
+    assert dask_model.fitted_
+
+    print("done training")

examples/python-guide/dataset_from_multi_hdf5.py

@@ -0,0 +1,110 @@
+from pathlib import Path
+
+import h5py
+import numpy as np
+import pandas as pd
+
+import lightgbm as lgb
+
+
+class HDFSequence(lgb.Sequence):
+    def __init__(self, hdf_dataset, batch_size):
+        """
+        Construct a sequence object from HDF5 with required interface.
+
+        Parameters
+        ----------
+        hdf_dataset : h5py.Dataset
+            Dataset in HDF5 file.
+        batch_size : int
+            Size of a batch. When reading data to construct lightgbm Dataset, each read reads batch_size rows.
+        """
+        # We can also open HDF5 file once and get access to
+        self.data = hdf_dataset
+        self.batch_size = batch_size
+
+    def __getitem__(self, idx):
+        return self.data[idx]
+
+    def __len__(self):
+        return len(self.data)
+
+
+def create_dataset_from_multiple_hdf(input_flist, batch_size):
+    data = []
+    ylist = []
+    for f in input_flist:
+        f = h5py.File(f, "r")
+        data.append(HDFSequence(f["X"], batch_size))
+        ylist.append(f["Y"][:])
+
+    params = {
+        "bin_construct_sample_cnt": 200000,
+        "max_bin": 255,
+    }
+    y = np.concatenate(ylist)
+    dataset = lgb.Dataset(data, label=y, params=params)
+    # With binary dataset created, we can use either Python API or cmdline version to train.
+    #
+    # Note: in order to create exactly the same dataset with the one created in simple_example.py, we need
+    # to modify simple_example.py to pass numpy array instead of pandas DataFrame to Dataset constructor.
+    # The reason is that DataFrame column names will be used in Dataset. For a DataFrame with Int64Index
+    # as columns, Dataset will use column names like ["0", "1", "2", ...]. While for numpy array, column names
+    # are using the default one assigned in C++ code (dataset_loader.cpp), like ["Column_0", "Column_1", ...].
+    dataset.save_binary("regression.train.from_hdf.bin")
+
+
+def save2hdf(input_data, fname, batch_size):
+    """Store numpy array to HDF5 file.
+
+    Please note chunk size settings in the implementation for I/O performance optimization.
+    """
+    with h5py.File(fname, "w") as f:
+        for name, data in input_data.items():
+            nrow, ncol = data.shape
+            if ncol == 1:
+                # Y has a single column and we read it in single shot. So store it as an 1-d array.
+                chunk = (nrow,)
+                data = data.values.flatten()
+            else:
+                # We use random access for data sampling when creating LightGBM Dataset from Sequence.
+                # When accessing any element in a HDF5 chunk, it's read entirely.
+                # To save I/O for sampling, we should keep number of total chunks much larger than sample count.
+                # Here we are just creating a chunk size that matches with batch_size.
+                #
+                # Also note that the data is stored in row major order to avoid extra copy when passing to
+                # lightgbm Dataset.
+                chunk = (batch_size, ncol)
+            f.create_dataset(name, data=data, chunks=chunk, compression="lzf")
+
+
+def generate_hdf(input_fname, output_basename, batch_size):
+    # Save to 2 HDF5 files for demonstration.
+    df = pd.read_csv(input_fname, header=None, sep="\t")
+
+    mid = len(df) // 2
+    df1 = df.iloc[:mid]
+    df2 = df.iloc[mid:]
+
+    # We can store multiple datasets inside a single HDF5 file.
+    # Separating X and Y for choosing best chunk size for data loading.
+    fname1 = f"{output_basename}1.h5"
+    fname2 = f"{output_basename}2.h5"
+    save2hdf({"Y": df1.iloc[:, :1], "X": df1.iloc[:, 1:]}, fname1, batch_size)
+    save2hdf({"Y": df2.iloc[:, :1], "X": df2.iloc[:, 1:]}, fname2, batch_size)
+
+    return [fname1, fname2]
+
+
+def main():
+    batch_size = 64
+    output_basename = "regression"
+    hdf_files = generate_hdf(
+        str(Path(__file__).absolute().parents[1] / "regression" / "regression.train"), output_basename, batch_size
+    )
+
+    create_dataset_from_multiple_hdf(hdf_files, batch_size=batch_size)
+
+
+if __name__ == "__main__":
+    main()

examples/python-guide/logistic_regression.py

@@ -0,0 +1,101 @@
+# coding: utf-8
+"""Comparison of `binary` and `xentropy` objectives.
+
+BLUF: The `xentropy` objective does logistic regression and generalizes
+to the case where labels are probabilistic (i.e. numbers between 0 and 1).
+
+Details: Both `binary` and `xentropy` minimize the log loss and use
+`boost_from_average = TRUE` by default. Possibly the only difference
+between them with default settings is that `binary` may achieve a slight
+speed improvement by assuming that the labels are binary instead of
+probabilistic.
+"""
+
+import time
+
+import numpy as np
+import pandas as pd
+from scipy.special import expit
+
+import lightgbm as lgb
+
+#################
+# Simulate some binary data with a single categorical and
+#   single continuous predictor
+rng = np.random.default_rng(seed=0)
+N = 1000
+X = pd.DataFrame({"continuous": range(N), "categorical": np.repeat([0, 1, 2, 3, 4], N / 5)})
+CATEGORICAL_EFFECTS = [-1, -1, -2, -2, 2]
+LINEAR_TERM = np.array(
+    [-0.5 + 0.01 * X["continuous"][k] + CATEGORICAL_EFFECTS[X["categorical"][k]] for k in range(X.shape[0])]
+) + rng.normal(loc=0, scale=1, size=X.shape[0])
+TRUE_PROB = expit(LINEAR_TERM)
+Y = rng.binomial(n=1, p=TRUE_PROB, size=N)
+DATA = {
+    "X": X,
+    "probability_labels": TRUE_PROB,
+    "binary_labels": Y,
+    "lgb_with_binary_labels": lgb.Dataset(X, Y),
+    "lgb_with_probability_labels": lgb.Dataset(X, TRUE_PROB),
+}
+
+
+#################
+# Set up a couple of utilities for our experiments
+def log_loss(preds, labels):
+    """Logarithmic loss with non-necessarily-binary labels."""
+    log_likelihood = np.sum(labels * np.log(preds)) / len(preds)
+    return -log_likelihood
+
+
+def experiment(objective, label_type, data):
+    """Measure performance of an objective.
+
+    Parameters
+    ----------
+    objective : {'binary', 'xentropy'}
+        Objective function.
+    label_type : {'binary', 'probability'}
+        Type of the label.
+    data : dict
+        Data for training.
+
+    Returns
+    -------
+    result : dict
+        Experiment summary stats.
+    """
+    nrounds = 5
+    lgb_data = data[f"lgb_with_{label_type}_labels"]
+    params = {"objective": objective, "feature_fraction": 1, "bagging_fraction": 1, "verbose": -1, "seed": 123}
+    time_zero = time.time()
+    gbm = lgb.train(params, lgb_data, num_boost_round=nrounds)
+    y_fitted = gbm.predict(data["X"])
+    y_true = data[f"{label_type}_labels"]
+    duration = time.time() - time_zero
+    return {"time": duration, "correlation": np.corrcoef(y_fitted, y_true)[0, 1], "logloss": log_loss(y_fitted, y_true)}
+
+
+#################
+# Observe the behavior of `binary` and `xentropy` objectives
+print("Performance of `binary` objective with binary labels:")
+print(experiment("binary", label_type="binary", data=DATA))
+
+print("Performance of `xentropy` objective with binary labels:")
+print(experiment("xentropy", label_type="binary", data=DATA))
+
+print("Performance of `xentropy` objective with probability labels:")
+print(experiment("xentropy", label_type="probability", data=DATA))
+
+# Trying this throws an error on non-binary values of y:
+#   experiment('binary', label_type='probability', DATA)
+
+# The speed of `binary` is not drastically different than
+#   `xentropy`. `xentropy` runs faster than `binary` in many cases, although
+#   there are reasons to suspect that `binary` should run faster when the
+#   label is an integer instead of a float
+K = 10
+A = [experiment("binary", label_type="binary", data=DATA)["time"] for k in range(K)]
+B = [experiment("xentropy", label_type="binary", data=DATA)["time"] for k in range(K)]
+print(f"Best `binary` time: {min(A)}")
+print(f"Best `xentropy` time: {min(B)}")

examples/python-guide/plot_example.py

@@ -0,0 +1,62 @@
+# coding: utf-8
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import pandas as pd
+
+import lightgbm as lgb
+
+print("Loading data...")
+# load or create your dataset
+regression_example_dir = Path(__file__).absolute().parents[1] / "regression"
+df_train = pd.read_csv(str(regression_example_dir / "regression.train"), header=None, sep="\t")
+df_test = pd.read_csv(str(regression_example_dir / "regression.test"), header=None, sep="\t")
+
+y_train = df_train[0]
+y_test = df_test[0]
+X_train = df_train.drop(0, axis=1)
+X_test = df_test.drop(0, axis=1)
+
+# create dataset for lightgbm
+lgb_train = lgb.Dataset(
+    X_train,
+    y_train,
+    feature_name=[f"f{i + 1}" for i in range(X_train.shape[-1])],
+    categorical_feature=[21],
+)
+lgb_test = lgb.Dataset(X_test, y_test, reference=lgb_train)
+
+# specify your configurations as a dict
+params = {"num_leaves": 5, "metric": ("l1", "l2"), "verbose": 0}
+
+evals_result = {}  # to record eval results for plotting
+
+print("Starting training...")
+# train
+gbm = lgb.train(
+    params,
+    lgb_train,
+    num_boost_round=100,
+    valid_sets=[lgb_train, lgb_test],
+    callbacks=[lgb.log_evaluation(10), lgb.record_evaluation(evals_result)],
+)
+
+print("Plotting metrics recorded during training...")
+ax = lgb.plot_metric(evals_result, metric="l1")
+plt.show()
+
+print("Plotting feature importances...")
+ax = lgb.plot_importance(gbm, max_num_features=10)
+plt.show()
+
+print("Plotting split value histogram...")
+ax = lgb.plot_split_value_histogram(gbm, feature="f26", bins="auto")
+plt.show()
+
+print("Plotting 54th tree...")  # one tree use categorical feature to split
+ax = lgb.plot_tree(gbm, tree_index=53, figsize=(15, 15), show_info=["split_gain"])
+plt.show()
+
+print("Plotting 54th tree with graphviz...")
+graph = lgb.create_tree_digraph(gbm, tree_index=53, name="Tree54")
+graph.render(view=True)

examples/python-guide/simple_example.py

@@ -0,0 +1,52 @@
+# coding: utf-8
+from pathlib import Path
+
+import pandas as pd
+from sklearn.metrics import mean_squared_error
+
+import lightgbm as lgb
+
+print("Loading data...")
+# load or create your dataset
+regression_example_dir = Path(__file__).absolute().parents[1] / "regression"
+df_train = pd.read_csv(str(regression_example_dir / "regression.train"), header=None, sep="\t")
+df_test = pd.read_csv(str(regression_example_dir / "regression.test"), header=None, sep="\t")
+
+y_train = df_train[0]
+y_test = df_test[0]
+X_train = df_train.drop(0, axis=1)
+X_test = df_test.drop(0, axis=1)
+
+# create dataset for lightgbm
+lgb_train = lgb.Dataset(X_train, y_train)
+lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
+
+# specify your configurations as a dict
+params = {
+    "boosting_type": "gbdt",
+    "objective": "regression",
+    "metric": {"l2", "l1"},
+    "num_leaves": 31,
+    "learning_rate": 0.05,
+    "feature_fraction": 0.9,
+    "bagging_fraction": 0.8,
+    "bagging_freq": 5,
+    "verbose": 0,
+}
+
+print("Starting training...")
+# train
+gbm = lgb.train(
+    params, lgb_train, num_boost_round=20, valid_sets=lgb_eval, callbacks=[lgb.early_stopping(stopping_rounds=5)]
+)
+
+print("Saving model...")
+# save model to file
+gbm.save_model("model.txt")
+
+print("Starting predicting...")
+# predict
+y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)
+# eval
+rmse_test = mean_squared_error(y_test, y_pred) ** 0.5
+print(f"The RMSE of prediction is: {rmse_test}")

examples/python-guide/sklearn_example.py

@@ -0,0 +1,80 @@
+# coding: utf-8
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import mean_squared_error
+from sklearn.model_selection import GridSearchCV
+
+import lightgbm as lgb
+
+print("Loading data...")
+# load or create your dataset
+regression_example_dir = Path(__file__).absolute().parents[1] / "regression"
+df_train = pd.read_csv(str(regression_example_dir / "regression.train"), header=None, sep="\t")
+df_test = pd.read_csv(str(regression_example_dir / "regression.test"), header=None, sep="\t")
+
+y_train = df_train[0]
+y_test = df_test[0]
+X_train = df_train.drop(0, axis=1)
+X_test = df_test.drop(0, axis=1)
+
+print("Starting training...")
+# train
+gbm = lgb.LGBMRegressor(num_leaves=31, learning_rate=0.05, n_estimators=20)
+gbm.fit(X_train, y_train, eval_X=(X_test,), eval_y=(y_test,), eval_metric="l1", callbacks=[lgb.early_stopping(5)])
+
+print("Starting predicting...")
+# predict
+y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration_)
+# eval
+rmse_test = mean_squared_error(y_test, y_pred) ** 0.5
+print(f"The RMSE of prediction is: {rmse_test}")
+
+# feature importances
+print(f"Feature importances: {list(gbm.feature_importances_)}")
+
+
+# self-defined eval metric
+# f(y_true: array, y_pred: array) -> metric_name: str, metric_value: float, maximize: bool
+# Root Mean Squared Logarithmic Error (RMSLE)
+def rmsle(y_true, y_pred):
+    return "RMSLE", np.sqrt(np.mean(np.power(np.log1p(y_pred) - np.log1p(y_true), 2))), False
+
+
+print("Starting training with custom eval function...")
+# train
+gbm.fit(X_train, y_train, eval_X=(X_test,), eval_y=(y_test,), eval_metric=rmsle, callbacks=[lgb.early_stopping(5)])
+
+
+# another self-defined eval metric
+# f(y_true: array, y_pred: array) -> metric_name: str, metric_value: float, maximize: bool
+# Relative Absolute Error (RAE)
+def rae(y_true, y_pred):
+    return "RAE", np.sum(np.abs(y_pred - y_true)) / np.sum(np.abs(np.mean(y_true) - y_true)), False
+
+
+print("Starting training with multiple custom eval functions...")
+# train
+gbm.fit(
+    X_train, y_train, eval_X=(X_test,), eval_y=(y_test,), eval_metric=[rmsle, rae], callbacks=[lgb.early_stopping(5)]
+)
+
+print("Starting predicting...")
+# predict
+y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration_)
+# eval
+rmsle_test = rmsle(y_test, y_pred)[1]
+rae_test = rae(y_test, y_pred)[1]
+print(f"The RMSLE of prediction is: {rmsle_test}")
+print(f"The RAE of prediction is: {rae_test}")
+
+# other scikit-learn modules
+estimator = lgb.LGBMRegressor(num_leaves=31)
+
+param_grid = {"learning_rate": [0.01, 0.1, 1], "n_estimators": [20, 40]}
+
+gbm = GridSearchCV(estimator, param_grid, cv=3)
+gbm.fit(X_train, y_train)
+
+print(f"Best parameters found by grid search are: {gbm.best_params_}")