wehub/lightgbm-org--lightgbm
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:27:18 +08:00
commit d72d1a58f0
553 changed files with 214565 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/application/application.cpp
+296
View File
@@ -0,0 +1,296 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/application.h>
#include <LightGBM/boosting.h>
#include <LightGBM/dataset.h>
#include <LightGBM/dataset_loader.h>
#include <LightGBM/metric.h>
#include <LightGBM/network.h>
#include <LightGBM/objective_function.h>
#include <LightGBM/prediction_early_stop.h>
#include <LightGBM/cuda/vector_cudahost.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <LightGBM/utils/text_reader.h>
#include <chrono>
#include <cstdio>
#include <ctime>
#include <fstream>
#include <memory>
#include <sstream>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "predictor.hpp"
namespace LightGBM {
Application::Application(int argc, char** argv) {
LoadParameters(argc, argv);
// set number of threads for openmp
OMP_SET_NUM_THREADS(config_.num_threads);
if (config_.data.size() == 0 && config_.task != TaskType::kConvertModel) {
Log::Fatal("No training/prediction data, application quit");
}
if (config_.device_type == std::string("cuda")) {
LGBM_config_::current_device = lgbm_device_cuda;
}
}
Application::~Application() {
if (config_.is_parallel) {
Network::Dispose();
}
}
void Application::LoadParameters(int argc, char** argv) {
std::unordered_map<std::string, std::vector<std::string>> all_params;
std::unordered_map<std::string, std::string> params;
for (int i = 1; i < argc; ++i) {
Config::KV2Map(&all_params, argv[i]);
}
// read parameters from config file
bool config_file_ok = true;
if (all_params.count("config") > 0) {
TextReader<size_t> config_reader(all_params["config"][0].c_str(), false);
config_reader.ReadAllLines();
if (!config_reader.Lines().empty()) {
for (auto& line : config_reader.Lines()) {
// remove str after "#"
if (line.size() > 0 && std::string::npos != line.find_first_of("#")) {
line.erase(line.find_first_of("#"));
}
line = Common::Trim(line);
if (line.size() == 0) {
continue;
}
Config::KV2Map(&all_params, line.c_str());
}
} else {
config_file_ok = false;
}
}
Config::SetVerbosity(all_params);
// de-duplicate params
Config::KeepFirstValues(all_params, &params);
if (!config_file_ok) {
Log::Warning("Config file %s doesn't exist, will ignore", params["config"].c_str());
}
ParameterAlias::KeyAliasTransform(&params);
config_.Set(params);
Log::Info("Finished loading parameters");
}
void Application::LoadData() {
auto start_time = std::chrono::high_resolution_clock::now();
std::unique_ptr<Predictor> predictor;
// prediction is needed if using input initial model(continued train)
PredictFunction predict_fun = nullptr;
// need to continue training
if (boosting_->NumberOfTotalModel() > 0 && config_.task != TaskType::KRefitTree) {
predictor.reset(new Predictor(boosting_.get(), 0, -1, true, false, false, false, -1, -1));
predict_fun = predictor->GetPredictFunction();
}
// sync up random seed for data partition
if (config_.is_data_based_parallel) {
config_.data_random_seed = Network::GlobalSyncUpByMin(config_.data_random_seed);
}
Log::Debug("Loading train file...");
DatasetLoader dataset_loader(config_, predict_fun,
config_.num_class, config_.data.c_str());
// load Training data
if (config_.is_data_based_parallel) {
// load data for distributed training
train_data_.reset(dataset_loader.LoadFromFile(config_.data.c_str(),
Network::rank(), Network::num_machines()));
} else {
// load data for single machine
train_data_.reset(dataset_loader.LoadFromFile(config_.data.c_str(), 0, 1));
}
// need save binary file
if (config_.save_binary) {
train_data_->SaveBinaryFile(nullptr);
}
// create training metric
if (config_.is_provide_training_metric) {
for (auto metric_type : config_.metric) {
auto metric = std::unique_ptr<Metric>(Metric::CreateMetric(metric_type, config_));
if (metric == nullptr) {
continue;
}
metric->Init(train_data_->metadata(), train_data_->num_data());
train_metric_.push_back(std::move(metric));
}
}
train_metric_.shrink_to_fit();
if (!config_.metric.empty()) {
// only when have metrics then need to construct validation data
// Add validation data, if it exists
for (size_t i = 0; i < config_.valid.size(); ++i) {
Log::Debug("Loading validation file #%zu...", (i + 1));
// add
auto new_dataset = std::unique_ptr<Dataset>(
dataset_loader.LoadFromFileAlignWithOtherDataset(
config_.valid[i].c_str(),
train_data_.get()));
valid_datas_.push_back(std::move(new_dataset));
// need save binary file
if (config_.save_binary) {
valid_datas_.back()->SaveBinaryFile(nullptr);
}
// add metric for validation data
valid_metrics_.emplace_back();
for (auto metric_type : config_.metric) {
auto metric = std::unique_ptr<Metric>(Metric::CreateMetric(metric_type, config_));
if (metric == nullptr) {
continue;
}
metric->Init(valid_datas_.back()->metadata(),
valid_datas_.back()->num_data());
valid_metrics_.back().push_back(std::move(metric));
}
valid_metrics_.back().shrink_to_fit();
}
valid_datas_.shrink_to_fit();
valid_metrics_.shrink_to_fit();
}
auto end_time = std::chrono::high_resolution_clock::now();
// output used time on each iteration
Log::Info("Finished loading data in %f seconds",
std::chrono::duration<double, std::milli>(end_time - start_time) * 1e-3);
}
void Application::InitTrain() {
if (config_.is_parallel) {
// need init network
Network::Init(config_);
Log::Info("Finished initializing network");
config_.feature_fraction_seed =
Network::GlobalSyncUpByMin(config_.feature_fraction_seed);
config_.feature_fraction =
Network::GlobalSyncUpByMin(config_.feature_fraction);
config_.drop_seed =
Network::GlobalSyncUpByMin(config_.drop_seed);
}
// create boosting
boosting_.reset(
Boosting::CreateBoosting(config_.boosting,
config_.input_model.c_str(), config_.device_type, config_.num_gpu));
// create objective function
objective_fun_.reset(
ObjectiveFunction::CreateObjectiveFunction(config_.objective,
config_));
// load training data
LoadData();
if (config_.task == TaskType::kSaveBinary) {
Log::Info("Save data as binary finished, exit");
exit(0);
}
// initialize the objective function
objective_fun_->Init(train_data_->metadata(), train_data_->num_data());
// initialize the boosting
boosting_->Init(&config_, train_data_.get(), objective_fun_.get(),
Common::ConstPtrInVectorWrapper<Metric>(train_metric_));
// add validation data into boosting
for (size_t i = 0; i < valid_datas_.size(); ++i) {
boosting_->AddValidDataset(valid_datas_[i].get(),
Common::ConstPtrInVectorWrapper<Metric>(valid_metrics_[i]));
Log::Debug("Number of data points in validation set #%zu: %d", i + 1, valid_datas_[i]->num_data());
}
Log::Info("Finished initializing training");
}
void Application::Train() {
Log::Info("Started training...");
boosting_->Train(config_.snapshot_freq, config_.output_model);
boosting_->SaveModelToFile(0, -1, config_.saved_feature_importance_type,
config_.output_model.c_str());
// convert model to if-else statement code
if (config_.convert_model_language == std::string("cpp")) {
boosting_->SaveModelToIfElse(-1, config_.convert_model.c_str());
}
Log::Info("Finished training");
}
void Application::Predict() {
if (config_.task == TaskType::KRefitTree) {
// create predictor
Predictor predictor(boosting_.get(), 0, -1, false, true, false, false, 1, 1);
predictor.Predict(config_.data.c_str(), config_.output_result.c_str(), config_.header, config_.predict_disable_shape_check,
config_.precise_float_parser);
TextReader<int> result_reader(config_.output_result.c_str(), false);
result_reader.ReadAllLines();
size_t nrow = result_reader.Lines().size();
size_t ncol = 0;
if (nrow > 0) {
ncol = Common::StringToArray<int>(result_reader.Lines()[0], '\t').size();
}
std::vector<int> pred_leaf;
pred_leaf.resize(nrow * ncol);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int irow = 0; irow < static_cast<int>(nrow); ++irow) {
auto line_vec = Common::StringToArray<int>(result_reader.Lines()[irow], '\t');
CHECK_EQ(line_vec.size(), ncol);
for (int i_row_item = 0; i_row_item < static_cast<int>(ncol); ++i_row_item) {
pred_leaf[irow * ncol + i_row_item] = line_vec[i_row_item];
}
// Free memory
result_reader.Lines()[irow].clear();
}
DatasetLoader dataset_loader(config_, nullptr,
config_.num_class, config_.data.c_str());
train_data_.reset(dataset_loader.LoadFromFile(config_.data.c_str(), 0, 1));
train_metric_.clear();
objective_fun_.reset(ObjectiveFunction::CreateObjectiveFunction(config_.objective,
config_));
objective_fun_->Init(train_data_->metadata(), train_data_->num_data());
boosting_->Init(&config_, train_data_.get(), objective_fun_.get(),
Common::ConstPtrInVectorWrapper<Metric>(train_metric_));
boosting_->RefitTree(pred_leaf.data(), nrow, ncol);
boosting_->SaveModelToFile(0, -1, config_.saved_feature_importance_type,
config_.output_model.c_str());
Log::Info("Finished RefitTree");
} else {
// create predictor
Predictor predictor(boosting_.get(), config_.start_iteration_predict, config_.num_iteration_predict, config_.predict_raw_score,
config_.predict_leaf_index, config_.predict_contrib,
config_.pred_early_stop, config_.pred_early_stop_freq,
config_.pred_early_stop_margin);
predictor.Predict(config_.data.c_str(),
config_.output_result.c_str(), config_.header, config_.predict_disable_shape_check,
config_.precise_float_parser);
Log::Info("Finished prediction");
}
}
void Application::InitPredict() {
boosting_.reset(
Boosting::CreateBoosting("gbdt", config_.input_model.c_str(), config_.device_type, config_.num_gpu));
Log::Info("Finished initializing prediction, total used %d iterations", boosting_->GetCurrentIteration());
}
void Application::ConvertModel() {
boosting_.reset(
Boosting::CreateBoosting(config_.boosting, config_.input_model.c_str(), config_.device_type, config_.num_gpu));
boosting_->SaveModelToIfElse(-1, config_.convert_model.c_str());
}
} // namespace LightGBM
src/application/predictor.hpp
+303
View File
@@ -0,0 +1,303 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_APPLICATION_PREDICTOR_HPP_
#define LIGHTGBM_SRC_APPLICATION_PREDICTOR_HPP_
#include <LightGBM/boosting.h>
#include <LightGBM/dataset.h>
#include <LightGBM/meta.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <LightGBM/utils/text_reader.h>
#include <string>
#include <cstdio>
#include <cstring>
#include <functional>
#include <map>
#include <memory>
#include <unordered_map>
#include <utility>
#include <vector>
namespace LightGBM {
/*!
* \brief Used to predict data with input model
*/
class Predictor {
public:
/*!
* \brief Constructor
* \param boosting Input boosting model
* \param start_iteration Start index of the iteration to predict
* \param num_iteration Number of boosting round
* \param is_raw_score True if need to predict result with raw score
* \param predict_leaf_index True to output leaf index instead of prediction score
* \param predict_contrib True to output feature contributions instead of prediction score
*/
Predictor(Boosting* boosting, int start_iteration, int num_iteration, bool is_raw_score,
bool predict_leaf_index, bool predict_contrib, bool early_stop,
int early_stop_freq, double early_stop_margin) {
early_stop_ = CreatePredictionEarlyStopInstance(
"none", LightGBM::PredictionEarlyStopConfig());
if (early_stop && !boosting->NeedAccuratePrediction()) {
PredictionEarlyStopConfig pred_early_stop_config;
CHECK_GT(early_stop_freq, 0);
CHECK_GE(early_stop_margin, 0);
pred_early_stop_config.margin_threshold = early_stop_margin;
pred_early_stop_config.round_period = early_stop_freq;
if (boosting->NumberOfClasses() == 1) {
early_stop_ =
CreatePredictionEarlyStopInstance("binary", pred_early_stop_config);
} else {
early_stop_ = CreatePredictionEarlyStopInstance("multiclass",
pred_early_stop_config);
}
}
boosting->InitPredict(start_iteration, num_iteration, predict_contrib);
boosting_ = boosting;
num_pred_one_row_ = boosting_->NumPredictOneRow(start_iteration,
num_iteration, predict_leaf_index, predict_contrib);
num_feature_ = boosting_->MaxFeatureIdx() + 1;
predict_buf_.resize(
OMP_NUM_THREADS(),
std::vector<double, Common::AlignmentAllocator<double, kAlignedSize>>(
num_feature_, 0.0f));
const int kFeatureThreshold = 100000;
const size_t KSparseThreshold = static_cast<size_t>(0.01 * num_feature_);
if (predict_leaf_index) {
predict_fun_ = [=](const std::vector<std::pair<int, double>>& features,
double* output) {
int tid = omp_get_thread_num();
if (num_feature_ > kFeatureThreshold &&
features.size() < KSparseThreshold) {
auto buf = CopyToPredictMap(features);
boosting_->PredictLeafIndexByMap(buf, output);
} else {
CopyToPredictBuffer(predict_buf_[tid].data(), features);
// get result for leaf index
boosting_->PredictLeafIndex(predict_buf_[tid].data(), output);
ClearPredictBuffer(predict_buf_[tid].data(), predict_buf_[tid].size(),
features);
}
};
} else if (predict_contrib) {
if (boosting_->IsLinear()) {
Log::Fatal("Predicting SHAP feature contributions is not implemented for linear trees.");
}
predict_fun_ = [=](const std::vector<std::pair<int, double>>& features,
double* output) {
int tid = omp_get_thread_num();
CopyToPredictBuffer(predict_buf_[tid].data(), features);
// get feature importances
boosting_->PredictContrib(predict_buf_[tid].data(), output);
ClearPredictBuffer(predict_buf_[tid].data(), predict_buf_[tid].size(),
features);
};
predict_sparse_fun_ = [=](const std::vector<std::pair<int, double>>& features,
std::vector<std::unordered_map<int, double>>* output) {
auto buf = CopyToPredictMap(features);
// get sparse feature importances
boosting_->PredictContribByMap(buf, output);
};
} else {
if (is_raw_score) {
predict_fun_ = [=](const std::vector<std::pair<int, double>>& features,
double* output) {
int tid = omp_get_thread_num();
if (num_feature_ > kFeatureThreshold &&
features.size() < KSparseThreshold) {
auto buf = CopyToPredictMap(features);
boosting_->PredictRawByMap(buf, output, &early_stop_);
} else {
CopyToPredictBuffer(predict_buf_[tid].data(), features);
boosting_->PredictRaw(predict_buf_[tid].data(), output,
&early_stop_);
ClearPredictBuffer(predict_buf_[tid].data(),
predict_buf_[tid].size(), features);
}
};
} else {
predict_fun_ = [=](const std::vector<std::pair<int, double>>& features,
double* output) {
int tid = omp_get_thread_num();
if (num_feature_ > kFeatureThreshold &&
features.size() < KSparseThreshold) {
auto buf = CopyToPredictMap(features);
boosting_->PredictByMap(buf, output, &early_stop_);
} else {
CopyToPredictBuffer(predict_buf_[tid].data(), features);
boosting_->Predict(predict_buf_[tid].data(), output, &early_stop_);
ClearPredictBuffer(predict_buf_[tid].data(),
predict_buf_[tid].size(), features);
}
};
}
}
}
/*!
* \brief Destructor
*/
~Predictor() {
}
inline const PredictFunction& GetPredictFunction() const {
return predict_fun_;
}
inline const PredictSparseFunction& GetPredictSparseFunction() const {
return predict_sparse_fun_;
}
/*!
* \brief predicting on data, then saving result to disk
* \param data_filename Filename of data
* \param result_filename Filename of output result
*/
void Predict(const char* data_filename, const char* result_filename, bool header, bool disable_shape_check, bool precise_float_parser) {
auto writer = VirtualFileWriter::Make(result_filename);
if (!writer->Init()) {
Log::Fatal("Prediction results file %s cannot be created", result_filename);
}
auto label_idx = header ? -1 : boosting_->LabelIdx();
auto parser = std::unique_ptr<Parser>(Parser::CreateParser(data_filename, header, boosting_->MaxFeatureIdx() + 1, label_idx,
precise_float_parser, boosting_->ParserConfigStr()));
if (parser == nullptr) {
Log::Fatal("Could not recognize the data format of data file %s", data_filename);
}
if (!header && !disable_shape_check && parser->NumFeatures() != boosting_->MaxFeatureIdx() + 1) {
Log::Fatal("The number of features in data (%d) is not the same as it was in training data (%d).\n" \
"You can set ``predict_disable_shape_check=true`` to discard this error, but please be aware what you are doing.", parser->NumFeatures(), boosting_->MaxFeatureIdx() + 1);
}
TextReader<data_size_t> predict_data_reader(data_filename, header);
std::vector<int> feature_remapper(parser->NumFeatures(), -1);
bool need_adjust = false;
// skip raw feature remapping if trained model has parser config str which may contain actual feature names.
if (header && boosting_->ParserConfigStr().empty()) {
std::string first_line = predict_data_reader.first_line();
std::vector<std::string> header_words = Common::Split(first_line.c_str(), "\t,");
std::unordered_map<std::string, int> header_mapper;
for (int i = 0; i < static_cast<int>(header_words.size()); ++i) {
if (header_mapper.count(header_words[i]) > 0) {
Log::Fatal("Feature (%s) appears more than one time.", header_words[i].c_str());
}
header_mapper[header_words[i]] = i;
}
const auto& fnames = boosting_->FeatureNames();
for (int i = 0; i < static_cast<int>(fnames.size()); ++i) {
if (header_mapper.count(fnames[i]) <= 0) {
Log::Warning("Feature (%s) is missed in data file. If it is weight/query/group/ignore_column, you can ignore this warning.", fnames[i].c_str());
} else {
feature_remapper[header_mapper.at(fnames[i])] = i;
}
}
for (int i = 0; i < static_cast<int>(feature_remapper.size()); ++i) {
if (feature_remapper[i] >= 0 && i != feature_remapper[i]) {
need_adjust = true;
break;
}
}
}
// function for parse data
std::function<void(const char*, std::vector<std::pair<int, double>>*)> parser_fun;
double tmp_label;
parser_fun = [&parser, &feature_remapper, &tmp_label, need_adjust]
(const char* buffer, std::vector<std::pair<int, double>>* feature) {
parser->ParseOneLine(buffer, feature, &tmp_label);
if (need_adjust) {
int i = 0, j = static_cast<int>(feature->size());
while (i < j) {
if (feature_remapper[(*feature)[i].first] >= 0) {
(*feature)[i].first = feature_remapper[(*feature)[i].first];
++i;
} else {
// move the non-used features to the end of the feature vector
std::swap((*feature)[i], (*feature)[--j]);
}
}
feature->resize(i);
}
};
std::function<void(data_size_t, const std::vector<std::string>&)>
process_fun = [&parser_fun, &writer, this](
data_size_t, const std::vector<std::string>& lines) {
std::vector<std::pair<int, double>> oneline_features;
std::vector<std::string> result_to_write(lines.size());
OMP_INIT_EX();
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) firstprivate(oneline_features)
for (data_size_t i = 0; i < static_cast<data_size_t>(lines.size()); ++i) {
OMP_LOOP_EX_BEGIN();
oneline_features.clear();
// parser
parser_fun(lines[i].c_str(), &oneline_features);
// predict
std::vector<double> result(num_pred_one_row_);
predict_fun_(oneline_features, result.data());
auto str_result = Common::Join<double>(result, "\t");
result_to_write[i] = str_result;
OMP_LOOP_EX_END();
}
OMP_THROW_EX();
for (data_size_t i = 0; i < static_cast<data_size_t>(result_to_write.size()); ++i) {
writer->Write(result_to_write[i].c_str(), result_to_write[i].size());
writer->Write("\n", 1);
}
};
predict_data_reader.ReadAllAndProcessParallel(process_fun);
}
private:
void CopyToPredictBuffer(double* pred_buf, const std::vector<std::pair<int, double>>& features) {
for (const auto &feature : features) {
if (feature.first < num_feature_) {
pred_buf[feature.first] = feature.second;
}
}
}
void ClearPredictBuffer(double* pred_buf, size_t buf_size, const std::vector<std::pair<int, double>>& features) {
if (features.size() > static_cast<size_t>(buf_size / 2)) {
std::memset(pred_buf, 0, sizeof(double)*(buf_size));
} else {
for (const auto &feature : features) {
if (feature.first < num_feature_) {
pred_buf[feature.first] = 0.0f;
}
}
}
}
std::unordered_map<int, double> CopyToPredictMap(const std::vector<std::pair<int, double>>& features) {
std::unordered_map<int, double> buf;
for (const auto &feature : features) {
if (feature.first < num_feature_) {
buf[feature.first] = feature.second;
}
}
return buf;
}
/*! \brief Boosting model */
const Boosting* boosting_;
/*! \brief function for prediction */
PredictFunction predict_fun_;
PredictSparseFunction predict_sparse_fun_;
PredictionEarlyStopInstance early_stop_;
int num_feature_;
int num_pred_one_row_;
std::vector<std::vector<double, Common::AlignmentAllocator<double, kAlignedSize>>> predict_buf_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_APPLICATION_PREDICTOR_HPP_
src/arrow/array.hpp
+413
View File
@@ -0,0 +1,413 @@
/*!
* Copyright (c) 2026-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*
* Author: Oliver Borchert
*/
#ifndef LIGHTGBM_SRC_ARROW_ARRAY_HPP_
#define LIGHTGBM_SRC_ARROW_ARRAY_HPP_
#include <LightGBM/arrow.h>
#include <LightGBM/utils/log.h>
#include <algorithm>
#include <limits>
#include <string>
#include <utility>
#include <vector>
#include <nanoarrow/nanoarrow.hpp>
namespace LightGBM {
/* ------------------------------------------- UTILS ------------------------------------------- */
inline struct ArrowSchemaView MakeSchemaView(const struct ArrowSchema* schema) {
struct ArrowSchemaView view;
struct ArrowError error;
auto ret = ArrowSchemaViewInit(&view, schema, &error);
if (ret != NANOARROW_OK) {
throw nanoarrow::Exception("Failed to initialize ArrowSchemaView: " +
std::string(ArrowErrorMessage(&error)));
}
return view;
}
/* --------------------------------------------------------------------------------------------- */
/* CHUNKED ARRAY */
/* --------------------------------------------------------------------------------------------- */
class ArrowChunkedArray {
enum ArrowType type_;
nanoarrow::UniqueSchema schema_;
std::vector<nanoarrow::UniqueArray> chunks_;
public:
class View;
template <typename ArrowT, typename OutputT>
class Visitor;
template <typename ArrowT, typename OutputT>
class Iterator;
/**
* @brief Construct a new chunked Arrow array from an array stream.
* The stream is consumed and the chunked Arrow array takes ownership of the schema and all
* chunks. Upon destruction, the release callback is called for the schema and all chunks.
*
* @param stream The Arrow array stream to consume.
*/
explicit ArrowChunkedArray(ArrowArrayStream* stream) {
nanoarrow::UniqueArrayStream stream_(stream);
// Extract the schema
auto ret = stream_->get_schema(stream_.get(), schema_.get());
if (ret != NANOARROW_OK) {
throw nanoarrow::Exception("Failed to get schema from Arrow array stream: " +
std::string(stream_->get_last_error(stream_.get())));
}
// Turn the schema into a type
type_ = MakeSchemaView(schema_.get()).type;
// Extract all chunks
while (true) {
nanoarrow::UniqueArray chunk;
auto ret = stream_->get_next(stream_.get(), chunk.get());
if (ret != NANOARROW_OK) {
throw nanoarrow::Exception("Failed to get next chunk from Arrow array stream: " +
std::string(stream_->get_last_error(stream_.get())));
}
if (chunk->release == nullptr) break;
chunks_.emplace_back(std::move(chunk));
}
}
/**
* @brief Construct a new chunked Arrow array from a list of Arrow arrays and a schema.
* The chunked Arrow array takes ownership of the schema and all chunks. Upon destruction,
* the release callback is called for the schema and all chunks.
*
* @param n_chunks The number of Arrow arrays.
* @param chunks Pointer to the list of Arrow arrays.
* @param schema Pointer to the schema of all Arrow arrays.
*/
explicit ArrowChunkedArray(int64_t n_chunks, struct ArrowArray* chunks,
struct ArrowSchema* schema) {
// Take ownership of schema
schema_ = nanoarrow::UniqueSchema(schema);
type_ = MakeSchemaView(schema_.get()).type;
// Take ownership of chunks
chunks_.reserve(n_chunks);
for (int64_t i = 0; i < n_chunks; ++i) {
chunks_.emplace_back(&chunks[i]);
}
}
/**
* @brief Whether the chunked array is a struct and has multiple fields.
* A struct chunked array is typically interpreted as a table.
*/
bool is_struct() const { return type_ == NANOARROW_TYPE_STRUCT; }
/**
* @brief Get the length of the chunked array as the sum of all chunk lengths.
*
* @return int64_t The total number of elements.
*/
int64_t get_length() const {
int64_t length = 0;
for (const auto& chunk : chunks_) {
length += chunk->length;
}
return length;
}
/**
* @brief Get the number of fields in the chunked array.
*
* @return int64_t The number of fields.
*/
int64_t get_num_fields() const {
if (!is_struct()) {
Log::Fatal("Expected struct type for array, got %s", ArrowTypeString(type_));
}
return schema_.get()->n_children;
}
/**
* @brief Obtain a view on the chunked array to visit its values.
*
* @return View The view on the array.
*/
View view() const {
std::vector<const struct ArrowArray*> chunk_ptrs;
chunk_ptrs.reserve(chunks_.size());
for (const auto& chunk : chunks_) {
// Skip empty chunks to avoid additional complexity in the iterator
if (chunk->length == 0) continue;
chunk_ptrs.push_back(chunk.get());
}
return View(type_, schema_.get(), std::move(chunk_ptrs));
}
/* ------------------------------------------------------------------------------------------- */
/* VIEW */
/* ------------------------------------------------------------------------------------------- */
class View {
friend class ArrowChunkedArray;
enum ArrowType type_;
const struct ArrowSchema* schema_;
std::vector<const struct ArrowArray*> chunks_;
View(enum ArrowType type, const struct ArrowSchema* schema,
std::vector<const struct ArrowArray*> chunks)
: type_(type), schema_(schema), chunks_(std::move(chunks)) {}
public:
explicit View(std::vector<View> views) {
type_ = views[0].type_;
schema_ = views[0].schema_;
for (auto it = views.begin(), end = views.end(); it != end; ++it) {
if (it->type_ != type_) {
Log::Fatal("All views must have the same type, but got %s and %s",
ArrowTypeString(it->type_), ArrowTypeString(type_));
}
chunks_.insert(chunks_.end(), it->chunks_.begin(), it->chunks_.end());
}
}
/**
* @brief Obtain a view on the field at the given index.
* This method assumes that the view has a type "struct".
*
* @param field_idx The index of the field to view.
* @return View A view on the field at the given index.
*/
View field(int64_t field_idx) const {
if (type_ != NANOARROW_TYPE_STRUCT) {
Log::Fatal("Expected struct type for array, got %s", ArrowTypeString(type_));
}
std::vector<const struct ArrowArray*> chunk_ptrs;
chunk_ptrs.reserve(chunks_.size());
for (const auto& chunk : chunks_) {
chunk_ptrs.push_back(chunk->children[field_idx]);
}
auto type = MakeSchemaView(schema_->children[field_idx]).type;
return View(type, schema_->children[field_idx], std::move(chunk_ptrs));
}
/**
* @brief Visit the chunked array with a visitor created from the schema type.
* The visitor allows accessing all values from the chunked array, casting to the desired
* output type.
*
* @tparam OutputT The desired output type.
* @tparam F The type of the visitor function, which must be invocable with a
* `Visitor<ArrowT, OutputT>`
* @param f The visitor function to invoke with the created visitor.
* @return decltype(auto) The result of invoking the visitor function with the created visitor.
*/
template <typename OutputT, typename F>
decltype(auto) visit(F&& f) const {
// Switch on the schema type and construct the appropriate visitor
switch (type_) {
case NANOARROW_TYPE_INT8:
return f(Visitor<int8_t, OutputT>(chunks_));
case NANOARROW_TYPE_INT16:
return f(Visitor<int16_t, OutputT>(chunks_));
case NANOARROW_TYPE_INT32:
return f(Visitor<int32_t, OutputT>(chunks_));
case NANOARROW_TYPE_INT64:
return f(Visitor<int64_t, OutputT>(chunks_));
case NANOARROW_TYPE_UINT8:
return f(Visitor<uint8_t, OutputT>(chunks_));
case NANOARROW_TYPE_UINT16:
return f(Visitor<uint16_t, OutputT>(chunks_));
case NANOARROW_TYPE_UINT32:
return f(Visitor<uint32_t, OutputT>(chunks_));
case NANOARROW_TYPE_UINT64:
return f(Visitor<uint64_t, OutputT>(chunks_));
case NANOARROW_TYPE_FLOAT:
return f(Visitor<float, OutputT>(chunks_));
case NANOARROW_TYPE_DOUBLE:
return f(Visitor<double, OutputT>(chunks_));
case NANOARROW_TYPE_BOOL:
return f(Visitor<bool, OutputT>(chunks_));
default:
Log::Fatal("Unsupported Arrow type: %s", ArrowTypeString(type_));
}
}
};
/* ------------------------------------------------------------------------------------------- */
/* VISITOR */
/* ------------------------------------------------------------------------------------------- */
template <typename ArrowT, typename OutputT>
class Visitor {
friend class View;
friend class Iterator<ArrowT, OutputT>;
std::vector<const struct ArrowArray*> chunks_;
std::vector<int64_t> chunk_offsets_;
explicit Visitor(std::vector<const struct ArrowArray*> chunks) : chunks_(chunks) {
// Derive chunk offsets
chunk_offsets_.reserve(chunks.size() + 1);
chunk_offsets_.push_back(0);
for (const auto& chunk : chunks) {
chunk_offsets_.push_back(chunk_offsets_.back() + chunk->length);
}
}
ArrowT is_valid(int64_t chunk_idx, int64_t element_idx) const {
auto arr = chunks_[chunk_idx];
return arr->buffers[0] == nullptr ||
ArrowBitGet(static_cast<const uint8_t*>(arr->buffers[0]), arr->offset + element_idx);
}
ArrowT get(int64_t chunk_idx, int64_t element_idx) const {
auto arr = chunks_[chunk_idx];
auto idx = arr->offset + element_idx;
if constexpr (std::is_same_v<ArrowT, bool>) {
return ArrowBitGet(static_cast<const uint8_t*>(arr->buffers[1]), idx);
} else {
return static_cast<const ArrowT*>(arr->buffers[1])[idx];
}
}
public:
Iterator<ArrowT, OutputT> begin() const { return Iterator<ArrowT, OutputT>(*this, 0, 0); }
Iterator<ArrowT, OutputT> end() const {
return Iterator<ArrowT, OutputT>(*this, chunks_.size(), 0);
}
};
/* ------------------------------------------------------------------------------------------- */
/* ITERATOR */
/* ------------------------------------------------------------------------------------------- */
template <typename ArrowT, typename OutputT>
class Iterator {
friend class Visitor<ArrowT, OutputT>;
const Visitor<ArrowT, OutputT>& visitor_;
int64_t chunk_idx_;
int64_t element_idx_;
Iterator(const Visitor<ArrowT, OutputT>& visitor, int64_t chunk_idx, int64_t element_idx)
: visitor_(visitor), chunk_idx_(chunk_idx), element_idx_(element_idx) {}
int64_t full_offset() const { return visitor_.chunk_offsets_[chunk_idx_] + element_idx_; }
public:
using iterator_category = std::random_access_iterator_tag;
using difference_type = int64_t;
using value_type = OutputT;
using pointer = value_type*;
using reference = value_type&;
/* --------------------------------------- OPERATORS --------------------------------------- */
public:
Iterator<ArrowT, OutputT>& operator++() {
if (element_idx_ + 1 >= visitor_.chunks_[chunk_idx_]->length) {
element_idx_ = 0;
chunk_idx_++;
} else {
element_idx_++;
}
return *this;
}
Iterator<ArrowT, OutputT>& operator+=(int64_t c) {
while (element_idx_ + c >= visitor_.chunks_[chunk_idx_]->length) {
c -= visitor_.chunks_[chunk_idx_]->length - element_idx_;
element_idx_ = 0;
chunk_idx_++;
}
element_idx_ += c;
return *this;
}
Iterator<ArrowT, OutputT>& operator--() {
if (element_idx_ == 0) {
chunk_idx_--;
element_idx_ = visitor_.chunks_[chunk_idx_]->length - 1;
} else {
element_idx_--;
}
return *this;
}
Iterator<ArrowT, OutputT>& operator-=(int64_t c) {
while (c > element_idx_) {
c -= element_idx_ + 1;
chunk_idx_--;
element_idx_ = visitor_.chunks_[chunk_idx_]->length - 1;
}
element_idx_ -= c;
return *this;
}
friend int64_t operator-(const Iterator<ArrowT, OutputT>& a,
const Iterator<ArrowT, OutputT>& b) {
auto full_offset_a = a.full_offset();
auto full_offset_b = b.full_offset();
return full_offset_a - full_offset_b;
}
friend bool operator==(const Iterator<ArrowT, OutputT>& a,
const Iterator<ArrowT, OutputT>& b) {
return a.chunk_idx_ == b.chunk_idx_ && a.element_idx_ == b.element_idx_;
}
friend bool operator!=(const Iterator<ArrowT, OutputT>& a,
const Iterator<ArrowT, OutputT>& b) {
return !(a == b);
}
/* ----------------------------------------- VALUE ----------------------------------------- */
private:
static constexpr OutputT null_default() {
if constexpr (std::is_floating_point_v<OutputT>) {
return std::numeric_limits<OutputT>::quiet_NaN();
} else {
return OutputT{0};
}
}
OutputT get(int64_t chunk_idx, int64_t element_idx) const {
if (visitor_.is_valid(chunk_idx, element_idx)) {
return static_cast<OutputT>(visitor_.get(chunk_idx, element_idx));
} else {
return null_default();
}
}
public:
OutputT operator*() const { return this->get(chunk_idx_, element_idx_); }
OutputT operator[](int64_t c) const {
if (visitor_.chunks_.size() == 1) {
return this->get(0, c);
}
auto it =
std::upper_bound(visitor_.chunk_offsets_.begin(), visitor_.chunk_offsets_.end(), c);
auto chunk_idx = std::distance(visitor_.chunk_offsets_.begin(), it) - 1;
auto element_idx = c - visitor_.chunk_offsets_[chunk_idx];
return this->get(chunk_idx, element_idx);
}
};
};
}; // namespace LightGBM
#endif // LIGHTGBM_SRC_ARROW_ARRAY_HPP_
src/boosting/bagging.hpp
+297
View File
@@ -0,0 +1,297 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_BOOSTING_BAGGING_HPP_
#define LIGHTGBM_SRC_BOOSTING_BAGGING_HPP_
#include <string>
#include <vector>
namespace LightGBM {
class BaggingSampleStrategy : public SampleStrategy {
public:
BaggingSampleStrategy(const Config* config, const Dataset* train_data, const ObjectiveFunction* objective_function, int num_tree_per_iteration)
: need_re_bagging_(false) {
config_ = config;
train_data_ = train_data;
num_data_ = train_data->num_data();
num_queries_ = train_data->metadata().num_queries();
query_boundaries_ = train_data->metadata().query_boundaries();
objective_function_ = objective_function;
num_tree_per_iteration_ = num_tree_per_iteration;
num_threads_ = OMP_NUM_THREADS();
}
~BaggingSampleStrategy() {}
void Bagging(int iter, TreeLearner* tree_learner, score_t* /*gradients*/, score_t* /*hessians*/) override {
Common::FunctionTimer fun_timer("GBDT::Bagging", global_timer);
// if need bagging
if ((bag_data_cnt_ < num_data_ && iter % config_->bagging_freq == 0) ||
need_re_bagging_) {
need_re_bagging_ = false;
if (!config_->bagging_by_query) {
auto left_cnt = bagging_runner_.Run<true>(
num_data_,
[=](int, data_size_t cur_start, data_size_t cur_cnt, data_size_t* left,
data_size_t*) {
data_size_t cur_left_count = 0;
if (balanced_bagging_) {
cur_left_count =
BalancedBaggingHelper(cur_start, cur_cnt, left);
} else {
cur_left_count = BaggingHelper(cur_start, cur_cnt, left);
}
return cur_left_count;
},
bag_data_indices_.data());
bag_data_cnt_ = left_cnt;
} else {
num_sampled_queries_ = bagging_runner_.Run<true>(
num_queries_,
[=](int, data_size_t cur_start, data_size_t cur_cnt, data_size_t* left,
data_size_t*) {
data_size_t cur_left_count = 0;
cur_left_count = BaggingHelper(cur_start, cur_cnt, left);
return cur_left_count;
}, bag_query_indices_.data());
sampled_query_boundaries_[0] = 0;
OMP_INIT_EX();
#pragma omp parallel for schedule(static) num_threads(num_threads_)
for (data_size_t i = 0; i < num_sampled_queries_; ++i) {
OMP_LOOP_EX_BEGIN();
sampled_query_boundaries_[i + 1] = query_boundaries_[bag_query_indices_[i] + 1] - query_boundaries_[bag_query_indices_[i]];
OMP_LOOP_EX_END();
}
OMP_THROW_EX();
const int num_blocks = Threading::For<data_size_t>(0, num_sampled_queries_ + 1, 128, [this](int thread_index, data_size_t start_index, data_size_t end_index) {
for (data_size_t i = start_index + 1; i < end_index; ++i) {
sampled_query_boundaries_[i] += sampled_query_boundaries_[i - 1];
}
sampled_query_boundaries_thread_buffer_[thread_index] = sampled_query_boundaries_[end_index - 1];
});
for (int thread_index = 1; thread_index < num_blocks; ++thread_index) {
sampled_query_boundaries_thread_buffer_[thread_index] += sampled_query_boundaries_thread_buffer_[thread_index - 1];
}
Threading::For<data_size_t>(0, num_sampled_queries_ + 1, 128, [this](int thread_index, data_size_t start_index, data_size_t end_index) {
if (thread_index > 0) {
for (data_size_t i = start_index; i < end_index; ++i) {
sampled_query_boundaries_[i] += sampled_query_boundaries_thread_buffer_[thread_index - 1];
}
}
});
bag_data_cnt_ = sampled_query_boundaries_[num_sampled_queries_];
Threading::For<data_size_t>(0, num_sampled_queries_, 1, [this](int /*thread_index*/, data_size_t start_index, data_size_t end_index) {
for (data_size_t sampled_query_id = start_index; sampled_query_id < end_index; ++sampled_query_id) {
const data_size_t query_index = bag_query_indices_[sampled_query_id];
const data_size_t data_index_start = query_boundaries_[query_index];
const data_size_t data_index_end = query_boundaries_[query_index + 1];
const data_size_t sampled_query_start = sampled_query_boundaries_[sampled_query_id];
for (data_size_t i = data_index_start; i < data_index_end; ++i) {
bag_data_indices_[sampled_query_start + i - data_index_start] = i;
}
}
});
}
Log::Debug("Re-bagging, using %d data to train", bag_data_cnt_);
// set bagging data to tree learner
if (!is_use_subset_) {
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda")) {
CopyFromHostToCUDADevice<data_size_t>(cuda_bag_data_indices_.RawData(), bag_data_indices_.data(), static_cast<size_t>(num_data_), __FILE__, __LINE__);
tree_learner->SetBaggingData(nullptr, cuda_bag_data_indices_.RawData(), bag_data_cnt_);
} else {
#endif // USE_CUDA
tree_learner->SetBaggingData(nullptr, bag_data_indices_.data(), bag_data_cnt_);
#ifdef USE_CUDA
}
#endif // USE_CUDA
} else {
// get subset
tmp_subset_->ReSize(bag_data_cnt_);
tmp_subset_->CopySubrow(train_data_, bag_data_indices_.data(),
bag_data_cnt_, false);
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda")) {
CopyFromHostToCUDADevice<data_size_t>(cuda_bag_data_indices_.RawData(), bag_data_indices_.data(), static_cast<size_t>(num_data_), __FILE__, __LINE__);
tree_learner->SetBaggingData(tmp_subset_.get(), cuda_bag_data_indices_.RawData(),
bag_data_cnt_);
} else {
#endif // USE_CUDA
tree_learner->SetBaggingData(tmp_subset_.get(), bag_data_indices_.data(),
bag_data_cnt_);
#ifdef USE_CUDA
}
#endif // USE_CUDA
}
}
}
void ResetSampleConfig(const Config* config, bool is_change_dataset) override {
need_resize_gradients_ = false;
// if need bagging, create buffer
data_size_t num_pos_data = 0;
if (objective_function_ != nullptr) {
num_pos_data = objective_function_->NumPositiveData();
}
bool balance_bagging_cond = (config->pos_bagging_fraction < 1.0 || config->neg_bagging_fraction < 1.0) && (num_pos_data > 0);
if ((config->bagging_fraction < 1.0 || balance_bagging_cond) && config->bagging_freq > 0) {
need_re_bagging_ = false;
if (!is_change_dataset &&
config_ != nullptr && config_->bagging_fraction == config->bagging_fraction && config_->bagging_freq == config->bagging_freq
&& config_->pos_bagging_fraction == config->pos_bagging_fraction && config_->neg_bagging_fraction == config->neg_bagging_fraction) {
config_ = config;
return;
}
config_ = config;
if (balance_bagging_cond) {
balanced_bagging_ = true;
bag_data_cnt_ = static_cast<data_size_t>(num_pos_data * config_->pos_bagging_fraction)
+ static_cast<data_size_t>((num_data_ - num_pos_data) * config_->neg_bagging_fraction);
} else {
bag_data_cnt_ = static_cast<data_size_t>(config_->bagging_fraction * num_data_);
}
bag_data_indices_.resize(num_data_);
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda")) {
cuda_bag_data_indices_.Resize(num_data_);
}
#endif // USE_CUDA
if (!config_->bagging_by_query) {
bagging_runner_.ReSize(num_data_);
} else {
bagging_runner_.ReSize(num_queries_);
sampled_query_boundaries_.resize(num_queries_ + 1, 0);
sampled_query_boundaries_thread_buffer_.resize(num_threads_, 0);
bag_query_indices_.resize(num_data_);
}
bagging_rands_.clear();
for (int i = 0;
i < (num_data_ + bagging_rand_block_ - 1) / bagging_rand_block_; ++i) {
bagging_rands_.emplace_back(config_->bagging_seed + i);
}
double average_bag_rate =
(static_cast<double>(bag_data_cnt_) / num_data_) / config_->bagging_freq;
is_use_subset_ = false;
if (config_->device_type != std::string("cuda")) {
const int group_threshold_usesubset = 100;
const double average_bag_rate_threshold = 0.5;
if (average_bag_rate <= average_bag_rate_threshold
&& (train_data_->num_feature_groups() < group_threshold_usesubset)) {
if (tmp_subset_ == nullptr || is_change_dataset) {
tmp_subset_.reset(new Dataset(bag_data_cnt_));
tmp_subset_->CopyFeatureMapperFrom(train_data_);
}
is_use_subset_ = true;
Log::Debug("Use subset for bagging");
}
}
need_re_bagging_ = true;
if (is_use_subset_ && bag_data_cnt_ < num_data_) {
// resize gradient vectors to copy the customized gradients for using subset data
need_resize_gradients_ = true;
}
} else {
bag_data_cnt_ = num_data_;
bag_data_indices_.clear();
#ifdef USE_CUDA
cuda_bag_data_indices_.Clear();
#endif // USE_CUDA
bagging_runner_.ReSize(0);
is_use_subset_ = false;
}
}
bool IsHessianChange() const override {
return false;
}
data_size_t num_sampled_queries() const override {
return num_sampled_queries_;
}
const data_size_t* sampled_query_indices() const override {
return bag_query_indices_.data();
}
private:
data_size_t BaggingHelper(data_size_t start, data_size_t cnt, data_size_t* buffer) {
if (cnt <= 0) {
return 0;
}
data_size_t cur_left_cnt = 0;
data_size_t cur_right_pos = cnt;
// random bagging, minimal unit is one record
for (data_size_t i = 0; i < cnt; ++i) {
auto cur_idx = start + i;
if (bagging_rands_[cur_idx / bagging_rand_block_].NextFloat() < config_->bagging_fraction) {
buffer[cur_left_cnt++] = cur_idx;
} else {
buffer[--cur_right_pos] = cur_idx;
}
}
return cur_left_cnt;
}
data_size_t BalancedBaggingHelper(data_size_t start, data_size_t cnt, data_size_t* buffer) {
if (cnt <= 0) {
return 0;
}
auto label_ptr = train_data_->metadata().label();
data_size_t cur_left_cnt = 0;
data_size_t cur_right_pos = cnt;
// random bagging, minimal unit is one record
for (data_size_t i = 0; i < cnt; ++i) {
auto cur_idx = start + i;
bool is_pos = label_ptr[start + i] > 0;
bool is_in_bag = false;
if (is_pos) {
is_in_bag = bagging_rands_[cur_idx / bagging_rand_block_].NextFloat() <
config_->pos_bagging_fraction;
} else {
is_in_bag = bagging_rands_[cur_idx / bagging_rand_block_].NextFloat() <
config_->neg_bagging_fraction;
}
if (is_in_bag) {
buffer[cur_left_cnt++] = cur_idx;
} else {
buffer[--cur_right_pos] = cur_idx;
}
}
return cur_left_cnt;
}
/*! \brief whether need restart bagging in continued training */
bool need_re_bagging_;
/*! \brief number of threads */
int num_threads_;
/*! \brief query boundaries of the in-bag queries */
std::vector<data_size_t> sampled_query_boundaries_;
/*! \brief buffer for calculating sampled_query_boundaries_ */
std::vector<data_size_t> sampled_query_boundaries_thread_buffer_;
/*! \brief in-bag query indices */
std::vector<data_size_t, Common::AlignmentAllocator<data_size_t, kAlignedSize>> bag_query_indices_;
/*! \brief number of queries in the training dataset */
data_size_t num_queries_;
/*! \brief number of in-bag queries */
data_size_t num_sampled_queries_;
/*! \brief query boundaries of the whole training dataset */
const data_size_t* query_boundaries_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_BOOSTING_BAGGING_HPP_
src/boosting/boosting.cpp
+102
View File
@@ -0,0 +1,102 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/boosting.h>
#include <memory>
#include <string>
#include "dart.hpp"
#include "gbdt.h"
#include "rf.hpp"
#ifdef USE_CUDA
#include "cuda/nccl_gbdt.hpp"
#endif // USE_CUDA
namespace LightGBM {
std::string GetBoostingTypeFromModelFile(const char* filename) {
TextReader<size_t> model_reader(filename, true);
std::string type = model_reader.first_line();
return type;
}
bool Boosting::LoadFileToBoosting(Boosting* boosting, const char* filename) {
auto start_time = std::chrono::steady_clock::now();
if (boosting != nullptr) {
TextReader<size_t> model_reader(filename, true);
size_t buffer_len = 0;
auto buffer = model_reader.ReadContent(&buffer_len);
if (!boosting->LoadModelFromString(buffer.data(), buffer_len)) {
return false;
}
}
std::chrono::duration<double, std::milli> delta = (std::chrono::steady_clock::now() - start_time);
Log::Debug("Time for loading model: %f seconds", 1e-3*delta);
return true;
}
Boosting* Boosting::CreateBoosting(const std::string& type, const char* filename,
const std::string&
#ifdef USE_CUDA
device_type
#endif // USE_CUDA
, const int
#ifdef USE_CUDA
num_gpu
#endif // USE_CUDA
) {
if (filename == nullptr || filename[0] == '\0') {
if (type == std::string("gbdt")) {
#ifdef USE_CUDA
if (device_type == std::string("cuda") && num_gpu > 1) {
return new NCCLGBDT<GBDT>();
} else {
#endif // USE_CUDA
return new GBDT();
#ifdef USE_CUDA
}
#endif // USE_CUDA
} else if (type == std::string("dart")) {
return new DART();
} else if (type == std::string("goss")) {
return new GBDT();
} else if (type == std::string("rf")) {
return new RF();
} else {
return nullptr;
}
} else {
std::unique_ptr<Boosting> ret;
if (GetBoostingTypeFromModelFile(filename) == std::string("tree")) {
if (type == std::string("gbdt")) {
#ifdef USE_CUDA
if (device_type == std::string("cuda") && num_gpu > 1) {
ret.reset(new NCCLGBDT<GBDT>());
} else {
#endif // USE_CUDA
ret.reset(new GBDT());
#ifdef USE_CUDA
}
#endif // USE_CUDA
} else if (type == std::string("dart")) {
ret.reset(new DART());
} else if (type == std::string("goss")) {
ret.reset(new GBDT());
} else if (type == std::string("rf")) {
ret.reset(new RF());
} else {
Log::Fatal("Unknown boosting type %s", type.c_str());
}
LoadFileToBoosting(ret.get(), filename);
} else {
Log::Fatal("Unknown model format or submodel type in model file %s", filename);
}
return ret.release();
}
}
} // namespace LightGBM
src/boosting/cuda/cuda_score_updater.cpp
+93
View File
@@ -0,0 +1,93 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include "cuda_score_updater.hpp"
#ifdef USE_CUDA
namespace LightGBM {
CUDAScoreUpdater::CUDAScoreUpdater(const Dataset* data, int num_tree_per_iteration, const bool boosting_on_cuda):
ScoreUpdater(data, num_tree_per_iteration), num_threads_per_block_(1024), boosting_on_cuda_(boosting_on_cuda) {
num_data_ = data->num_data();
int64_t total_size = static_cast<int64_t>(num_data_) * num_tree_per_iteration;
InitCUDA(total_size);
has_init_score_ = false;
const double* init_score = data->metadata().init_score();
// if exists initial score, will start from it
if (init_score != nullptr) {
if ((data->metadata().num_init_score() % num_data_) != 0
|| (data->metadata().num_init_score() / num_data_) != num_tree_per_iteration) {
Log::Fatal("Number of class for initial score error");
}
has_init_score_ = true;
CopyFromHostToCUDADevice<double>(cuda_score_.RawData(), init_score, total_size, __FILE__, __LINE__);
} else {
SetCUDAMemory<double>(cuda_score_.RawData(), 0, static_cast<size_t>(total_size), __FILE__, __LINE__);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
if (boosting_on_cuda_) {
// clear host score buffer
score_.clear();
score_.shrink_to_fit();
}
}
void CUDAScoreUpdater::InitCUDA(const size_t total_size) {
cuda_score_.Resize(total_size);
}
CUDAScoreUpdater::~CUDAScoreUpdater() {}
inline void CUDAScoreUpdater::AddScore(double val, int cur_tree_id) {
Common::FunctionTimer fun_timer("CUDAScoreUpdater::AddScore", global_timer);
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
LaunchAddScoreConstantKernel(val, offset);
if (!boosting_on_cuda_) {
CopyFromCUDADeviceToHost<double>(score_.data() + offset, cuda_score_.RawData() + offset, static_cast<size_t>(num_data_), __FILE__, __LINE__);
}
}
inline void CUDAScoreUpdater::AddScore(const Tree* tree, int cur_tree_id) {
Common::FunctionTimer fun_timer("ScoreUpdater::AddScore", global_timer);
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
tree->AddPredictionToScore(data_, num_data_, cuda_score_.RawData() + offset);
if (!boosting_on_cuda_) {
CopyFromCUDADeviceToHost<double>(score_.data() + offset, cuda_score_.RawData() + offset, static_cast<size_t>(num_data_), __FILE__, __LINE__);
}
}
inline void CUDAScoreUpdater::AddScore(const TreeLearner* tree_learner, const Tree* tree, int cur_tree_id) {
Common::FunctionTimer fun_timer("ScoreUpdater::AddScore", global_timer);
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
tree_learner->AddPredictionToScore(tree, cuda_score_.RawData() + offset);
if (!boosting_on_cuda_) {
CopyFromCUDADeviceToHost<double>(score_.data() + offset, cuda_score_.RawData() + offset, static_cast<size_t>(num_data_), __FILE__, __LINE__);
}
}
inline void CUDAScoreUpdater::AddScore(const Tree* tree, const data_size_t* data_indices,
data_size_t data_cnt, int cur_tree_id) {
Common::FunctionTimer fun_timer("ScoreUpdater::AddScore", global_timer);
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
tree->AddPredictionToScore(data_, data_indices, data_cnt, cuda_score_.RawData() + offset);
if (!boosting_on_cuda_) {
CopyFromCUDADeviceToHost<double>(score_.data() + offset, cuda_score_.RawData() + offset, static_cast<size_t>(num_data_), __FILE__, __LINE__);
}
}
inline void CUDAScoreUpdater::MultiplyScore(double val, int cur_tree_id) {
Common::FunctionTimer fun_timer("CUDAScoreUpdater::MultiplyScore", global_timer);
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
LaunchMultiplyScoreConstantKernel(val, offset);
if (!boosting_on_cuda_) {
CopyFromCUDADeviceToHost<double>(score_.data() + offset, cuda_score_.RawData() + offset, static_cast<size_t>(num_data_), __FILE__, __LINE__);
}
}
} // namespace LightGBM
#endif // USE_CUDA
src/boosting/cuda/cuda_score_updater.cu
+46
View File
@@ -0,0 +1,46 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include "cuda_score_updater.hpp"
#ifdef USE_CUDA
namespace LightGBM {
__global__ void AddScoreConstantKernel(
const double val,
const data_size_t num_data,
double* score) {
const data_size_t data_index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
if (data_index < num_data) {
score[data_index] += val;
}
}
void CUDAScoreUpdater::LaunchAddScoreConstantKernel(const double val, const size_t offset) {
const int num_blocks = (num_data_ + num_threads_per_block_) / num_threads_per_block_;
Log::Debug("Adding init score = %lf", val);
AddScoreConstantKernel<<<num_blocks, num_threads_per_block_>>>(val, num_data_, cuda_score_.RawData() + offset);
}
__global__ void MultiplyScoreConstantKernel(
const double val,
const data_size_t num_data,
double* score) {
const data_size_t data_index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
if (data_index < num_data) {
score[data_index] *= val;
}
}
void CUDAScoreUpdater::LaunchMultiplyScoreConstantKernel(const double val, const size_t offset) {
const int num_blocks = (num_data_ + num_threads_per_block_) / num_threads_per_block_;
MultiplyScoreConstantKernel<<<num_blocks, num_threads_per_block_>>>(val, num_data_, cuda_score_.RawData() + offset);
}
} // namespace LightGBM
#endif // USE_CUDA
src/boosting/cuda/cuda_score_updater.hpp
+66
View File
@@ -0,0 +1,66 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_BOOSTING_CUDA_CUDA_SCORE_UPDATER_HPP_
#define LIGHTGBM_SRC_BOOSTING_CUDA_CUDA_SCORE_UPDATER_HPP_
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_utils.hu>
#include "../score_updater.hpp"
namespace LightGBM {
class CUDAScoreUpdater: public ScoreUpdater {
public:
CUDAScoreUpdater(const Dataset* data, int num_tree_per_iteration, const bool boosting_on_cuda);
~CUDAScoreUpdater();
void AddScore(double val, int cur_tree_id) override;
inline void AddScore(const Tree* tree, int cur_tree_id) override;
void AddScore(const TreeLearner* tree_learner, const Tree* tree, int cur_tree_id) override;
inline void AddScore(const Tree* tree, const data_size_t* data_indices,
data_size_t data_cnt, int cur_tree_id) override;
inline void MultiplyScore(double val, int cur_tree_id) override;
inline const double* score() const override {
if (boosting_on_cuda_) {
return cuda_score_.RawData();
} else {
return score_.data();
}
}
/*! \brief Disable copy */
CUDAScoreUpdater& operator=(const CUDAScoreUpdater&) = delete;
CUDAScoreUpdater(const CUDAScoreUpdater&) = delete;
private:
void InitCUDA(const size_t total_size);
void LaunchAddScoreConstantKernel(const double val, const size_t offset);
void LaunchMultiplyScoreConstantKernel(const double val, const size_t offset);
CUDAVector<double> cuda_score_;
const int num_threads_per_block_;
const bool boosting_on_cuda_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_BOOSTING_CUDA_CUDA_SCORE_UPDATER_HPP_
src/boosting/cuda/nccl_gbdt.cpp
+210
View File
@@ -0,0 +1,210 @@
/*!
* Copyright (c) 2023-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2023-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/metric.h>
#include <set>
#include <vector>
#include "nccl_gbdt.hpp"
#include "nccl_gbdt_component.hpp"
#ifdef USE_CUDA
namespace LightGBM {
template <typename GBDT_T>
NCCLGBDT<GBDT_T>::NCCLGBDT(): GBDT_T() {}
template <typename GBDT_T>
NCCLGBDT<GBDT_T>::~NCCLGBDT() {}
template <typename GBDT_T>
void NCCLGBDT<GBDT_T>::Init(
const Config* gbdt_config, const Dataset* train_data,
const ObjectiveFunction* objective_function,
const std::vector<const Metric*>& training_metrics) {
GBDT_T::Init(gbdt_config, train_data, objective_function, training_metrics);
this->tree_learner_.reset();
nccl_topology_.reset(new NCCLTopology(this->config_->gpu_device_id, this->config_->num_gpu, this->config_->gpu_device_id_list, train_data->num_data()));
nccl_topology_->InitNCCL();
nccl_topology_->InitPerDevice<NCCLGBDTComponent>(&nccl_gbdt_components_);
nccl_topology_->RunPerDevice<NCCLGBDTComponent, void>(nccl_gbdt_components_, [this, gbdt_config, train_data]
(NCCLGBDTComponent* nccl_gbdt_component) { nccl_gbdt_component->Init(
gbdt_config, train_data, this->num_tree_per_iteration_, this->boosting_on_gpu_, this->is_constant_hessian_);
});
}
template <typename GBDT_T>
void NCCLGBDT<GBDT_T>::BoostingThread(NCCLGBDTComponent* thread_data) {
const ObjectiveFunction* objective_function = thread_data->objective_function();
score_t* gradients = thread_data->gradients();
score_t* hessians = thread_data->hessians();
const double* score = thread_data->train_score_updater()->score();
objective_function->GetGradients(score, gradients, hessians);
}
template <typename GBDT_T>
void NCCLGBDT<GBDT_T>::Boosting() {
Common::FunctionTimer fun_timer("NCCLGBDT::Boosting", global_timer);
if (this->objective_function_ == nullptr) {
Log::Fatal("No object function provided");
}
nccl_topology_->DispatchPerDevice<NCCLGBDTComponent>(&nccl_gbdt_components_, BoostingThread);
}
template <typename GBDT_T>
double NCCLGBDT<GBDT_T>::BoostFromAverage(int class_id, bool update_scorer) {
double init_score = GBDT_T::BoostFromAverage(class_id, update_scorer);
if (init_score != 0.0) {
nccl_topology_->RunPerDevice<NCCLGBDTComponent, void>(nccl_gbdt_components_, [init_score, class_id] (NCCLGBDTComponent* thread_data) {
thread_data->train_score_updater()->AddScore(init_score, class_id);
});
}
return init_score;
}
template <typename GBDT_T>
void NCCLGBDT<GBDT_T>::TrainTreeLearnerThread(NCCLGBDTComponent* thread_data, const int class_id, const bool is_first_tree) {
const data_size_t num_data_in_gpu = thread_data->num_data_in_gpu();
const score_t* gradients = thread_data->gradients() + class_id * num_data_in_gpu;
const score_t* hessians = thread_data->hessians() + class_id * num_data_in_gpu;
thread_data->SetTree(thread_data->tree_learner()->Train(gradients, hessians, is_first_tree));
}
template <typename GBDT_T>
bool NCCLGBDT<GBDT_T>::TrainOneIter(const score_t* gradients, const score_t* hessians) {
Common::FunctionTimer fun_timer("NCCLGBDT::TrainOneIter", global_timer);
std::vector<double> init_scores(this->num_tree_per_iteration_, 0.0);
// boosting first
if (gradients == nullptr || hessians == nullptr) {
for (int cur_tree_id = 0; cur_tree_id < this->num_tree_per_iteration_; ++cur_tree_id) {
init_scores[cur_tree_id] = BoostFromAverage(cur_tree_id, true);
}
Boosting();
} else {
nccl_topology_->RunPerDevice<NCCLGBDTComponent, void>(nccl_gbdt_components_, [this, gradients, hessians] (NCCLGBDTComponent* thread_data) {
const data_size_t data_start_index = thread_data->data_start_index();
const data_size_t num_data_in_gpu = thread_data->num_data_in_gpu();
for (int class_id = 0; class_id < this->num_class_; ++class_id) {
CopyFromHostToCUDADevice<score_t>(
thread_data->gradients() + class_id * num_data_in_gpu,
gradients + class_id * this->num_data_ + data_start_index, num_data_in_gpu, __FILE__, __LINE__);
CopyFromHostToCUDADevice<score_t>(
thread_data->hessians() + class_id * num_data_in_gpu,
hessians + class_id * this->num_data_ + data_start_index, num_data_in_gpu, __FILE__, __LINE__);
}
});
}
bool should_continue = false;
for (int cur_tree_id = 0; cur_tree_id < this->num_tree_per_iteration_; ++cur_tree_id) {
if (this->class_need_train_[cur_tree_id] && this->train_data_->num_features() > 0) {
if (this->data_sample_strategy_->is_use_subset() && this->data_sample_strategy_->bag_data_cnt() < this->num_data_) {
Log::Fatal("Bagging is not supported for NCCLGBDT");
}
bool is_first_tree = this->models_.size() < static_cast<size_t>(this->num_tree_per_iteration_);
nccl_topology_->DispatchPerDevice<NCCLGBDTComponent>(&nccl_gbdt_components_,
[is_first_tree, cur_tree_id] (NCCLGBDTComponent* thread_data) -> void {
TrainTreeLearnerThread(thread_data, cur_tree_id, is_first_tree);
});
}
nccl_topology_->DispatchPerDevice<NCCLGBDTComponent>(&nccl_gbdt_components_, [cur_tree_id, this, init_scores] (NCCLGBDTComponent* thread_data) -> void {
this->UpdateScoreThread(thread_data, cur_tree_id, this->config_->learning_rate, init_scores[cur_tree_id]);
});
nccl_topology_->RunOnMasterDevice<NCCLGBDTComponent, void>(nccl_gbdt_components_, [&should_continue, this, cur_tree_id] (NCCLGBDTComponent* thread_data) -> void {
if (thread_data->new_tree()->num_leaves() > 1) {
should_continue = true;
}
for (auto& score_updater : this->valid_score_updater_) {
score_updater->AddScore(thread_data->new_tree(), cur_tree_id);
}
});
if (!should_continue) {
if (this->models_.size() < static_cast<size_t>(this->num_tree_per_iteration_)) {
Log::Warning("Training stopped with no splits.");
}
}
// add model
nccl_topology_->RunOnMasterDevice<NCCLGBDTComponent, void>(nccl_gbdt_components_, [this] (NCCLGBDTComponent* thread_data) -> void {
this->models_.emplace_back(thread_data->release_new_tree());
});
nccl_topology_->RunOnNonMasterDevice<NCCLGBDTComponent, void>(nccl_gbdt_components_, [this] (NCCLGBDTComponent* thread_data) -> void {
thread_data->clear_new_tree();
});
}
if (!should_continue) {
Log::Warning("Stopped training because there are no more leaves that meet the split requirements");
if (this->models_.size() > static_cast<size_t>(this->num_tree_per_iteration_)) {
for (int cur_tree_id = 0; cur_tree_id < this->num_tree_per_iteration_; ++cur_tree_id) {
this->models_.pop_back();
}
}
return true;
}
++this->iter_;
return false;
}
template <typename GBDT_T>
void NCCLGBDT<GBDT_T>::UpdateScoreThread(NCCLGBDTComponent* thread_data, const int cur_tree_id, const double shrinkage_rate, const double init_score) {
if (thread_data->new_tree()->num_leaves() > 1) {
// TODO(shiyu1994): implement bagging
if (thread_data->objective_function() != nullptr && thread_data->objective_function()->IsRenewTreeOutput()) {
// TODO(shiyu1994): implement renewing
}
thread_data->new_tree()->Shrinkage(shrinkage_rate);
thread_data->train_score_updater()->AddScore(
thread_data->tree_learner(),
thread_data->new_tree(),
cur_tree_id);
if (std::fabs(init_score) > kEpsilon) {
thread_data->new_tree()->AddBias(init_score);
}
}
}
template <typename GBDT_T>
std::vector<double> NCCLGBDT<GBDT_T>::EvalOneMetric(const Metric* metric, const double* score, const data_size_t num_data) const {
if (score == this->train_score_updater_->score()) {
// delegate to per gpu train score updater
std::vector<double> tmp_score(num_data * this->num_class_, 0.0f);
nccl_topology_->RunPerDevice<NCCLGBDTComponent, void>(nccl_gbdt_components_, [this, &tmp_score] (NCCLGBDTComponent* thread_data) {
const data_size_t data_start = thread_data->data_start_index();
const data_size_t num_data_in_gpu = thread_data->num_data_in_gpu();
for (int class_id = 0; class_id < this->num_class_; ++class_id) {
CopyFromCUDADeviceToHost<double>(tmp_score.data() + class_id * this->num_data_ + data_start,
thread_data->train_score_updater()->score() + class_id * num_data_in_gpu,
static_cast<size_t>(num_data_in_gpu), __FILE__, __LINE__);
}
});
return metric->Eval(tmp_score.data(), this->objective_function_);
} else {
return GBDT_T::EvalOneMetric(metric, score, num_data);
}
}
template class NCCLGBDT<GBDT>;
} // namespace LightGBM
#endif // USE_CUDA
src/boosting/cuda/nccl_gbdt.hpp
+146
View File
@@ -0,0 +1,146 @@
/*!
* Copyright (c) 2023-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2023-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_BOOSTING_CUDA_NCCL_GBDT_HPP_
#define LIGHTGBM_SRC_BOOSTING_CUDA_NCCL_GBDT_HPP_
#ifdef USE_CUDA
#include <pthread.h>
#include <LightGBM/objective_function.h>
#include <LightGBM/network.h>
#include <memory>
#include <vector>
#include <LightGBM/cuda/cuda_nccl_topology.hpp>
#include "cuda_score_updater.hpp"
#include "nccl_gbdt_component.hpp"
#include "../gbdt.h"
namespace LightGBM {
template <typename GBDT_T>
class NCCLGBDT: public GBDT_T {
public:
NCCLGBDT();
~NCCLGBDT();
void Init(const Config* gbdt_config, const Dataset* train_data,
const ObjectiveFunction* objective_function,
const std::vector<const Metric*>& training_metrics) override;
void Boosting() override;
void RefitTree(const int* /*tree_leaf_prediction*/, const size_t /*nrow*/, const size_t /*ncol*/) override {
Log::Fatal("RefitTree is not supported for NCCLGBDT.");
}
bool TrainOneIter(const score_t* gradients, const score_t* hessians) override;
const double* GetTrainingScore(int64_t* /*out_len*/) override {
Log::Fatal("GetTrainingScore is not supported for NCCLGBDT.");
}
void ResetTrainingData(const Dataset* /*train_data*/, const ObjectiveFunction* /*objective_function*/,
const std::vector<const Metric*>& /*training_metrics*/) override {
Log::Fatal("ResetTrainingData is not supported for NCCLGBDT.");
}
void ResetConfig(const Config* /*gbdt_config*/) override {
Log::Fatal("ResetConfig is not supported for NCCLGBDT.");
}
private:
struct BoostingThreadData {
int gpu_index;
ObjectiveFunction* gpu_objective_function;
score_t* gradients;
score_t* hessians;
const double* score;
BoostingThreadData() {
gpu_index = 0;
gpu_objective_function = nullptr;
}
};
struct TrainTreeLearnerThreadData {
int gpu_index;
TreeLearner* gpu_tree_learner;
const score_t* gradients;
const score_t* hessians;
bool is_first_time;
int class_id;
data_size_t num_data_in_gpu;
std::unique_ptr<Tree> tree;
TrainTreeLearnerThreadData() {
gpu_index = 0;
gpu_tree_learner = nullptr;
gradients = nullptr;
hessians = nullptr;
is_first_time = false;
class_id = 0;
num_data_in_gpu = 0;
tree.reset(nullptr);
}
};
struct UpdateScoreThreadData {
int gpu_index;
ScoreUpdater* gpu_score_updater;
TreeLearner* gpu_tree_learner;
Tree* tree;
int cur_tree_id;
UpdateScoreThreadData() {
gpu_index = 0;
gpu_score_updater = nullptr;
gpu_tree_learner = nullptr;
tree = nullptr;
cur_tree_id = 0;
}
};
static void BoostingThread(NCCLGBDTComponent* thread_data);
static void TrainTreeLearnerThread(NCCLGBDTComponent* thread_data, const int class_id, const bool is_first_tree);
static void UpdateScoreThread(NCCLGBDTComponent* thread_data, const int cur_tree_id, const double shrinkage_rate, const double init_score);
double BoostFromAverage(int class_id, bool update_scorer) override;
void UpdateScore(const std::vector<std::unique_ptr<Tree>>& tree, const int cur_tree_id);
void UpdateScore(const Tree* /*tree*/, const int /*cur_tree_id*/) {
Log::Fatal("UpdateScore is not supported for NCCLGBDT.");
}
void RollbackOneIter() override {
Log::Fatal("RollbackOneIter is not supported for NCCLGBDT.");
}
std::vector<double> EvalOneMetric(const Metric* metric, const double* score, const data_size_t num_data) const override;
int num_threads_;
std::unique_ptr<NCCLTopology> nccl_topology_;
std::vector<int> nccl_gpu_rank_;
std::vector<ncclComm_t> nccl_communicators_;
std::vector<std::unique_ptr<NCCLGBDTComponent>> nccl_gbdt_components_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_BOOSTING_CUDA_NCCL_GBDT_HPP_
src/boosting/cuda/nccl_gbdt_component.hpp
+104
View File
@@ -0,0 +1,104 @@
/*!
* Copyright (c) 2023-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2023-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_BOOSTING_CUDA_NCCL_GBDT_COMPONENT_HPP_
#define LIGHTGBM_SRC_BOOSTING_CUDA_NCCL_GBDT_COMPONENT_HPP_
#ifdef USE_CUDA
#include <LightGBM/objective_function.h>
#include <LightGBM/tree.h>
#include <algorithm>
#include <vector>
#include <memory>
#include <LightGBM/cuda/cuda_objective_function.hpp>
#include "cuda_score_updater.hpp"
#include "../../treelearner/cuda/cuda_single_gpu_tree_learner.hpp"
namespace LightGBM {
class NCCLGBDTComponent: public NCCLInfo {
public:
NCCLGBDTComponent() {}
~NCCLGBDTComponent() {}
void Init(const Config* config, const Dataset* train_data, const int num_tree_per_iteration, const bool boosting_on_gpu, const bool is_constant_hessian) {
CUDASUCCESS_OR_FATAL(cudaGetDeviceCount(&num_gpu_in_node_));
const data_size_t num_data_per_gpu = (train_data->num_data() + num_gpu_in_node_ - 1) / num_gpu_in_node_;
data_start_index_ = num_data_per_gpu * local_gpu_rank_;
data_end_index_ = std::min<data_size_t>(data_start_index_ + num_data_per_gpu, train_data->num_data());
num_data_in_gpu_ = data_end_index_ - data_start_index_;
dataset_.reset(new Dataset(num_data_in_gpu_));
dataset_->ReSize(num_data_in_gpu_);
dataset_->CopyFeatureMapperFrom(train_data);
std::vector<data_size_t> used_indices(num_data_in_gpu_);
for (data_size_t data_index = data_start_index_; data_index < data_end_index_; ++data_index) {
used_indices[data_index - data_start_index_] = data_index;
}
dataset_->CopySubrowToDevice(train_data, used_indices.data(), num_data_in_gpu_, true, gpu_device_id_);
objective_function_.reset(ObjectiveFunction::CreateObjectiveFunctionCUDA(config->objective, *config));
objective_function_->SetNCCLInfo(nccl_communicator_, nccl_gpu_rank_, local_gpu_rank_, gpu_device_id_, train_data->num_data());
train_score_updater_.reset(new CUDAScoreUpdater(dataset_.get(), num_tree_per_iteration, boosting_on_gpu));
gradients_.reset(new CUDAVector<score_t>(num_data_in_gpu_));
hessians_.reset(new CUDAVector<score_t>(num_data_in_gpu_));
tree_learner_.reset(new CUDASingleGPUTreeLearner(config, boosting_on_gpu));
tree_learner_->SetNCCLInfo(nccl_communicator_, nccl_gpu_rank_, local_gpu_rank_, gpu_device_id_, train_data->num_data());
objective_function_->Init(dataset_->metadata(), dataset_->num_data());
tree_learner_->Init(dataset_.get(), is_constant_hessian);
}
ObjectiveFunction* objective_function() { return objective_function_.get(); }
ScoreUpdater* train_score_updater() { return train_score_updater_.get(); }
score_t* gradients() { return gradients_->RawData(); }
score_t* hessians() { return hessians_->RawData(); }
data_size_t num_data_in_gpu() const { return num_data_in_gpu_; }
CUDASingleGPUTreeLearner* tree_learner() { return tree_learner_.get(); }
void SetTree(Tree* tree) {
new_tree_.reset(tree);
}
data_size_t data_start_index() const { return data_start_index_; }
data_size_t data_end_index() const { return data_end_index_; }
Tree* new_tree() { return new_tree_.get(); }
Tree* release_new_tree() { return new_tree_.release(); }
void clear_new_tree() { new_tree_.reset(nullptr); }
private:
std::unique_ptr<ObjectiveFunction> objective_function_;
std::unique_ptr<ScoreUpdater> train_score_updater_;
std::unique_ptr<CUDAVector<score_t>> gradients_;
std::unique_ptr<CUDAVector<score_t>> hessians_;
std::unique_ptr<Dataset> dataset_;
std::unique_ptr<CUDASingleGPUTreeLearner> tree_learner_;
std::unique_ptr<Tree> new_tree_;
data_size_t data_start_index_;
data_size_t data_end_index_;
data_size_t num_data_in_gpu_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_BOOSTING_CUDA_NCCL_GBDT_COMPONENT_HPP_
src/boosting/dart.hpp
+212
View File
@@ -0,0 +1,212 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_BOOSTING_DART_HPP_
#define LIGHTGBM_SRC_BOOSTING_DART_HPP_
#include <LightGBM/boosting.h>
#include <string>
#include <algorithm>
#include <cstdio>
#include <fstream>
#include <vector>
#include "gbdt.h"
#include "score_updater.hpp"
namespace LightGBM {
/*!
* \brief DART algorithm implementation. including Training, prediction, bagging.
*/
class DART: public GBDT {
public:
/*!
* \brief Constructor
*/
DART() : GBDT() { }
/*!
* \brief Destructor
*/
~DART() { }
/*!
* \brief Initialization logic
* \param config Config for boosting
* \param train_data Training data
* \param objective_function Training objective function
* \param training_metrics Training metrics
* \param output_model_filename Filename of output model
*/
void Init(const Config* config, const Dataset* train_data,
const ObjectiveFunction* objective_function,
const std::vector<const Metric*>& training_metrics) override {
GBDT::Init(config, train_data, objective_function, training_metrics);
random_for_drop_ = Random(config_->drop_seed);
sum_weight_ = 0.0f;
}
void ResetConfig(const Config* config) override {
GBDT::ResetConfig(config);
random_for_drop_ = Random(config_->drop_seed);
sum_weight_ = 0.0f;
}
/*!
* \brief one training iteration
*/
bool TrainOneIter(const score_t* gradient, const score_t* hessian) override {
is_update_score_cur_iter_ = false;
bool ret = GBDT::TrainOneIter(gradient, hessian);
if (ret) {
return ret;
}
// normalize
Normalize();
if (!config_->uniform_drop) {
tree_weight_.push_back(shrinkage_rate_);
sum_weight_ += shrinkage_rate_;
}
return false;
}
/*!
* \brief Get current training score
* \param out_len length of returned score
* \return training score
*/
const double* GetTrainingScore(int64_t* out_len) override {
if (!is_update_score_cur_iter_) {
// only drop one time in one iteration
DroppingTrees();
is_update_score_cur_iter_ = true;
}
*out_len = static_cast<int64_t>(train_score_updater_->num_data()) * num_class_;
return train_score_updater_->score();
}
bool EvalAndCheckEarlyStopping() override {
GBDT::OutputMetric(iter_);
return false;
}
private:
/*!
* \brief drop trees based on drop_rate
*/
void DroppingTrees() {
drop_index_.clear();
bool is_skip = random_for_drop_.NextFloat() < config_->skip_drop;
// select dropping tree indices based on drop_rate and tree weights
if (!is_skip) {
double drop_rate = config_->drop_rate;
if (!config_->uniform_drop) {
double inv_average_weight = static_cast<double>(tree_weight_.size()) / sum_weight_;
if (config_->max_drop > 0) {
drop_rate = std::min(drop_rate, config_->max_drop * inv_average_weight / sum_weight_);
}
for (int i = 0; i < iter_; ++i) {
if (random_for_drop_.NextFloat() < drop_rate * tree_weight_[i] * inv_average_weight) {
drop_index_.push_back(num_init_iteration_ + i);
if (drop_index_.size() >= static_cast<size_t>(config_->max_drop)) {
break;
}
}
}
} else {
if (config_->max_drop > 0) {
drop_rate = std::min(drop_rate, config_->max_drop / static_cast<double>(iter_));
}
for (int i = 0; i < iter_; ++i) {
if (random_for_drop_.NextFloat() < drop_rate) {
drop_index_.push_back(num_init_iteration_ + i);
if (drop_index_.size() >= static_cast<size_t>(config_->max_drop)) {
break;
}
}
}
}
}
// drop trees
for (auto i : drop_index_) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
auto curr_tree = i * num_tree_per_iteration_ + cur_tree_id;
models_[curr_tree]->Shrinkage(-1.0);
train_score_updater_->AddScore(models_[curr_tree].get(), cur_tree_id);
}
}
if (!config_->xgboost_dart_mode) {
shrinkage_rate_ = config_->learning_rate / (1.0f + static_cast<double>(drop_index_.size()));
} else {
if (drop_index_.empty()) {
shrinkage_rate_ = config_->learning_rate;
} else {
shrinkage_rate_ = config_->learning_rate / (config_->learning_rate + static_cast<double>(drop_index_.size()));
}
}
}
/*!
* \brief normalize dropped trees
* NOTE: num_drop_tree(k), learning_rate(lr), shrinkage_rate_ = lr / (k + 1)
* step 1: shrink tree to -1 -> drop tree
* step 2: shrink tree to k / (k + 1) - 1 from -1, by 1/(k+1)
* -> normalize for valid data
* step 3: shrink tree to k / (k + 1) from k / (k + 1) - 1, by -k
* -> normalize for train data
* end with tree weight = (k / (k + 1)) * old_weight
*/
void Normalize() {
double k = static_cast<double>(drop_index_.size());
if (!config_->xgboost_dart_mode) {
for (auto i : drop_index_) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
auto curr_tree = i * num_tree_per_iteration_ + cur_tree_id;
// update validation score
models_[curr_tree]->Shrinkage(1.0f / (k + 1.0f));
for (auto& score_updater : valid_score_updater_) {
score_updater->AddScore(models_[curr_tree].get(), cur_tree_id);
}
// update training score
models_[curr_tree]->Shrinkage(-k);
train_score_updater_->AddScore(models_[curr_tree].get(), cur_tree_id);
}
if (!config_->uniform_drop) {
sum_weight_ -= tree_weight_[i - num_init_iteration_] * (1.0f / (k + 1.0f));
tree_weight_[i - num_init_iteration_] *= (k / (k + 1.0f));
}
}
} else {
for (auto i : drop_index_) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
auto curr_tree = i * num_tree_per_iteration_ + cur_tree_id;
// update validation score
models_[curr_tree]->Shrinkage(shrinkage_rate_);
for (auto& score_updater : valid_score_updater_) {
score_updater->AddScore(models_[curr_tree].get(), cur_tree_id);
}
// update training score
models_[curr_tree]->Shrinkage(-k / config_->learning_rate);
train_score_updater_->AddScore(models_[curr_tree].get(), cur_tree_id);
}
if (!config_->uniform_drop) {
sum_weight_ -= tree_weight_[i - num_init_iteration_] * (1.0f / (k + config_->learning_rate));;
tree_weight_[i - num_init_iteration_] *= (k / (k + config_->learning_rate));
}
}
}
}
/*! \brief The weights of all trees, used to choose drop trees */
std::vector<double> tree_weight_;
/*! \brief sum weights of all trees */
double sum_weight_;
/*! \brief The indices of dropping trees */
std::vector<int> drop_index_;
/*! \brief Random generator, used to select dropping trees */
Random random_for_drop_;
/*! \brief Flag that the score is update on current iter or not*/
bool is_update_score_cur_iter_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_BOOSTING_DART_HPP_
src/boosting/gbdt.cpp
+890
View File
@@ -0,0 +1,890 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include "gbdt.h"
#include <LightGBM/metric.h>
#include <LightGBM/network.h>
#include <LightGBM/objective_function.h>
#include <LightGBM/prediction_early_stop.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <LightGBM/sample_strategy.h>
#include <algorithm>
#include <chrono>
#include <ctime>
#include <memory>
#include <queue>
#include <sstream>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace LightGBM {
Common::Timer global_timer;
int LGBM_config_::current_device = lgbm_device_cpu;
int LGBM_config_::current_learner = use_cpu_learner;
GBDT::GBDT()
: iter_(0),
train_data_(nullptr),
config_(nullptr),
objective_function_(nullptr),
early_stopping_round_(0),
early_stopping_min_delta_(0.0),
es_first_metric_only_(false),
max_feature_idx_(0),
num_tree_per_iteration_(1),
num_class_(1),
num_iteration_for_pred_(0),
shrinkage_rate_(0.1f),
num_init_iteration_(0) {
average_output_ = false;
tree_learner_ = nullptr;
linear_tree_ = false;
data_sample_strategy_.reset(nullptr);
gradients_pointer_ = nullptr;
hessians_pointer_ = nullptr;
boosting_on_gpu_ = false;
}
GBDT::~GBDT() {
}
void GBDT::Init(const Config* config, const Dataset* train_data, const ObjectiveFunction* objective_function,
const std::vector<const Metric*>& training_metrics) {
CHECK_NOTNULL(train_data);
train_data_ = train_data;
if (!config->monotone_constraints.empty()) {
CHECK_EQ(static_cast<size_t>(train_data_->num_total_features()), config->monotone_constraints.size());
}
if (!config->feature_contri.empty()) {
CHECK_EQ(static_cast<size_t>(train_data_->num_total_features()), config->feature_contri.size());
}
iter_ = 0;
num_iteration_for_pred_ = 0;
max_feature_idx_ = 0;
num_class_ = config->num_class;
config_ = std::unique_ptr<Config>(new Config(*config));
early_stopping_round_ = config_->early_stopping_round;
early_stopping_min_delta_ = config->early_stopping_min_delta;
es_first_metric_only_ = config_->first_metric_only;
shrinkage_rate_ = config_->learning_rate;
if (config_->device_type == std::string("cuda")) {
LGBM_config_::current_learner = use_cuda_learner;
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda")) {
const int gpu_device_id = config_->gpu_device_id >= 0 ? config_->gpu_device_id : 0;
CUDASUCCESS_OR_FATAL(cudaSetDevice(gpu_device_id));
}
#endif // USE_CUDA
}
// load forced_splits file
if (!config->forcedsplits_filename.empty()) {
std::ifstream forced_splits_file(config->forcedsplits_filename.c_str());
std::stringstream buffer;
buffer << forced_splits_file.rdbuf();
std::string err;
forced_splits_json_ = Json::parse(buffer.str(), &err);
}
objective_function_ = objective_function;
num_tree_per_iteration_ = num_class_;
if (objective_function_ != nullptr) {
num_tree_per_iteration_ = objective_function_->NumModelPerIteration();
if (objective_function_->IsRenewTreeOutput() && !config->monotone_constraints.empty()) {
Log::Fatal("Cannot use ``monotone_constraints`` in %s objective, please disable it.", objective_function_->GetName());
}
}
data_sample_strategy_.reset(SampleStrategy::CreateSampleStrategy(config_.get(), train_data_, objective_function_, num_tree_per_iteration_));
is_constant_hessian_ = GetIsConstHessian(objective_function);
boosting_on_gpu_ = objective_function_ != nullptr && objective_function_->IsCUDAObjective() &&
!data_sample_strategy_->IsHessianChange(); // for sample strategy with Hessian change, fall back to boosting on CPU
tree_learner_ = std::unique_ptr<TreeLearner>(TreeLearner::CreateTreeLearner(config_->tree_learner, config_->device_type,
config_.get(), boosting_on_gpu_));
// init tree learner
tree_learner_->Init(train_data_, is_constant_hessian_);
tree_learner_->SetForcedSplit(&forced_splits_json_);
// push training metrics
training_metrics_.clear();
for (const auto& metric : training_metrics) {
training_metrics_.push_back(metric);
}
training_metrics_.shrink_to_fit();
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda")) {
train_score_updater_.reset(new CUDAScoreUpdater(train_data_, num_tree_per_iteration_, boosting_on_gpu_));
} else {
#endif // USE_CUDA
train_score_updater_.reset(new ScoreUpdater(train_data_, num_tree_per_iteration_));
#ifdef USE_CUDA
}
#endif // USE_CUDA
num_data_ = train_data_->num_data();
// get max feature index
max_feature_idx_ = train_data_->num_total_features() - 1;
// get label index
label_idx_ = train_data_->label_idx();
// get feature names
feature_names_ = train_data_->feature_names();
feature_infos_ = train_data_->feature_infos();
monotone_constraints_ = config->monotone_constraints;
// get parser config file content
parser_config_str_ = train_data_->parser_config_str();
// check that forced splits does not use feature indices larger than dataset size
CheckForcedSplitFeatures();
// if need bagging, create buffer
data_sample_strategy_->ResetSampleConfig(config_.get(), true);
ResetGradientBuffers();
class_need_train_ = std::vector<bool>(num_tree_per_iteration_, true);
if (objective_function_ != nullptr && objective_function_->SkipEmptyClass()) {
CHECK_EQ(num_tree_per_iteration_, num_class_);
for (int i = 0; i < num_class_; ++i) {
class_need_train_[i] = objective_function_->ClassNeedTrain(i);
}
}
if (config_->linear_tree) {
linear_tree_ = true;
}
}
void GBDT::CheckForcedSplitFeatures() {
std::queue<Json> forced_split_nodes;
forced_split_nodes.push(forced_splits_json_);
while (!forced_split_nodes.empty()) {
Json node = forced_split_nodes.front();
forced_split_nodes.pop();
const int feature_index = node["feature"].int_value();
if (feature_index > max_feature_idx_) {
Log::Fatal("Forced splits file includes feature index %d, but maximum feature index in dataset is %d",
feature_index, max_feature_idx_);
}
if (node.object_items().count("left") > 0) {
forced_split_nodes.push(node["left"]);
}
if (node.object_items().count("right") > 0) {
forced_split_nodes.push(node["right"]);
}
}
}
void GBDT::AddValidDataset(const Dataset* valid_data,
const std::vector<const Metric*>& valid_metrics) {
if (!train_data_->CheckAlign(*valid_data)) {
Log::Fatal("Cannot add validation data, since it has different bin mappers with training data");
}
// for a validation dataset, we need its score and metric
auto new_score_updater =
#ifdef USE_CUDA
config_->device_type == std::string("cuda") ?
std::unique_ptr<CUDAScoreUpdater>(new CUDAScoreUpdater(valid_data, num_tree_per_iteration_,
objective_function_ != nullptr && objective_function_->IsCUDAObjective())) :
#endif // USE_CUDA
std::unique_ptr<ScoreUpdater>(new ScoreUpdater(valid_data, num_tree_per_iteration_));
// update score
for (int i = 0; i < iter_; ++i) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
auto curr_tree = (i + num_init_iteration_) * num_tree_per_iteration_ + cur_tree_id;
new_score_updater->AddScore(models_[curr_tree].get(), cur_tree_id);
}
}
valid_score_updater_.push_back(std::move(new_score_updater));
valid_metrics_.emplace_back();
for (const auto& metric : valid_metrics) {
valid_metrics_.back().push_back(metric);
}
valid_metrics_.back().shrink_to_fit();
if (early_stopping_round_ > 0) {
auto num_metrics = valid_metrics.size();
if (es_first_metric_only_) {
num_metrics = 1;
}
best_iter_.emplace_back(num_metrics, 0);
best_score_.emplace_back(num_metrics, kMinScore);
best_msg_.emplace_back(num_metrics);
}
}
void GBDT::Boosting() {
Common::FunctionTimer fun_timer("GBDT::Boosting", global_timer);
if (objective_function_ == nullptr) {
Log::Fatal("No objective function provided");
}
// objective function will calculate gradients and hessians
int64_t num_score = 0;
if (config_->bagging_by_query) {
data_sample_strategy_->Bagging(iter_, tree_learner_.get(), gradients_.data(), hessians_.data());
objective_function_->
GetGradientsWithSampledQueries(GetTrainingScore(&num_score), data_sample_strategy_->num_sampled_queries(), data_sample_strategy_->sampled_query_indices(), gradients_pointer_, hessians_pointer_);
} else {
objective_function_->
GetGradients(GetTrainingScore(&num_score), gradients_pointer_, hessians_pointer_);
}
}
void GBDT::Train(int snapshot_freq, const std::string& model_output_path) {
Common::FunctionTimer fun_timer("GBDT::Train", global_timer);
bool is_finished = false;
auto start_time = std::chrono::steady_clock::now();
for (int iter = 0; iter < config_->num_iterations && !is_finished; ++iter) {
is_finished = TrainOneIter(nullptr, nullptr);
if (!is_finished) {
is_finished = EvalAndCheckEarlyStopping();
}
auto end_time = std::chrono::steady_clock::now();
// output used time per iteration
Log::Info("%f seconds elapsed, finished iteration %d", std::chrono::duration<double,
std::milli>(end_time - start_time) * 1e-3, iter + 1);
if (snapshot_freq > 0
&& (iter + 1) % snapshot_freq == 0) {
std::string snapshot_out = model_output_path + ".snapshot_iter_" + std::to_string(iter + 1);
SaveModelToFile(0, -1, config_->saved_feature_importance_type, snapshot_out.c_str());
}
}
}
void GBDT::RefitTree(const int* tree_leaf_prediction, const size_t nrow, const size_t ncol) {
CHECK_GT(nrow * ncol, 0);
CHECK_EQ(static_cast<size_t>(num_data_), nrow);
CHECK_EQ(models_.size(), ncol);
int num_iterations = static_cast<int>(models_.size() / num_tree_per_iteration_);
std::vector<int> leaf_pred(num_data_);
if (linear_tree_) {
std::vector<int> max_leaves_by_thread = std::vector<int>(OMP_NUM_THREADS(), 0);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int i = 0; i < static_cast<int>(nrow); ++i) {
int tid = omp_get_thread_num();
for (size_t j = 0; j < ncol; ++j) {
max_leaves_by_thread[tid] = std::max(max_leaves_by_thread[tid], tree_leaf_prediction[i * ncol + j]);
}
}
int max_leaves = *std::max_element(max_leaves_by_thread.begin(), max_leaves_by_thread.end());
max_leaves += 1;
tree_learner_->InitLinear(train_data_, max_leaves);
}
for (int iter = 0; iter < num_iterations; ++iter) {
Boosting();
for (int tree_id = 0; tree_id < num_tree_per_iteration_; ++tree_id) {
int model_index = iter * num_tree_per_iteration_ + tree_id;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int i = 0; i < num_data_; ++i) {
leaf_pred[i] = tree_leaf_prediction[i * ncol + model_index];
CHECK_LT(leaf_pred[i], models_[model_index]->num_leaves());
}
size_t offset = static_cast<size_t>(tree_id) * num_data_;
auto grad = gradients_pointer_ + offset;
auto hess = hessians_pointer_ + offset;
auto new_tree = tree_learner_->FitByExistingTree(models_[model_index].get(), leaf_pred, grad, hess);
train_score_updater_->AddScore(tree_learner_.get(), new_tree, tree_id);
models_[model_index].reset(new_tree);
}
}
}
/* If the custom "average" is implemented it will be used in place of the label average (if enabled)
*
* An improvement to this is to have options to explicitly choose
* (i) standard average
* (ii) custom average if available
* (iii) any user defined scalar bias (e.g. using a new option "init_score" that overrides (i) and (ii) )
*
* (i) and (ii) could be selected as say "auto_init_score" = 0 or 1 etc..
*
*/
double ObtainAutomaticInitialScore(const ObjectiveFunction* fobj, int class_id) {
double init_score = 0.0;
if (fobj != nullptr) {
init_score = fobj->BoostFromScore(class_id);
}
if (Network::num_machines() > 1) {
init_score = Network::GlobalSyncUpByMean(init_score);
}
return init_score;
}
double GBDT::BoostFromAverage(int class_id, bool update_scorer) {
Common::FunctionTimer fun_timer("GBDT::BoostFromAverage", global_timer);
// boosting from average label; or customized "average" if implemented for the current objective
if (models_.empty() && !train_score_updater_->has_init_score() && objective_function_ != nullptr) {
if (config_->boost_from_average || (train_data_ != nullptr && train_data_->num_features() == 0)) {
double init_score = ObtainAutomaticInitialScore(objective_function_, class_id);
if (std::fabs(init_score) > kEpsilon) {
if (update_scorer) {
train_score_updater_->AddScore(init_score, class_id);
for (auto& score_updater : valid_score_updater_) {
score_updater->AddScore(init_score, class_id);
}
}
Log::Info("Start training from score %lf", init_score);
return init_score;
}
} else if (std::string(objective_function_->GetName()) == std::string("regression_l1")
|| std::string(objective_function_->GetName()) == std::string("quantile")
|| std::string(objective_function_->GetName()) == std::string("mape")) {
Log::Warning("Disabling boost_from_average in %s may cause the slow convergence", objective_function_->GetName());
}
}
return 0.0f;
}
bool GBDT::TrainOneIter(const score_t* gradients, const score_t* hessians) {
Common::FunctionTimer fun_timer("GBDT::TrainOneIter", global_timer);
std::vector<double> init_scores(num_tree_per_iteration_, 0.0);
// boosting first
if (gradients == nullptr || hessians == nullptr) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
init_scores[cur_tree_id] = BoostFromAverage(cur_tree_id, true);
}
Boosting();
gradients = gradients_pointer_;
hessians = hessians_pointer_;
} else {
// use customized objective function
// the check below fails unless objective=custom is provided in the parameters on Booster creation
CHECK(objective_function_ == nullptr);
if (data_sample_strategy_->IsHessianChange()) {
// need to copy customized gradients when using GOSS
int64_t total_size = static_cast<int64_t>(num_data_) * num_tree_per_iteration_;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int64_t i = 0; i < total_size; ++i) {
gradients_[i] = gradients[i];
hessians_[i] = hessians[i];
}
CHECK_EQ(gradients_pointer_, gradients_.data());
CHECK_EQ(hessians_pointer_, hessians_.data());
gradients = gradients_pointer_;
hessians = hessians_pointer_;
}
}
// bagging logic
if (!config_->bagging_by_query) {
data_sample_strategy_->Bagging(iter_, tree_learner_.get(), gradients_.data(), hessians_.data());
}
const bool is_use_subset = data_sample_strategy_->is_use_subset();
const data_size_t bag_data_cnt = data_sample_strategy_->bag_data_cnt();
const std::vector<data_size_t, Common::AlignmentAllocator<data_size_t, kAlignedSize>>& bag_data_indices = data_sample_strategy_->bag_data_indices();
if (objective_function_ == nullptr && is_use_subset && bag_data_cnt < num_data_ && !boosting_on_gpu_ && !data_sample_strategy_->IsHessianChange()) {
ResetGradientBuffers();
}
bool should_continue = false;
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
const size_t offset = static_cast<size_t>(cur_tree_id) * num_data_;
std::unique_ptr<Tree> new_tree(new Tree(2, false, false));
if (class_need_train_[cur_tree_id] && train_data_->num_features() > 0) {
auto grad = gradients + offset;
auto hess = hessians + offset;
// need to copy gradients for bagging subset.
if (is_use_subset && bag_data_cnt < num_data_ && !boosting_on_gpu_) {
for (int i = 0; i < bag_data_cnt; ++i) {
gradients_pointer_[offset + i] = grad[bag_data_indices[i]];
hessians_pointer_[offset + i] = hess[bag_data_indices[i]];
}
grad = gradients_pointer_ + offset;
hess = hessians_pointer_ + offset;
}
bool is_first_tree = models_.size() < static_cast<size_t>(num_tree_per_iteration_);
new_tree.reset(tree_learner_->Train(grad, hess, is_first_tree));
}
if (new_tree->num_leaves() > 1) {
should_continue = true;
auto score_ptr = train_score_updater_->score() + offset;
auto residual_getter = [score_ptr](const label_t* label, int i) {return static_cast<double>(label[i]) - score_ptr[i]; };
tree_learner_->RenewTreeOutput(new_tree.get(), objective_function_, residual_getter,
num_data_, bag_data_indices.data(), bag_data_cnt, train_score_updater_->score());
// shrinkage by learning rate
new_tree->Shrinkage(shrinkage_rate_);
// update score
UpdateScore(new_tree.get(), cur_tree_id);
if (std::fabs(init_scores[cur_tree_id]) > kEpsilon) {
new_tree->AddBias(init_scores[cur_tree_id]);
}
} else {
// only add default score one-time
if (models_.size() < static_cast<size_t>(num_tree_per_iteration_)) {
if (objective_function_ != nullptr && !config_->boost_from_average && !train_score_updater_->has_init_score()) {
init_scores[cur_tree_id] = ObtainAutomaticInitialScore(objective_function_, cur_tree_id);
// updates scores
train_score_updater_->AddScore(init_scores[cur_tree_id], cur_tree_id);
for (auto& score_updater : valid_score_updater_) {
score_updater->AddScore(init_scores[cur_tree_id], cur_tree_id);
}
}
new_tree->AsConstantTree(init_scores[cur_tree_id], num_data_);
} else {
// extend init_scores with zeros
new_tree->AsConstantTree(0, num_data_);
}
}
// add model
models_.push_back(std::move(new_tree));
}
if (!should_continue) {
Log::Warning("Stopped training because there are no more leaves that meet the split requirements");
if (models_.size() > static_cast<size_t>(num_tree_per_iteration_)) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
models_.pop_back();
}
}
return true;
}
++iter_;
return false;
}
void GBDT::RollbackOneIter() {
if (iter_ <= 0) {
return;
}
// reset score
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
auto curr_tree = models_.size() - num_tree_per_iteration_ + cur_tree_id;
models_[curr_tree]->Shrinkage(-1.0);
train_score_updater_->AddScore(models_[curr_tree].get(), cur_tree_id);
for (auto& score_updater : valid_score_updater_) {
score_updater->AddScore(models_[curr_tree].get(), cur_tree_id);
}
}
// remove model
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
models_.pop_back();
}
--iter_;
}
bool GBDT::EvalAndCheckEarlyStopping() {
bool is_met_early_stopping = false;
// print message for metric
auto best_msg = OutputMetric(iter_);
is_met_early_stopping = !best_msg.empty();
if (is_met_early_stopping) {
Log::Info("Early stopping at iteration %d, the best iteration round is %d",
iter_, iter_ - early_stopping_round_);
Log::Info("Output of best iteration round:\n%s", best_msg.c_str());
// pop last early_stopping_round_ models
for (int i = 0; i < early_stopping_round_ * num_tree_per_iteration_; ++i) {
models_.pop_back();
}
}
return is_met_early_stopping;
}
void GBDT::UpdateScore(const Tree* tree, const int cur_tree_id) {
Common::FunctionTimer fun_timer("GBDT::UpdateScore", global_timer);
// update training score
if (!data_sample_strategy_->is_use_subset()) {
train_score_updater_->AddScore(tree_learner_.get(), tree, cur_tree_id);
const data_size_t bag_data_cnt = data_sample_strategy_->bag_data_cnt();
// we need to predict out-of-bag scores of data for boosting
if (num_data_ - bag_data_cnt > 0) {
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda")) {
train_score_updater_->AddScore(tree, data_sample_strategy_->cuda_bag_data_indices().RawData() + bag_data_cnt, num_data_ - bag_data_cnt, cur_tree_id);
} else {
#endif // USE_CUDA
train_score_updater_->AddScore(tree, data_sample_strategy_->bag_data_indices().data() + bag_data_cnt, num_data_ - bag_data_cnt, cur_tree_id);
#ifdef USE_CUDA
}
#endif // USE_CUDA
}
} else {
train_score_updater_->AddScore(tree, cur_tree_id);
}
// update validation score
for (auto& score_updater : valid_score_updater_) {
score_updater->AddScore(tree, cur_tree_id);
}
}
#ifdef USE_CUDA
std::vector<double> GBDT::EvalOneMetric(const Metric* metric, const double* score, const data_size_t num_data) const {
#else
std::vector<double> GBDT::EvalOneMetric(const Metric* metric, const double* score, const data_size_t /*num_data*/) const {
#endif // USE_CUDA
#ifdef USE_CUDA
const bool evaluation_on_cuda = metric->IsCUDAMetric();
if ((boosting_on_gpu_ && evaluation_on_cuda) || (!boosting_on_gpu_ && !evaluation_on_cuda)) {
#endif // USE_CUDA
return metric->Eval(score, objective_function_);
#ifdef USE_CUDA
} else if (boosting_on_gpu_ && !evaluation_on_cuda) {
const size_t total_size = static_cast<size_t>(num_data) * static_cast<size_t>(num_tree_per_iteration_);
if (total_size > host_score_.size()) {
host_score_.resize(total_size, 0.0f);
}
CopyFromCUDADeviceToHost<double>(host_score_.data(), score, total_size, __FILE__, __LINE__);
return metric->Eval(host_score_.data(), objective_function_);
} else {
const size_t total_size = static_cast<size_t>(num_data) * static_cast<size_t>(num_tree_per_iteration_);
if (total_size > cuda_score_.Size()) {
cuda_score_.Resize(total_size);
}
CopyFromHostToCUDADevice<double>(cuda_score_.RawData(), score, total_size, __FILE__, __LINE__);
return metric->Eval(cuda_score_.RawData(), objective_function_);
}
#endif // USE_CUDA
}
std::string GBDT::OutputMetric(int iter) {
bool need_output = (iter % config_->metric_freq) == 0;
std::string ret = "";
std::stringstream msg_buf;
std::vector<std::pair<size_t, size_t>> meet_early_stopping_pairs;
// print training metric
if (need_output) {
for (auto& sub_metric : training_metrics_) {
auto name = sub_metric->GetName();
auto scores = EvalOneMetric(sub_metric, train_score_updater_->score(), train_score_updater_->num_data());
for (size_t k = 0; k < name.size(); ++k) {
std::stringstream tmp_buf;
tmp_buf << "Iteration:" << iter
<< ", training " << name[k]
<< " : " << scores[k];
Log::Info(tmp_buf.str().c_str());
if (early_stopping_round_ > 0) {
msg_buf << tmp_buf.str() << '\n';
}
}
}
}
// print validation metric
if (need_output || early_stopping_round_ > 0) {
for (size_t i = 0; i < valid_metrics_.size(); ++i) {
for (size_t j = 0; j < valid_metrics_[i].size(); ++j) {
auto test_scores = EvalOneMetric(valid_metrics_[i][j], valid_score_updater_[i]->score(), valid_score_updater_[i]->num_data());
auto name = valid_metrics_[i][j]->GetName();
for (size_t k = 0; k < name.size(); ++k) {
std::stringstream tmp_buf;
tmp_buf << "Iteration:" << iter
<< ", valid_" << i + 1 << " " << name[k]
<< " : " << test_scores[k];
if (need_output) {
Log::Info(tmp_buf.str().c_str());
}
if (early_stopping_round_ > 0) {
msg_buf << tmp_buf.str() << '\n';
}
}
if (es_first_metric_only_ && j > 0) {
continue;
}
if (ret.empty() && early_stopping_round_ > 0) {
auto cur_score = valid_metrics_[i][j]->factor_to_bigger_better() * test_scores.back();
if (cur_score - best_score_[i][j] > early_stopping_min_delta_) {
best_score_[i][j] = cur_score;
best_iter_[i][j] = iter;
meet_early_stopping_pairs.emplace_back(i, j);
} else {
if (iter - best_iter_[i][j] >= early_stopping_round_) {
ret = best_msg_[i][j];
}
}
}
}
}
}
for (auto& pair : meet_early_stopping_pairs) {
best_msg_[pair.first][pair.second] = msg_buf.str();
}
return ret;
}
/*! \brief Get eval result */
std::vector<double> GBDT::GetEvalAt(int data_idx) const {
CHECK(data_idx >= 0 && data_idx <= static_cast<int>(valid_score_updater_.size()));
std::vector<double> ret;
if (data_idx == 0) {
for (auto& sub_metric : training_metrics_) {
auto scores = EvalOneMetric(sub_metric, train_score_updater_->score(), train_score_updater_->num_data());
for (auto score : scores) {
ret.push_back(score);
}
}
} else {
auto used_idx = data_idx - 1;
for (size_t j = 0; j < valid_metrics_[used_idx].size(); ++j) {
auto test_scores = EvalOneMetric(valid_metrics_[used_idx][j], valid_score_updater_[used_idx]->score(), valid_score_updater_[used_idx]->num_data());
for (auto score : test_scores) {
ret.push_back(score);
}
}
}
return ret;
}
/*! \brief Get training scores result */
const double* GBDT::GetTrainingScore(int64_t* out_len) {
*out_len = static_cast<int64_t>(train_score_updater_->num_data()) * num_class_;
return train_score_updater_->score();
}
void GBDT::PredictContrib(const double* features, double* output) const {
// set zero
const int num_features = max_feature_idx_ + 1;
std::memset(output, 0, sizeof(double) * num_tree_per_iteration_ * (num_features + 1));
const int end_iteration_for_pred = start_iteration_for_pred_ + num_iteration_for_pred_;
for (int i = start_iteration_for_pred_; i < end_iteration_for_pred; ++i) {
// predict all the trees for one iteration
for (int k = 0; k < num_tree_per_iteration_; ++k) {
models_[i * num_tree_per_iteration_ + k]->PredictContrib(features, num_features, output + k*(num_features + 1));
}
}
}
void GBDT::PredictContribByMap(const std::unordered_map<int, double>& features,
std::vector<std::unordered_map<int, double>>* output) const {
const int num_features = max_feature_idx_ + 1;
const int end_iteration_for_pred = start_iteration_for_pred_ + num_iteration_for_pred_;
for (int i = start_iteration_for_pred_; i < end_iteration_for_pred; ++i) {
// predict all the trees for one iteration
for (int k = 0; k < num_tree_per_iteration_; ++k) {
models_[i * num_tree_per_iteration_ + k]->PredictContribByMap(features, num_features, &((*output)[k]));
}
}
}
void GBDT::GetPredictAt(int data_idx, double* out_result, int64_t* out_len) {
CHECK(data_idx >= 0 && data_idx <= static_cast<int>(valid_score_updater_.size()));
const double* raw_scores = nullptr;
data_size_t num_data = 0;
if (data_idx == 0) {
raw_scores = GetTrainingScore(out_len);
num_data = train_score_updater_->num_data();
} else {
auto used_idx = data_idx - 1;
raw_scores = valid_score_updater_[used_idx]->score();
num_data = valid_score_updater_[used_idx]->num_data();
*out_len = static_cast<int64_t>(num_data) * num_class_;
}
#ifdef USE_CUDA
std::vector<double> host_raw_scores;
if (boosting_on_gpu_) {
host_raw_scores.resize(static_cast<size_t>(*out_len), 0.0);
CopyFromCUDADeviceToHost<double>(host_raw_scores.data(), raw_scores, static_cast<size_t>(*out_len), __FILE__, __LINE__);
raw_scores = host_raw_scores.data();
}
#endif // USE_CUDA
if (objective_function_ != nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data; ++i) {
std::vector<double> tree_pred(num_tree_per_iteration_);
for (int j = 0; j < num_tree_per_iteration_; ++j) {
tree_pred[j] = raw_scores[j * num_data + i];
}
std::vector<double> tmp_result(num_class_);
objective_function_->ConvertOutput(tree_pred.data(), tmp_result.data());
for (int j = 0; j < num_class_; ++j) {
out_result[j * num_data + i] = static_cast<double>(tmp_result[j]);
}
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data; ++i) {
for (int j = 0; j < num_tree_per_iteration_; ++j) {
out_result[j * num_data + i] = static_cast<double>(raw_scores[j * num_data + i]);
}
}
}
}
double GBDT::GetUpperBoundValue() const {
double max_value = 0.0;
for (const auto &tree : models_) {
max_value += tree->GetUpperBoundValue();
}
return max_value;
}
double GBDT::GetLowerBoundValue() const {
double min_value = 0.0;
for (const auto &tree : models_) {
min_value += tree->GetLowerBoundValue();
}
return min_value;
}
void GBDT::ResetTrainingData(const Dataset* train_data, const ObjectiveFunction* objective_function,
const std::vector<const Metric*>& training_metrics) {
if (train_data != train_data_ && !train_data_->CheckAlign(*train_data)) {
Log::Fatal("Cannot reset training data, since new training data has different bin mappers");
}
objective_function_ = objective_function;
data_sample_strategy_->UpdateObjectiveFunction(objective_function);
if (objective_function_ != nullptr) {
CHECK_EQ(num_tree_per_iteration_, objective_function_->NumModelPerIteration());
if (objective_function_->IsRenewTreeOutput() && !config_->monotone_constraints.empty()) {
Log::Fatal("Cannot use ``monotone_constraints`` in %s objective, please disable it.", objective_function_->GetName());
}
}
is_constant_hessian_ = GetIsConstHessian(objective_function);
// push training metrics
training_metrics_.clear();
for (const auto& metric : training_metrics) {
training_metrics_.push_back(metric);
}
training_metrics_.shrink_to_fit();
#ifdef USE_CUDA
boosting_on_gpu_ = objective_function_ != nullptr && objective_function_->IsCUDAObjective() &&
!data_sample_strategy_->IsHessianChange(); // for sample strategy with Hessian change, fall back to boosting on CPU
tree_learner_->ResetBoostingOnGPU(boosting_on_gpu_);
#endif // USE_CUDA
if (train_data != train_data_) {
train_data_ = train_data;
data_sample_strategy_->UpdateTrainingData(train_data);
// not same training data, need reset score and others
// create score tracker
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda")) {
train_score_updater_.reset(new CUDAScoreUpdater(train_data_, num_tree_per_iteration_, boosting_on_gpu_));
} else {
#endif // USE_CUDA
train_score_updater_.reset(new ScoreUpdater(train_data_, num_tree_per_iteration_));
#ifdef USE_CUDA
}
#endif // USE_CUDA
// update score
for (int i = 0; i < iter_; ++i) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
auto curr_tree = (i + num_init_iteration_) * num_tree_per_iteration_ + cur_tree_id;
train_score_updater_->AddScore(models_[curr_tree].get(), cur_tree_id);
}
}
num_data_ = train_data_->num_data();
ResetGradientBuffers();
max_feature_idx_ = train_data_->num_total_features() - 1;
label_idx_ = train_data_->label_idx();
feature_names_ = train_data_->feature_names();
feature_infos_ = train_data_->feature_infos();
parser_config_str_ = train_data_->parser_config_str();
tree_learner_->ResetTrainingData(train_data, is_constant_hessian_);
data_sample_strategy_->ResetSampleConfig(config_.get(), true);
} else {
tree_learner_->ResetIsConstantHessian(is_constant_hessian_);
}
}
void GBDT::ResetConfig(const Config* config) {
auto new_config = std::unique_ptr<Config>(new Config(*config));
if (!config->monotone_constraints.empty()) {
CHECK_EQ(static_cast<size_t>(train_data_->num_total_features()), config->monotone_constraints.size());
}
if (!config->feature_contri.empty()) {
CHECK_EQ(static_cast<size_t>(train_data_->num_total_features()), config->feature_contri.size());
}
if (objective_function_ != nullptr && objective_function_->IsRenewTreeOutput() && !config->monotone_constraints.empty()) {
Log::Fatal("Cannot use ``monotone_constraints`` in %s objective, please disable it.", objective_function_->GetName());
}
early_stopping_round_ = new_config->early_stopping_round;
shrinkage_rate_ = new_config->learning_rate;
if (tree_learner_ != nullptr) {
tree_learner_->ResetConfig(new_config.get());
}
boosting_on_gpu_ = objective_function_ != nullptr && objective_function_->IsCUDAObjective() &&
!data_sample_strategy_->IsHessianChange(); // for sample strategy with Hessian change, fall back to boosting on CPU
tree_learner_->ResetBoostingOnGPU(boosting_on_gpu_);
if (train_data_ != nullptr) {
data_sample_strategy_->ResetSampleConfig(new_config.get(), false);
if (data_sample_strategy_->NeedResizeGradients()) {
// resize gradient vectors to copy the customized gradients for goss or bagging with subset
ResetGradientBuffers();
}
}
if (config_.get() != nullptr && config_->forcedsplits_filename != new_config->forcedsplits_filename) {
// load forced_splits file
if (!new_config->forcedsplits_filename.empty()) {
std::ifstream forced_splits_file(
new_config->forcedsplits_filename.c_str());
std::stringstream buffer;
buffer << forced_splits_file.rdbuf();
std::string err;
forced_splits_json_ = Json::parse(buffer.str(), &err);
tree_learner_->SetForcedSplit(&forced_splits_json_);
} else {
forced_splits_json_ = Json();
tree_learner_->SetForcedSplit(nullptr);
}
}
config_.reset(new_config.release());
}
void GBDT::ResetGradientBuffers() {
const size_t total_size = static_cast<size_t>(num_data_) * num_tree_per_iteration_;
const bool is_use_subset = data_sample_strategy_->is_use_subset();
const data_size_t bag_data_cnt = data_sample_strategy_->bag_data_cnt();
if (objective_function_ != nullptr) {
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda") && boosting_on_gpu_) {
if (cuda_gradients_.Size() < total_size) {
cuda_gradients_.Resize(total_size);
cuda_hessians_.Resize(total_size);
}
gradients_pointer_ = cuda_gradients_.RawData();
hessians_pointer_ = cuda_hessians_.RawData();
} else {
#endif // USE_CUDA
if (gradients_.size() < total_size) {
gradients_.resize(total_size);
hessians_.resize(total_size);
}
gradients_pointer_ = gradients_.data();
hessians_pointer_ = hessians_.data();
#ifdef USE_CUDA
}
#endif // USE_CUDA
} else if (data_sample_strategy_->IsHessianChange() || (is_use_subset && bag_data_cnt < num_data_ && !boosting_on_gpu_)) {
if (gradients_.size() < total_size) {
gradients_.resize(total_size);
hessians_.resize(total_size);
}
gradients_pointer_ = gradients_.data();
hessians_pointer_ = hessians_.data();
}
}
} // namespace LightGBM
src/boosting/gbdt.h
+625
View File
@@ -0,0 +1,625 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_BOOSTING_GBDT_H_
#define LIGHTGBM_SRC_BOOSTING_GBDT_H_
#include <LightGBM/boosting.h>
#include <LightGBM/objective_function.h>
#include <LightGBM/prediction_early_stop.h>
#include <LightGBM/cuda/vector_cudahost.h>
#include <LightGBM/utils/json11.h>
#include <LightGBM/utils/threading.h>
#include <LightGBM/sample_strategy.h>
#include <string>
#include <algorithm>
#include <cstdio>
#include <fstream>
#include <map>
#include <memory>
#include <mutex>
#include <unordered_map>
#include <utility>
#include <vector>
#include "cuda/cuda_score_updater.hpp"
#include "score_updater.hpp"
namespace LightGBM {
using json11_internal_lightgbm::Json;
/*!
* \brief GBDT algorithm implementation. including Training, prediction, bagging.
*/
class GBDT : public GBDTBase {
public:
/*!
* \brief Constructor
*/
GBDT();
/*!
* \brief Destructor
*/
~GBDT();
/*!
* \brief Initialization logic
* \param gbdt_config Config for boosting
* \param train_data Training data
* \param objective_function Training objective function
* \param training_metrics Training metrics
*/
void Init(const Config* gbdt_config, const Dataset* train_data,
const ObjectiveFunction* objective_function,
const std::vector<const Metric*>& training_metrics) override;
/*!
* \brief Traverse the tree of forced splits and check that all indices are less than the number of features.
*/
void CheckForcedSplitFeatures();
/*!
* \brief Merge model from other boosting object. Will insert to the front of current boosting object
* \param other
*/
void MergeFrom(const Boosting* other) override {
auto other_gbdt = reinterpret_cast<const GBDT*>(other);
// tmp move to other vector
auto original_models = std::move(models_);
models_ = std::vector<std::unique_ptr<Tree>>();
// push model from other first
for (const auto& tree : other_gbdt->models_) {
auto new_tree = std::unique_ptr<Tree>(new Tree(*(tree.get())));
models_.push_back(std::move(new_tree));
}
num_init_iteration_ = static_cast<int>(models_.size()) / num_tree_per_iteration_;
// push model in current object
for (const auto& tree : original_models) {
auto new_tree = std::unique_ptr<Tree>(new Tree(*(tree.get())));
models_.push_back(std::move(new_tree));
}
num_iteration_for_pred_ = static_cast<int>(models_.size()) / num_tree_per_iteration_;
}
void ShuffleModels(int start_iter, int end_iter) override {
int total_iter = static_cast<int>(models_.size()) / num_tree_per_iteration_;
start_iter = std::max(0, start_iter);
if (end_iter <= 0) {
end_iter = total_iter;
}
end_iter = std::min(total_iter, end_iter);
auto original_models = std::move(models_);
std::vector<int> indices(total_iter);
for (int i = 0; i < total_iter; ++i) {
indices[i] = i;
}
Random tmp_rand(17);
for (int i = start_iter; i < end_iter - 1; ++i) {
int j = tmp_rand.NextShort(i + 1, end_iter);
std::swap(indices[i], indices[j]);
}
models_ = std::vector<std::unique_ptr<Tree>>();
for (int i = 0; i < total_iter; ++i) {
for (int j = 0; j < num_tree_per_iteration_; ++j) {
int tree_idx = indices[i] * num_tree_per_iteration_ + j;
auto new_tree = std::unique_ptr<Tree>(new Tree(*(original_models[tree_idx].get())));
models_.push_back(std::move(new_tree));
}
}
}
/*!
* \brief Reset the training data
* \param train_data New Training data
* \param objective_function Training objective function
* \param training_metrics Training metrics
*/
void ResetTrainingData(const Dataset* train_data, const ObjectiveFunction* objective_function,
const std::vector<const Metric*>& training_metrics) override;
/*!
* \brief Reset Boosting Config
* \param gbdt_config Config for boosting
*/
void ResetConfig(const Config* gbdt_config) override;
/*!
* \brief Adding a validation dataset
* \param valid_data Validation dataset
* \param valid_metrics Metrics for validation dataset
*/
void AddValidDataset(const Dataset* valid_data,
const std::vector<const Metric*>& valid_metrics) override;
/*!
* \brief Perform a full training procedure
* \param snapshot_freq frequency of snapshot
* \param model_output_path path of model file
*/
void Train(int snapshot_freq, const std::string& model_output_path) override;
void RefitTree(const int* tree_leaf_prediction, const size_t nrow, const size_t ncol) override;
/*!
* \brief Training logic
* \param gradients nullptr for using default objective, otherwise use self-defined boosting
* \param hessians nullptr for using default objective, otherwise use self-defined boosting
* \return True if cannot train any more
*/
bool TrainOneIter(const score_t* gradients, const score_t* hessians) override;
/*!
* \brief Rollback one iteration
*/
void RollbackOneIter() override;
/*!
* \brief Get current iteration
*/
int GetCurrentIteration() const override { return static_cast<int>(models_.size()) / num_tree_per_iteration_; }
/*!
* \brief Get parameters as a JSON string
*/
std::string GetLoadedParam() const override {
if (loaded_parameter_.empty()) {
return std::string("{}");
}
const auto param_types = Config::ParameterTypes();
const auto lines = Common::Split(loaded_parameter_.c_str(), "\n");
bool first = true;
std::stringstream str_buf;
str_buf << "{";
for (const auto& line : lines) {
const auto pair = Common::Split(line.c_str(), ":");
if (pair[1] == " ]")
continue;
const auto param = pair[0].substr(1);
const auto value_str = pair[1].substr(1, pair[1].size() - 2);
auto iter = param_types.find(param);
if (iter == param_types.end()) {
Log::Warning("Ignoring unrecognized parameter '%s' found in model string.", param.c_str());
continue;
}
std::string param_type = iter->second;
if (first) {
first = false;
str_buf << "\"";
} else {
str_buf << ",\"";
}
str_buf << param << "\": ";
if (param_type == "string") {
str_buf << "\"" << value_str << "\"";
} else if (param_type == "int") {
int value;
Common::Atoi(value_str.c_str(), &value);
str_buf << value;
} else if (param_type == "double") {
double value;
Common::Atof(value_str.c_str(), &value);
str_buf << value;
} else if (param_type == "bool") {
bool value = value_str == "1";
str_buf << std::boolalpha << value;
} else if (param_type.substr(0, 6) == "vector") {
str_buf << "[";
if (param_type.substr(7, 6) == "string") {
const auto parts = Common::Split(value_str.c_str(), ",");
str_buf << "\"" << Common::Join(parts, "\",\"") << "\"";
} else {
str_buf << value_str;
}
str_buf << "]";
}
}
str_buf << "}";
return str_buf.str();
}
/*!
* \brief Can use early stopping for prediction or not
* \return True if cannot use early stopping for prediction
*/
bool NeedAccuratePrediction() const override {
if (objective_function_ == nullptr) {
return true;
} else {
return objective_function_->NeedAccuratePrediction();
}
}
/*!
* \brief Get evaluation result at data_idx data
* \param data_idx 0: training data, 1: 1st validation data
* \return evaluation result
*/
std::vector<double> GetEvalAt(int data_idx) const override;
/*!
* \brief Get current training score
* \param out_len length of returned score
* \return training score
*/
const double* GetTrainingScore(int64_t* out_len) override;
/*!
* \brief Get size of prediction at data_idx data
* \param data_idx 0: training data, 1: 1st validation data
* \return The size of prediction
*/
int64_t GetNumPredictAt(int data_idx) const override {
CHECK(data_idx >= 0 && data_idx <= static_cast<int>(valid_score_updater_.size()));
data_size_t num_data = train_data_->num_data();
if (data_idx > 0) {
num_data = valid_score_updater_[data_idx - 1]->num_data();
}
return static_cast<int64_t>(num_data) * num_class_;
}
/*!
* \brief Get prediction result at data_idx data
* \param data_idx 0: training data, 1: 1st validation data
* \param result used to store prediction result, should allocate memory before call this function
* \param out_len length of returned score
*/
void GetPredictAt(int data_idx, double* out_result, int64_t* out_len) override;
/*!
* \brief Get number of prediction for one data
* \param start_iteration Start index of the iteration to predict
* \param num_iteration number of used iterations
* \param is_pred_leaf True if predicting leaf index
* \param is_pred_contrib True if predicting feature contribution
* \return number of prediction
*/
inline int NumPredictOneRow(int start_iteration, int num_iteration, bool is_pred_leaf, bool is_pred_contrib) const override {
int num_pred_in_one_row = num_class_;
if (is_pred_leaf) {
int max_iteration = GetCurrentIteration();
start_iteration = std::max(start_iteration, 0);
start_iteration = std::min(start_iteration, max_iteration);
if (num_iteration > 0) {
num_pred_in_one_row *= static_cast<int>(std::min(max_iteration - start_iteration, num_iteration));
} else {
num_pred_in_one_row *= (max_iteration - start_iteration);
}
} else if (is_pred_contrib) {
num_pred_in_one_row = num_tree_per_iteration_ * (max_feature_idx_ + 2); // +1 for 0-based indexing, +1 for baseline
}
return num_pred_in_one_row;
}
void PredictRaw(const double* features, double* output,
const PredictionEarlyStopInstance* earlyStop) const override;
void PredictRawByMap(const std::unordered_map<int, double>& features, double* output,
const PredictionEarlyStopInstance* early_stop) const override;
void Predict(const double* features, double* output,
const PredictionEarlyStopInstance* earlyStop) const override;
void PredictByMap(const std::unordered_map<int, double>& features, double* output,
const PredictionEarlyStopInstance* early_stop) const override;
void PredictLeafIndex(const double* features, double* output) const override;
void PredictLeafIndexByMap(const std::unordered_map<int, double>& features, double* output) const override;
void PredictContrib(const double* features, double* output) const override;
void PredictContribByMap(const std::unordered_map<int, double>& features,
std::vector<std::unordered_map<int, double>>* output) const override;
/*!
* \brief Dump model to json format string
* \param start_iteration The model will be saved start from
* \param num_iteration Number of iterations that want to dump, -1 means dump all
* \param feature_importance_type Type of feature importance, 0: split, 1: gain
* \return Json format string of model
*/
std::string DumpModel(int start_iteration, int num_iteration,
int feature_importance_type) const override;
/*!
* \brief Translate model to if-else statement
* \param num_iteration Number of iterations that want to translate, -1 means translate all
* \return if-else format codes of model
*/
std::string ModelToIfElse(int num_iteration) const override;
/*!
* \brief Translate model to if-else statement
* \param num_iteration Number of iterations that want to translate, -1 means translate all
* \param filename Filename that want to save to
* \return is_finish Is training finished or not
*/
bool SaveModelToIfElse(int num_iteration, const char* filename) const override;
/*!
* \brief Save model to file
* \param start_iteration The model will be saved start from
* \param num_iterations Number of model that want to save, -1 means save all
* \param feature_importance_type Type of feature importance, 0: split, 1: gain
* \param filename Filename that want to save to
* \return is_finish Is training finished or not
*/
bool SaveModelToFile(int start_iteration, int num_iterations,
int feature_importance_type,
const char* filename) const override;
/*!
* \brief Save model to string
* \param start_iteration The model will be saved start from
* \param num_iterations Number of model that want to save, -1 means save all
* \param feature_importance_type Type of feature importance, 0: split, 1: gain
* \return Non-empty string if succeeded
*/
std::string SaveModelToString(int start_iteration, int num_iterations, int feature_importance_type) const override;
/*!
* \brief Restore from a serialized buffer
*/
bool LoadModelFromString(const char* buffer, size_t len) override;
/*!
* \brief Calculate feature importances
* \param num_iteration Number of model that want to use for feature importance, -1 means use all
* \param importance_type: 0 for split, 1 for gain
* \return vector of feature_importance
*/
std::vector<double> FeatureImportance(int num_iteration, int importance_type) const override;
/*!
* \brief Calculate upper bound value
* \return upper bound value
*/
double GetUpperBoundValue() const override;
/*!
* \brief Calculate lower bound value
* \return lower bound value
*/
double GetLowerBoundValue() const override;
/*!
* \brief Get max feature index of this model
* \return Max feature index of this model
*/
inline int MaxFeatureIdx() const override { return max_feature_idx_; }
/*!
* \brief Get feature names of this model
* \return Feature names of this model
*/
inline std::vector<std::string> FeatureNames() const override { return feature_names_; }
/*!
* \brief Get index of label column
* \return index of label column
*/
inline int LabelIdx() const override { return label_idx_; }
/*!
* \brief Get number of weak sub-models
* \return Number of weak sub-models
*/
inline int NumberOfTotalModel() const override { return static_cast<int>(models_.size()); }
/*!
* \brief Get number of tree per iteration
* \return number of tree per iteration
*/
inline int NumModelPerIteration() const override { return num_tree_per_iteration_; }
/*!
* \brief Get number of classes
* \return Number of classes
*/
inline int NumberOfClasses() const override { return num_class_; }
inline void InitPredict(int start_iteration, int num_iteration, bool is_pred_contrib) override {
num_iteration_for_pred_ = static_cast<int>(models_.size()) / num_tree_per_iteration_;
start_iteration = std::max(start_iteration, 0);
start_iteration = std::min(start_iteration, num_iteration_for_pred_);
if (num_iteration > 0) {
num_iteration_for_pred_ = std::min(num_iteration, num_iteration_for_pred_ - start_iteration);
} else {
num_iteration_for_pred_ = num_iteration_for_pred_ - start_iteration;
}
start_iteration_for_pred_ = start_iteration;
if (is_pred_contrib && !models_initialized_) {
std::lock_guard<std::mutex> lock(instance_mutex_);
if (models_initialized_)
return;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int i = 0; i < static_cast<int>(models_.size()); ++i) {
models_[i]->RecomputeMaxDepth();
}
models_initialized_ = true;
}
}
inline double GetLeafValue(int tree_idx, int leaf_idx) const override {
CHECK(tree_idx >= 0 && static_cast<size_t>(tree_idx) < models_.size());
CHECK(leaf_idx >= 0 && leaf_idx < models_[tree_idx]->num_leaves());
return models_[tree_idx]->LeafOutput(leaf_idx);
}
inline void SetLeafValue(int tree_idx, int leaf_idx, double val) override {
CHECK(tree_idx >= 0 && static_cast<size_t>(tree_idx) < models_.size());
CHECK(leaf_idx >= 0 && leaf_idx < models_[tree_idx]->num_leaves());
models_[tree_idx]->SetLeafOutput(leaf_idx, val);
}
/*!
* \brief Get Type name of this boosting object
*/
const char* SubModelName() const override { return "tree"; }
bool IsLinear() const override { return linear_tree_; }
inline std::string ParserConfigStr() const override {return parser_config_str_;}
protected:
virtual bool GetIsConstHessian(const ObjectiveFunction* objective_function) {
if (objective_function != nullptr && !data_sample_strategy_->IsHessianChange()) {
return objective_function->IsConstantHessian();
} else {
return false;
}
}
/*!
* \brief Print eval result and check early stopping
*/
virtual bool EvalAndCheckEarlyStopping();
/*!
* \brief reset config for bagging
*/
void ResetBaggingConfig(const Config* config, bool is_change_dataset);
/*!
* \brief calculate the objective function
*/
virtual void Boosting();
/*!
* \brief updating score after tree was trained
* \param tree Trained tree of this iteration
* \param cur_tree_id Current tree for multiclass training
*/
virtual void UpdateScore(const Tree* tree, const int cur_tree_id);
/*!
* \brief eval results for one metric
*/
virtual std::vector<double> EvalOneMetric(const Metric* metric, const double* score, const data_size_t num_data) const;
/*!
* \brief Print metric result of current iteration
* \param iter Current iteration
* \return best_msg if met early_stopping
*/
std::string OutputMetric(int iter);
virtual double BoostFromAverage(int class_id, bool update_scorer);
/*!
* \brief Reset gradient buffers, must be called after sample strategy is reset
*/
void ResetGradientBuffers();
/*! \brief current iteration */
int iter_;
/*! \brief Pointer to training data */
const Dataset* train_data_;
/*! \brief Config of gbdt */
std::unique_ptr<Config> config_;
/*! \brief Tree learner, will use this class to learn trees */
std::unique_ptr<TreeLearner> tree_learner_;
/*! \brief Objective function */
const ObjectiveFunction* objective_function_;
/*! \brief Store and update training data's score */
std::unique_ptr<ScoreUpdater> train_score_updater_;
/*! \brief Metrics for training data */
std::vector<const Metric*> training_metrics_;
/*! \brief Store and update validation data's scores */
std::vector<std::unique_ptr<ScoreUpdater>> valid_score_updater_;
/*! \brief Metric for validation data */
std::vector<std::vector<const Metric*>> valid_metrics_;
/*! \brief Number of rounds for early stopping */
int early_stopping_round_;
/*! \brief Minimum improvement for early stopping */
double early_stopping_min_delta_;
/*! \brief Only use first metric for early stopping */
bool es_first_metric_only_;
/*! \brief Best iteration(s) for early stopping */
std::vector<std::vector<int>> best_iter_;
/*! \brief Best score(s) for early stopping */
std::vector<std::vector<double>> best_score_;
/*! \brief output message of best iteration */
std::vector<std::vector<std::string>> best_msg_;
/*! \brief Trained models(trees) */
std::vector<std::unique_ptr<Tree>> models_;
/*! \brief Max feature index of training data*/
int max_feature_idx_;
/*! \brief Parser config file content */
std::string parser_config_str_ = "";
/*! \brief Are the models initialized (passed RecomputeMaxDepth phase) */
bool models_initialized_ = false;
/*! \brief Mutex for exclusive models initialization */
std::mutex instance_mutex_;
#ifdef USE_CUDA
/*! \brief First order derivative of training data */
std::vector<score_t, CHAllocator<score_t>> gradients_;
/*! \brief Second order derivative of training data */
std::vector<score_t, CHAllocator<score_t>> hessians_;
#else
/*! \brief First order derivative of training data */
std::vector<score_t, Common::AlignmentAllocator<score_t, kAlignedSize>> gradients_;
/*! \brief Second order derivative of training data */
std::vector<score_t, Common::AlignmentAllocator<score_t, kAlignedSize>> hessians_;
#endif
/*! \brief Pointer to gradient vector, can be on CPU or GPU */
score_t* gradients_pointer_;
/*! \brief Pointer to hessian vector, can be on CPU or GPU */
score_t* hessians_pointer_;
/*! \brief Whether boosting is done on GPU, used for device_type=cuda */
bool boosting_on_gpu_;
#ifdef USE_CUDA
/*! \brief Gradient vector on GPU */
CUDAVector<score_t> cuda_gradients_;
/*! \brief Hessian vector on GPU */
CUDAVector<score_t> cuda_hessians_;
/*! \brief Buffer for scores when boosting is on GPU but evaluation is not, used only with device_type=cuda */
mutable std::vector<double> host_score_;
/*! \brief Buffer for scores when boosting is not on GPU but evaluation is, used only with device_type=cuda */
mutable CUDAVector<double> cuda_score_;
#endif // USE_CUDA
/*! \brief Number of training data */
data_size_t num_data_;
/*! \brief Number of trees per iterations */
int num_tree_per_iteration_;
/*! \brief Number of class */
int num_class_;
/*! \brief Index of label column */
data_size_t label_idx_;
/*! \brief number of used model */
int num_iteration_for_pred_;
/*! \brief Start iteration of used model */
int start_iteration_for_pred_;
/*! \brief Shrinkage rate for one iteration */
double shrinkage_rate_;
/*! \brief Number of loaded initial models */
int num_init_iteration_;
/*! \brief Feature names */
std::vector<std::string> feature_names_;
std::vector<std::string> feature_infos_;
std::vector<bool> class_need_train_;
bool is_constant_hessian_;
std::unique_ptr<ObjectiveFunction> loaded_objective_;
bool average_output_;
bool need_re_bagging_;
bool balanced_bagging_;
std::string loaded_parameter_;
std::vector<int8_t> monotone_constraints_;
Json forced_splits_json_;
bool linear_tree_;
std::unique_ptr<SampleStrategy> data_sample_strategy_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_BOOSTING_GBDT_H_
src/boosting/gbdt_model_text.cpp
+667
View File
@@ -0,0 +1,667 @@
/*!
* Copyright (c) 2017-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2017-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/config.h>
#include <LightGBM/metric.h>
#include <LightGBM/objective_function.h>
#include <LightGBM/utils/array_args.h>
#include <LightGBM/utils/common.h>
#include <algorithm>
#include <string>
#include <sstream>
#include <unordered_map>
#include <utility>
#include <vector>
#include "gbdt.h"
namespace LightGBM {
const char* kModelVersion = "v4";
std::string GBDT::DumpModel(int start_iteration, int num_iteration, int feature_importance_type) const {
std::stringstream str_buf;
Common::C_stringstream(str_buf);
str_buf << "{";
str_buf << "\"name\":\"" << SubModelName() << "\"," << '\n';
str_buf << "\"version\":\"" << kModelVersion << "\"," << '\n';
str_buf << "\"num_class\":" << num_class_ << "," << '\n';
str_buf << "\"num_tree_per_iteration\":" << num_tree_per_iteration_ << "," << '\n';
str_buf << "\"label_index\":" << label_idx_ << "," << '\n';
str_buf << "\"max_feature_idx\":" << max_feature_idx_ << "," << '\n';
if (objective_function_ != nullptr) {
str_buf << "\"objective\":\"" << objective_function_->ToString() << "\",\n";
}
str_buf << "\"average_output\":" << (average_output_ ? "true" : "false") << ",\n";
str_buf << "\"feature_names\":[\"" << CommonC::Join(feature_names_, "\",\"")
<< "\"]," << '\n';
str_buf << "\"monotone_constraints\":["
<< CommonC::Join(monotone_constraints_, ",") << "]," << '\n';
str_buf << "\"feature_infos\":" << "{";
bool first_obj = true;
for (size_t i = 0; i < feature_infos_.size(); ++i) {
std::stringstream json_str_buf;
Common::C_stringstream(json_str_buf);
auto strs = Common::Split(feature_infos_[i].c_str(), ":");
if (strs[0][0] == '[') {
strs[0].erase(0, 1); // remove '['
strs[1].erase(strs[1].size() - 1); // remove ']'
double max_, min_;
Common::Atof(strs[0].c_str(), &min_);
Common::Atof(strs[1].c_str(), &max_);
json_str_buf << std::setprecision(std::numeric_limits<double>::digits10 + 2);
json_str_buf << "{\"min_value\":" << Common::AvoidInf(min_) << ",";
json_str_buf << "\"max_value\":" << Common::AvoidInf(max_) << ",";
json_str_buf << "\"values\":[]}";
} else if (strs[0] != "none") { // categorical feature
auto vals = CommonC::StringToArray<int>(feature_infos_[i], ':');
auto max_idx = ArrayArgs<int>::ArgMax(vals);
auto min_idx = ArrayArgs<int>::ArgMin(vals);
json_str_buf << "{\"min_value\":" << vals[min_idx] << ",";
json_str_buf << "\"max_value\":" << vals[max_idx] << ",";
json_str_buf << "\"values\":[" << CommonC::Join(vals, ",") << "]}";
} else { // unused feature
continue;
}
if (!first_obj) {
str_buf << ",";
}
str_buf << "\"" << feature_names_[i] << "\":";
str_buf << json_str_buf.str();
first_obj = false;
}
str_buf << "}," << '\n';
str_buf << "\"tree_info\":[";
int num_used_model = static_cast<int>(models_.size());
int total_iteration = num_used_model / num_tree_per_iteration_;
start_iteration = std::max(start_iteration, 0);
start_iteration = std::min(start_iteration, total_iteration);
if (num_iteration > 0) {
int end_iteration = start_iteration + num_iteration;
num_used_model = std::min(end_iteration * num_tree_per_iteration_ , num_used_model);
}
int start_model = start_iteration * num_tree_per_iteration_;
for (int i = start_model; i < num_used_model; ++i) {
if (i > start_model) {
str_buf << ",";
}
str_buf << "{";
str_buf << "\"tree_index\":" << i << ",";
str_buf << models_[i]->ToJSON();
str_buf << "}";
}
str_buf << "]," << '\n';
std::vector<double> feature_importances = FeatureImportance(
num_iteration, feature_importance_type);
// store the importance first
std::vector<std::pair<size_t, std::string>> pairs;
for (size_t i = 0; i < feature_importances.size(); ++i) {
size_t feature_importances_int = static_cast<size_t>(feature_importances[i]);
if (feature_importances_int > 0) {
pairs.emplace_back(feature_importances_int, feature_names_[i]);
}
}
str_buf << '\n' << "\"feature_importances\":" << "{";
for (size_t i = 0; i < pairs.size(); ++i) {
if (i > 0) {
str_buf << ",";
}
str_buf << "\"" << pairs[i].second << "\":" << std::to_string(pairs[i].first);
}
str_buf << "}" << '\n';
str_buf << "}" << '\n';
return str_buf.str();
}
std::string GBDT::ModelToIfElse(int num_iteration) const {
std::stringstream str_buf;
Common::C_stringstream(str_buf);
str_buf << "#include \"gbdt.h\"" << '\n';
str_buf << "#include <LightGBM/utils/common.h>" << '\n';
str_buf << "#include <LightGBM/objective_function.h>" << '\n';
str_buf << "#include <LightGBM/metric.h>" << '\n';
str_buf << "#include <LightGBM/prediction_early_stop.h>" << '\n';
str_buf << "#include <ctime>" << '\n';
str_buf << "#include <sstream>" << '\n';
str_buf << "#include <chrono>" << '\n';
str_buf << "#include <string>" << '\n';
str_buf << "#include <vector>" << '\n';
str_buf << "#include <utility>" << '\n';
str_buf << "namespace LightGBM {" << '\n';
int num_used_model = static_cast<int>(models_.size());
if (num_iteration > 0) {
num_used_model = std::min(num_iteration * num_tree_per_iteration_, num_used_model);
}
// PredictRaw
for (int i = 0; i < num_used_model; ++i) {
str_buf << models_[i]->ToIfElse(i, false) << '\n';
}
str_buf << "double (*PredictTreePtr[])(const double*) = { ";
for (int i = 0; i < num_used_model; ++i) {
if (i > 0) {
str_buf << " , ";
}
str_buf << "PredictTree" << i;
}
str_buf << " };" << '\n' << '\n';
std::stringstream pred_str_buf;
Common::C_stringstream(pred_str_buf);
pred_str_buf << "\t" << "int early_stop_round_counter = 0;" << '\n';
pred_str_buf << "\t" << "std::memset(output, 0, sizeof(double) * num_tree_per_iteration_);" << '\n';
pred_str_buf << "\t" << "for (int i = 0; i < num_iteration_for_pred_; ++i) {" << '\n';
pred_str_buf << "\t\t" << "for (int k = 0; k < num_tree_per_iteration_; ++k) {" << '\n';
pred_str_buf << "\t\t\t" << "output[k] += (*PredictTreePtr[i * num_tree_per_iteration_ + k])(features);" << '\n';
pred_str_buf << "\t\t" << "}" << '\n';
pred_str_buf << "\t\t" << "++early_stop_round_counter;" << '\n';
pred_str_buf << "\t\t" << "if (early_stop->round_period == early_stop_round_counter) {" << '\n';
pred_str_buf << "\t\t\t" << "if (early_stop->callback_function(output, num_tree_per_iteration_))" << '\n';
pred_str_buf << "\t\t\t\t" << "return;" << '\n';
pred_str_buf << "\t\t\t" << "early_stop_round_counter = 0;" << '\n';
pred_str_buf << "\t\t" << "}" << '\n';
pred_str_buf << "\t" << "}" << '\n';
str_buf << "void GBDT::PredictRaw(const double* features, double *output, const PredictionEarlyStopInstance* early_stop) const {" << '\n';
str_buf << pred_str_buf.str();
str_buf << "}" << '\n';
str_buf << '\n';
// PredictRawByMap
str_buf << "double (*PredictTreeByMapPtr[])(const std::unordered_map<int, double>&) = { ";
for (int i = 0; i < num_used_model; ++i) {
if (i > 0) {
str_buf << " , ";
}
str_buf << "PredictTree" << i << "ByMap";
}
str_buf << " };" << '\n' << '\n';
std::stringstream pred_str_buf_map;
Common::C_stringstream(pred_str_buf_map);
pred_str_buf_map << "\t" << "int early_stop_round_counter = 0;" << '\n';
pred_str_buf_map << "\t" << "std::memset(output, 0, sizeof(double) * num_tree_per_iteration_);" << '\n';
pred_str_buf_map << "\t" << "for (int i = 0; i < num_iteration_for_pred_; ++i) {" << '\n';
pred_str_buf_map << "\t\t" << "for (int k = 0; k < num_tree_per_iteration_; ++k) {" << '\n';
pred_str_buf_map << "\t\t\t" << "output[k] += (*PredictTreeByMapPtr[i * num_tree_per_iteration_ + k])(features);" << '\n';
pred_str_buf_map << "\t\t" << "}" << '\n';
pred_str_buf_map << "\t\t" << "++early_stop_round_counter;" << '\n';
pred_str_buf_map << "\t\t" << "if (early_stop->round_period == early_stop_round_counter) {" << '\n';
pred_str_buf_map << "\t\t\t" << "if (early_stop->callback_function(output, num_tree_per_iteration_))" << '\n';
pred_str_buf_map << "\t\t\t\t" << "return;" << '\n';
pred_str_buf_map << "\t\t\t" << "early_stop_round_counter = 0;" << '\n';
pred_str_buf_map << "\t\t" << "}" << '\n';
pred_str_buf_map << "\t" << "}" << '\n';
str_buf << "void GBDT::PredictRawByMap(const std::unordered_map<int, double>& features, double* output, const PredictionEarlyStopInstance* early_stop) const {" << '\n';
str_buf << pred_str_buf_map.str();
str_buf << "}" << '\n';
str_buf << '\n';
// Predict
str_buf << "void GBDT::Predict(const double* features, double *output, const PredictionEarlyStopInstance* early_stop) const {" << '\n';
str_buf << "\t" << "PredictRaw(features, output, early_stop);" << '\n';
str_buf << "\t" << "if (average_output_) {" << '\n';
str_buf << "\t\t" << "for (int k = 0; k < num_tree_per_iteration_; ++k) {" << '\n';
str_buf << "\t\t\t" << "output[k] /= num_iteration_for_pred_;" << '\n';
str_buf << "\t\t" << "}" << '\n';
str_buf << "\t" << "}" << '\n';
str_buf << "\t" << "if (objective_function_ != nullptr) {" << '\n';
str_buf << "\t\t" << "objective_function_->ConvertOutput(output, output);" << '\n';
str_buf << "\t" << "}" << '\n';
str_buf << "}" << '\n';
str_buf << '\n';
// PredictByMap
str_buf << "void GBDT::PredictByMap(const std::unordered_map<int, double>& features, double* output, const PredictionEarlyStopInstance* early_stop) const {" << '\n';
str_buf << "\t" << "PredictRawByMap(features, output, early_stop);" << '\n';
str_buf << "\t" << "if (average_output_) {" << '\n';
str_buf << "\t\t" << "for (int k = 0; k < num_tree_per_iteration_; ++k) {" << '\n';
str_buf << "\t\t\t" << "output[k] /= num_iteration_for_pred_;" << '\n';
str_buf << "\t\t" << "}" << '\n';
str_buf << "\t" << "}" << '\n';
str_buf << "\t" << "if (objective_function_ != nullptr) {" << '\n';
str_buf << "\t\t" << "objective_function_->ConvertOutput(output, output);" << '\n';
str_buf << "\t" << "}" << '\n';
str_buf << "}" << '\n';
str_buf << '\n';
// PredictLeafIndex
for (int i = 0; i < num_used_model; ++i) {
str_buf << models_[i]->ToIfElse(i, true) << '\n';
}
str_buf << "double (*PredictTreeLeafPtr[])(const double*) = { ";
for (int i = 0; i < num_used_model; ++i) {
if (i > 0) {
str_buf << " , ";
}
str_buf << "PredictTree" << i << "Leaf";
}
str_buf << " };" << '\n' << '\n';
str_buf << "void GBDT::PredictLeafIndex(const double* features, double *output) const {" << '\n';
str_buf << "\t" << "int total_tree = num_iteration_for_pred_ * num_tree_per_iteration_;" << '\n';
str_buf << "\t" << "for (int i = 0; i < total_tree; ++i) {" << '\n';
str_buf << "\t\t" << "output[i] = (*PredictTreeLeafPtr[i])(features);" << '\n';
str_buf << "\t" << "}" << '\n';
str_buf << "}" << '\n';
// PredictLeafIndexByMap
str_buf << "double (*PredictTreeLeafByMapPtr[])(const std::unordered_map<int, double>&) = { ";
for (int i = 0; i < num_used_model; ++i) {
if (i > 0) {
str_buf << " , ";
}
str_buf << "PredictTree" << i << "LeafByMap";
}
str_buf << " };" << '\n' << '\n';
str_buf << "void GBDT::PredictLeafIndexByMap(const std::unordered_map<int, double>& features, double* output) const {" << '\n';
str_buf << "\t" << "int total_tree = num_iteration_for_pred_ * num_tree_per_iteration_;" << '\n';
str_buf << "\t" << "for (int i = 0; i < total_tree; ++i) {" << '\n';
str_buf << "\t\t" << "output[i] = (*PredictTreeLeafByMapPtr[i])(features);" << '\n';
str_buf << "\t" << "}" << '\n';
str_buf << "}" << '\n';
str_buf << "} // namespace LightGBM" << '\n';
return str_buf.str();
}
bool GBDT::SaveModelToIfElse(int num_iteration, const char* filename) const {
/*! \brief File to write models */
std::ofstream output_file;
std::ifstream ifs(filename);
if (ifs.good()) {
std::string origin((std::istreambuf_iterator<char>(ifs)),
(std::istreambuf_iterator<char>()));
output_file.open(filename);
output_file << "#define USE_HARD_CODE 0" << '\n';
output_file << "#ifndef USE_HARD_CODE" << '\n';
output_file << origin << '\n';
output_file << "#else" << '\n';
output_file << ModelToIfElse(num_iteration);
output_file << "#endif" << '\n';
} else {
output_file.open(filename);
output_file << ModelToIfElse(num_iteration);
}
ifs.close();
output_file.close();
return static_cast<bool>(output_file);
}
std::string GBDT::SaveModelToString(int start_iteration, int num_iteration, int feature_importance_type) const {
std::stringstream ss;
Common::C_stringstream(ss);
// output model type
ss << SubModelName() << '\n';
ss << "version=" << kModelVersion << '\n';
// output number of class
ss << "num_class=" << num_class_ << '\n';
ss << "num_tree_per_iteration=" << num_tree_per_iteration_ << '\n';
// output label index
ss << "label_index=" << label_idx_ << '\n';
// output max_feature_idx
ss << "max_feature_idx=" << max_feature_idx_ << '\n';
// output objective
if (objective_function_ != nullptr) {
ss << "objective=" << objective_function_->ToString() << '\n';
}
if (average_output_) {
ss << "average_output" << '\n';
}
ss << "feature_names=" << CommonC::Join(feature_names_, " ") << '\n';
if (monotone_constraints_.size() != 0) {
ss << "monotone_constraints=" << CommonC::Join(monotone_constraints_, " ")
<< '\n';
}
ss << "feature_infos=" << CommonC::Join(feature_infos_, " ") << '\n';
int num_used_model = static_cast<int>(models_.size());
int total_iteration = num_used_model / num_tree_per_iteration_;
start_iteration = std::max(start_iteration, 0);
start_iteration = std::min(start_iteration, total_iteration);
if (num_iteration > 0) {
int end_iteration = start_iteration + num_iteration;
num_used_model = std::min(end_iteration * num_tree_per_iteration_, num_used_model);
}
int start_model = start_iteration * num_tree_per_iteration_;
std::vector<std::string> tree_strs(num_used_model - start_model);
std::vector<size_t> tree_sizes(num_used_model - start_model);
// output tree models
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int i = start_model; i < num_used_model; ++i) {
const int idx = i - start_model;
tree_strs[idx] = "Tree=" + std::to_string(idx) + '\n';
tree_strs[idx] += models_[i]->ToString() + '\n';
tree_sizes[idx] = tree_strs[idx].size();
}
ss << "tree_sizes=" << CommonC::Join(tree_sizes, " ") << '\n';
ss << '\n';
for (int i = 0; i < num_used_model - start_model; ++i) {
ss << tree_strs[i];
tree_strs[i].clear();
}
ss << "end of trees" << "\n";
std::vector<double> feature_importances = FeatureImportance(
num_iteration, feature_importance_type);
// store the importance first
std::vector<std::pair<size_t, std::string>> pairs;
for (size_t i = 0; i < feature_importances.size(); ++i) {
size_t feature_importances_int = static_cast<size_t>(feature_importances[i]);
if (feature_importances_int > 0) {
pairs.emplace_back(feature_importances_int, feature_names_[i]);
}
}
// sort the importance
std::stable_sort(pairs.begin(), pairs.end(),
[](const std::pair<size_t, std::string>& lhs,
const std::pair<size_t, std::string>& rhs) {
return lhs.first > rhs.first;
});
ss << '\n' << "feature_importances:" << '\n';
for (size_t i = 0; i < pairs.size(); ++i) {
ss << pairs[i].second << "=" << std::to_string(pairs[i].first) << '\n';
}
if (config_ != nullptr) {
ss << "\nparameters:" << '\n';
ss << config_->ToString() << "\n";
ss << "end of parameters" << '\n';
} else if (!loaded_parameter_.empty()) {
ss << "\nparameters:" << '\n';
ss << loaded_parameter_ << "\n";
ss << "end of parameters" << '\n';
}
if (!parser_config_str_.empty()) {
ss << "\nparser:" << '\n';
ss << parser_config_str_ << "\n";
ss << "end of parser" << '\n';
}
return ss.str();
}
bool GBDT::SaveModelToFile(int start_iteration, int num_iteration, int feature_importance_type, const char* filename) const {
/*! \brief File to write models */
auto writer = VirtualFileWriter::Make(filename);
if (!writer->Init()) {
Log::Fatal("Model file %s is not available for writes", filename);
}
std::string str_to_write = SaveModelToString(start_iteration, num_iteration, feature_importance_type);
auto size = writer->Write(str_to_write.c_str(), str_to_write.size());
return size > 0;
}
bool GBDT::LoadModelFromString(const char* buffer, size_t len) {
// use serialized string to restore this object
models_.clear();
auto c_str = buffer;
auto p = c_str;
auto end = p + len;
std::unordered_map<std::string, std::string> key_vals;
while (p < end) {
auto line_len = Common::GetLine(p);
if (line_len > 0) {
std::string cur_line(p, line_len);
if (!Common::StartsWith(cur_line, "Tree=")) {
auto strs = Common::Split(cur_line.c_str(), '=');
if (strs.size() == 1) {
key_vals[strs[0]] = "";
} else if (strs.size() == 2) {
key_vals[strs[0]] = strs[1];
} else if (strs.size() > 2) {
if (strs[0] == "feature_names") {
key_vals[strs[0]] = cur_line.substr(std::strlen("feature_names="));
} else if (strs[0] == "monotone_constraints") {
key_vals[strs[0]] = cur_line.substr(std::strlen("monotone_constraints="));
} else {
// Use first 128 chars to avoid exceed the message buffer.
Log::Fatal("Wrong line at model file: %s", cur_line.substr(0, std::min<size_t>(128, cur_line.size())).c_str());
}
}
} else {
break;
}
}
p += line_len;
p = Common::SkipNewLine(p);
}
// get number of classes
if (key_vals.count("num_class")) {
Common::Atoi(key_vals["num_class"].c_str(), &num_class_);
} else {
Log::Fatal("Model file doesn't specify the number of classes");
return false;
}
if (key_vals.count("num_tree_per_iteration")) {
Common::Atoi(key_vals["num_tree_per_iteration"].c_str(), &num_tree_per_iteration_);
} else {
num_tree_per_iteration_ = num_class_;
}
// get index of label
if (key_vals.count("label_index")) {
Common::Atoi(key_vals["label_index"].c_str(), &label_idx_);
} else {
Log::Fatal("Model file doesn't specify the label index");
return false;
}
// get max_feature_idx first
if (key_vals.count("max_feature_idx")) {
Common::Atoi(key_vals["max_feature_idx"].c_str(), &max_feature_idx_);
} else {
Log::Fatal("Model file doesn't specify max_feature_idx");
return false;
}
// get average_output
if (key_vals.count("average_output")) {
average_output_ = true;
}
// get feature names
if (key_vals.count("feature_names")) {
feature_names_ = Common::Split(key_vals["feature_names"].c_str(), ' ');
if (feature_names_.size() != static_cast<size_t>(max_feature_idx_ + 1)) {
Log::Fatal("Wrong size of feature_names");
return false;
}
} else {
Log::Fatal("Model file doesn't contain feature_names");
return false;
}
// get monotone_constraints
if (key_vals.count("monotone_constraints")) {
monotone_constraints_ = CommonC::StringToArray<int8_t>(key_vals["monotone_constraints"].c_str(), ' ');
if (monotone_constraints_.size() != static_cast<size_t>(max_feature_idx_ + 1)) {
Log::Fatal("Wrong size of monotone_constraints");
return false;
}
}
if (key_vals.count("feature_infos")) {
feature_infos_ = Common::Split(key_vals["feature_infos"].c_str(), ' ');
if (feature_infos_.size() != static_cast<size_t>(max_feature_idx_ + 1)) {
Log::Fatal("Wrong size of feature_infos");
return false;
}
} else {
Log::Fatal("Model file doesn't contain feature_infos");
return false;
}
if (key_vals.count("objective")) {
auto str = key_vals["objective"];
loaded_objective_.reset(ObjectiveFunction::CreateObjectiveFunction(ParseObjectiveAlias(str)));
objective_function_ = loaded_objective_.get();
}
if (!key_vals.count("tree_sizes")) {
while (p < end) {
auto line_len = Common::GetLine(p);
if (line_len > 0) {
std::string cur_line(p, line_len);
if (Common::StartsWith(cur_line, "Tree=")) {
p += line_len;
p = Common::SkipNewLine(p);
size_t used_len = 0;
models_.emplace_back(new Tree(p, &used_len));
p += used_len;
} else {
break;
}
}
p = Common::SkipNewLine(p);
}
} else {
std::vector<size_t> tree_sizes = CommonC::StringToArray<size_t>(key_vals["tree_sizes"].c_str(), ' ');
std::vector<size_t> tree_boundaries(tree_sizes.size() + 1, 0);
int num_trees = static_cast<int>(tree_sizes.size());
for (int i = 0; i < num_trees; ++i) {
tree_boundaries[i + 1] = tree_boundaries[i] + tree_sizes[i];
models_.emplace_back(nullptr);
}
OMP_INIT_EX();
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int i = 0; i < num_trees; ++i) {
OMP_LOOP_EX_BEGIN();
auto cur_p = p + tree_boundaries[i];
auto line_len = Common::GetLine(cur_p);
std::string cur_line(cur_p, line_len);
if (Common::StartsWith(cur_line, "Tree=")) {
cur_p += line_len;
cur_p = Common::SkipNewLine(cur_p);
size_t used_len = 0;
models_[i].reset(new Tree(cur_p, &used_len));
} else {
Log::Fatal("Model format error, expect a tree here. met %s", cur_line.c_str());
}
OMP_LOOP_EX_END();
}
OMP_THROW_EX();
}
num_iteration_for_pred_ = static_cast<int>(models_.size()) / num_tree_per_iteration_;
num_init_iteration_ = num_iteration_for_pred_;
iter_ = 0;
bool is_inparameter = false, is_inparser = false;
std::stringstream ss;
Common::C_stringstream(ss);
while (p < end) {
auto line_len = Common::GetLine(p);
if (line_len > 0) {
std::string cur_line(p, line_len);
if (cur_line == std::string("parameters:")) {
is_inparameter = true;
} else if (cur_line == std::string("end of parameters")) {
break;
} else if (is_inparameter) {
ss << cur_line << "\n";
if (Common::StartsWith(cur_line, "[linear_tree: ")) {
int is_linear = 0;
Common::Atoi(cur_line.substr(14, 1).c_str(), &is_linear);
linear_tree_ = static_cast<bool>(is_linear);
}
}
}
p += line_len;
p = Common::SkipNewLine(p);
}
if (!ss.str().empty()) {
loaded_parameter_ = ss.str();
}
ss.clear();
ss.str("");
while (p < end) {
auto line_len = Common::GetLine(p);
if (line_len > 0) {
std::string cur_line(p, line_len);
if (cur_line == std::string("parser:")) {
is_inparser = true;
} else if (cur_line == std::string("end of parser")) {
p += line_len;
p = Common::SkipNewLine(p);
break;
} else if (is_inparser) {
ss << cur_line << "\n";
}
}
p += line_len;
p = Common::SkipNewLine(p);
}
parser_config_str_ = ss.str();
ss.clear();
ss.str("");
return true;
}
std::vector<double> GBDT::FeatureImportance(int num_iteration, int importance_type) const {
int num_used_model = static_cast<int>(models_.size());
if (num_iteration > 0) {
num_iteration += 0;
num_used_model = std::min(num_iteration * num_tree_per_iteration_, num_used_model);
}
std::vector<double> feature_importances(max_feature_idx_ + 1, 0.0);
if (importance_type == 0) {
for (int iter = 0; iter < num_used_model; ++iter) {
for (int split_idx = 0; split_idx < models_[iter]->num_leaves() - 1; ++split_idx) {
if (models_[iter]->split_gain(split_idx) > 0) {
#ifdef DEBUG
CHECK_GE(models_[iter]->split_feature(split_idx), 0);
#endif
feature_importances[models_[iter]->split_feature(split_idx)] += 1.0;
}
}
}
} else if (importance_type == 1) {
for (int iter = 0; iter < num_used_model; ++iter) {
for (int split_idx = 0; split_idx < models_[iter]->num_leaves() - 1; ++split_idx) {
if (models_[iter]->split_gain(split_idx) > 0) {
#ifdef DEBUG
CHECK_GE(models_[iter]->split_feature(split_idx), 0);
#endif
feature_importances[models_[iter]->split_feature(split_idx)] += models_[iter]->split_gain(split_idx);
}
}
}
} else {
Log::Fatal("Unknown importance type: only support split=0 and gain=1");
}
return feature_importances;
}
} // namespace LightGBM
src/boosting/gbdt_prediction.cpp
+100
View File
@@ -0,0 +1,100 @@
/*!
* Copyright (c) 2017-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2017-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/objective_function.h>
#include <LightGBM/prediction_early_stop.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <unordered_map>
#include "gbdt.h"
namespace LightGBM {
void GBDT::PredictRaw(const double* features, double* output, const PredictionEarlyStopInstance* early_stop) const {
int early_stop_round_counter = 0;
// set zero
std::memset(output, 0, sizeof(double) * num_tree_per_iteration_);
const int end_iteration_for_pred = start_iteration_for_pred_ + num_iteration_for_pred_;
for (int i = start_iteration_for_pred_; i < end_iteration_for_pred; ++i) {
// predict all the trees for one iteration
for (int k = 0; k < num_tree_per_iteration_; ++k) {
output[k] += models_[i * num_tree_per_iteration_ + k]->Predict(features);
}
// check early stopping
++early_stop_round_counter;
if (early_stop->round_period == early_stop_round_counter) {
if (early_stop->callback_function(output, num_tree_per_iteration_)) {
return;
}
early_stop_round_counter = 0;
}
}
}
void GBDT::PredictRawByMap(const std::unordered_map<int, double>& features, double* output, const PredictionEarlyStopInstance* early_stop) const {
int early_stop_round_counter = 0;
// set zero
std::memset(output, 0, sizeof(double) * num_tree_per_iteration_);
const int end_iteration_for_pred = start_iteration_for_pred_ + num_iteration_for_pred_;
for (int i = start_iteration_for_pred_; i < end_iteration_for_pred; ++i) {
// predict all the trees for one iteration
for (int k = 0; k < num_tree_per_iteration_; ++k) {
output[k] += models_[i * num_tree_per_iteration_ + k]->PredictByMap(features);
}
// check early stopping
++early_stop_round_counter;
if (early_stop->round_period == early_stop_round_counter) {
if (early_stop->callback_function(output, num_tree_per_iteration_)) {
return;
}
early_stop_round_counter = 0;
}
}
}
void GBDT::Predict(const double* features, double* output, const PredictionEarlyStopInstance* early_stop) const {
PredictRaw(features, output, early_stop);
if (average_output_) {
for (int k = 0; k < num_tree_per_iteration_; ++k) {
output[k] /= num_iteration_for_pred_;
}
}
if (objective_function_ != nullptr) {
objective_function_->ConvertOutput(output, output);
}
}
void GBDT::PredictByMap(const std::unordered_map<int, double>& features, double* output, const PredictionEarlyStopInstance* early_stop) const {
PredictRawByMap(features, output, early_stop);
if (average_output_) {
for (int k = 0; k < num_tree_per_iteration_; ++k) {
output[k] /= num_iteration_for_pred_;
}
}
if (objective_function_ != nullptr) {
objective_function_->ConvertOutput(output, output);
}
}
void GBDT::PredictLeafIndex(const double* features, double* output) const {
int start_tree = start_iteration_for_pred_ * num_tree_per_iteration_;
int num_trees = num_iteration_for_pred_ * num_tree_per_iteration_;
const auto* models_ptr = models_.data() + start_tree;
for (int i = 0; i < num_trees; ++i) {
output[i] = models_ptr[i]->PredictLeafIndex(features);
}
}
void GBDT::PredictLeafIndexByMap(const std::unordered_map<int, double>& features, double* output) const {
int start_tree = start_iteration_for_pred_ * num_tree_per_iteration_;
int num_trees = num_iteration_for_pred_ * num_tree_per_iteration_;
const auto* models_ptr = models_.data() + start_tree;
for (int i = 0; i < num_trees; ++i) {
output[i] = models_ptr[i]->PredictLeafIndexByMap(features);
}
}
} // namespace LightGBM
src/boosting/goss.hpp
+173
View File
@@ -0,0 +1,173 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_BOOSTING_GOSS_HPP_
#define LIGHTGBM_SRC_BOOSTING_GOSS_HPP_
#include <LightGBM/utils/array_args.h>
#include <LightGBM/sample_strategy.h>
#include <algorithm>
#include <string>
#include <vector>
namespace LightGBM {
class GOSSStrategy : public SampleStrategy {
public:
GOSSStrategy(const Config* config, const Dataset* train_data, int num_tree_per_iteration) {
config_ = config;
train_data_ = train_data;
num_tree_per_iteration_ = num_tree_per_iteration;
num_data_ = train_data->num_data();
}
~GOSSStrategy() {
}
void Bagging(int iter, TreeLearner* tree_learner, score_t* gradients, score_t* hessians) override {
bag_data_cnt_ = num_data_;
// not subsample for first iterations
if (iter < static_cast<int>(1.0f / config_->learning_rate)) {
return;
}
auto left_cnt = bagging_runner_.Run<true>(
num_data_,
[=](int, data_size_t cur_start, data_size_t cur_cnt, data_size_t* left,
data_size_t*) {
data_size_t cur_left_count = 0;
cur_left_count = Helper(cur_start, cur_cnt, left, gradients, hessians);
return cur_left_count;
},
bag_data_indices_.data());
bag_data_cnt_ = left_cnt;
// set bagging data to tree learner
if (!is_use_subset_) {
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda")) {
CopyFromHostToCUDADevice<data_size_t>(cuda_bag_data_indices_.RawData(), bag_data_indices_.data(), static_cast<size_t>(num_data_), __FILE__, __LINE__);
tree_learner->SetBaggingData(nullptr, cuda_bag_data_indices_.RawData(), bag_data_cnt_);
} else {
#endif // USE_CUDA
tree_learner->SetBaggingData(nullptr, bag_data_indices_.data(), bag_data_cnt_);
#ifdef USE_CUDA
}
#endif // USE_CUDA
} else {
// get subset
tmp_subset_->ReSize(bag_data_cnt_);
tmp_subset_->CopySubrow(train_data_, bag_data_indices_.data(),
bag_data_cnt_, false);
#ifdef USE_CUDA
if (config_->device_type == std::string("cuda")) {
CopyFromHostToCUDADevice<data_size_t>(cuda_bag_data_indices_.RawData(), bag_data_indices_.data(), static_cast<size_t>(num_data_), __FILE__, __LINE__);
tree_learner->SetBaggingData(tmp_subset_.get(), cuda_bag_data_indices_.RawData(),
bag_data_cnt_);
} else {
#endif // USE_CUDA
tree_learner->SetBaggingData(tmp_subset_.get(), bag_data_indices_.data(),
bag_data_cnt_);
#ifdef USE_CUDA
}
#endif // USE_CUDA
}
}
void ResetSampleConfig(const Config* config, bool /*is_change_dataset*/) override {
// Cannot use bagging in GOSS
config_ = config;
need_resize_gradients_ = false;
if (objective_function_ == nullptr) {
// resize gradient vectors to copy the customized gradients for goss
need_resize_gradients_ = true;
}
CHECK_LE(config_->top_rate + config_->other_rate, 1.0f);
CHECK(config_->top_rate > 0.0f && config_->other_rate > 0.0f);
if (config_->bagging_freq > 0 && config_->bagging_fraction != 1.0f) {
Log::Fatal("Cannot use bagging in GOSS");
}
Log::Info("Using GOSS");
balanced_bagging_ = false;
bag_data_indices_.resize(num_data_);
bagging_runner_.ReSize(num_data_);
bagging_rands_.clear();
for (int i = 0;
i < (num_data_ + bagging_rand_block_ - 1) / bagging_rand_block_; ++i) {
bagging_rands_.emplace_back(config_->bagging_seed + i);
}
is_use_subset_ = false;
if (config_->top_rate + config_->other_rate <= 0.5) {
auto bag_data_cnt = static_cast<data_size_t>((config_->top_rate + config_->other_rate) * num_data_);
bag_data_cnt = std::max(1, bag_data_cnt);
tmp_subset_.reset(new Dataset(bag_data_cnt));
tmp_subset_->CopyFeatureMapperFrom(train_data_);
is_use_subset_ = true;
}
// flag to not bagging first
bag_data_cnt_ = num_data_;
}
bool IsHessianChange() const override {
return true;
}
private:
data_size_t Helper(data_size_t start, data_size_t cnt, data_size_t* buffer, score_t* gradients, score_t* hessians) {
if (cnt <= 0) {
return 0;
}
std::vector<score_t> tmp_gradients(cnt, 0.0f);
for (data_size_t i = 0; i < cnt; ++i) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
size_t idx = static_cast<size_t>(cur_tree_id) * num_data_ + start + i;
tmp_gradients[i] += std::fabs(gradients[idx] * hessians[idx]);
}
}
data_size_t top_k = static_cast<data_size_t>(cnt * config_->top_rate);
data_size_t other_k = static_cast<data_size_t>(cnt * config_->other_rate);
top_k = std::max(1, top_k);
ArrayArgs<score_t>::ArgMaxAtK(&tmp_gradients, 0, static_cast<int>(tmp_gradients.size()), top_k - 1);
score_t threshold = tmp_gradients[top_k - 1];
score_t multiply = static_cast<score_t>(cnt - top_k) / other_k;
data_size_t cur_left_cnt = 0;
data_size_t cur_right_pos = cnt;
data_size_t big_weight_cnt = 0;
for (data_size_t i = 0; i < cnt; ++i) {
auto cur_idx = start + i;
score_t grad = 0.0f;
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
size_t idx = static_cast<size_t>(cur_tree_id) * num_data_ + cur_idx;
grad += std::fabs(gradients[idx] * hessians[idx]);
}
if (grad >= threshold) {
buffer[cur_left_cnt++] = cur_idx;
++big_weight_cnt;
} else {
data_size_t sampled = cur_left_cnt - big_weight_cnt;
data_size_t rest_need = other_k - sampled;
data_size_t rest_all = (cnt - i) - (top_k - big_weight_cnt);
double prob = (rest_need) / static_cast<double>(rest_all);
if (bagging_rands_[cur_idx / bagging_rand_block_].NextFloat() < prob) {
buffer[cur_left_cnt++] = cur_idx;
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
size_t idx = static_cast<size_t>(cur_tree_id) * num_data_ + cur_idx;
gradients[idx] *= multiply;
hessians[idx] *= multiply;
}
} else {
buffer[--cur_right_pos] = cur_idx;
}
}
}
return cur_left_cnt;
}
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_BOOSTING_GOSS_HPP_
src/boosting/prediction_early_stop.cpp
+94
View File
@@ -0,0 +1,94 @@
/*!
* Copyright (c) 2017-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2017-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/prediction_early_stop.h>
#include <LightGBM/utils/log.h>
#include <algorithm>
#include <cmath>
#include <functional>
#include <limits>
#include <string>
#include <vector>
namespace LightGBM {
PredictionEarlyStopInstance CreateNone(const PredictionEarlyStopConfig&) {
return PredictionEarlyStopInstance{
[](const double*, int) {
return false;
},
std::numeric_limits<int>::max() // make sure the lambda is almost never called
};
}
PredictionEarlyStopInstance CreateMulticlass(const PredictionEarlyStopConfig& config) {
// margin_threshold will be captured by value
const double margin_threshold = config.margin_threshold;
return PredictionEarlyStopInstance{
[margin_threshold](const double* pred, int sz) {
if (sz < 2) {
Log::Fatal("Multiclass early stopping needs predictions to be of length two or larger");
}
// copy and sort
std::vector<double> votes(static_cast<size_t>(sz));
for (int i = 0; i < sz; ++i) {
votes[i] = pred[i];
}
std::partial_sort(votes.begin(), votes.begin() + 2, votes.end(), std::greater<double>());
const auto margin = votes[0] - votes[1];
if (margin > margin_threshold) {
return true;
}
return false;
},
config.round_period
};
}
PredictionEarlyStopInstance CreateBinary(const PredictionEarlyStopConfig& config) {
// margin_threshold will be captured by value
const double margin_threshold = config.margin_threshold;
return PredictionEarlyStopInstance{
[margin_threshold](const double* pred, int sz) {
if (sz != 1) {
Log::Fatal("Binary early stopping needs predictions to be of length one");
}
const auto margin = 2.0 * fabs(pred[0]);
if (margin > margin_threshold) {
return true;
}
return false;
},
config.round_period
};
}
PredictionEarlyStopInstance CreatePredictionEarlyStopInstance(const std::string& type,
const PredictionEarlyStopConfig& config) {
if (type == "none") {
return CreateNone(config);
} else if (type == "multiclass") {
return CreateMulticlass(config);
} else if (type == "binary") {
return CreateBinary(config);
} else {
Log::Fatal("Unknown early stopping type: %s", type.c_str());
}
// Fix for compiler warnings about reaching end of control
return CreateNone(config);
}
} // namespace LightGBM
src/boosting/rf.hpp
+237
View File
@@ -0,0 +1,237 @@
/*!
* Copyright (c) 2017-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2017-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_BOOSTING_RF_HPP_
#define LIGHTGBM_SRC_BOOSTING_RF_HPP_
#include <LightGBM/boosting.h>
#include <LightGBM/metric.h>
#include <string>
#include <cstdio>
#include <fstream>
#include <memory>
#include <utility>
#include <vector>
#include "gbdt.h"
#include "score_updater.hpp"
namespace LightGBM {
/*!
* \brief Random Forest implementation
*/
class RF : public GBDT {
public:
RF() : GBDT() {
average_output_ = true;
}
~RF() {}
void Init(const Config* config, const Dataset* train_data, const ObjectiveFunction* objective_function,
const std::vector<const Metric*>& training_metrics) override {
if (config->data_sample_strategy == std::string("bagging")) {
CHECK((config->bagging_freq > 0 && config->bagging_fraction < 1.0f && config->bagging_fraction > 0.0f) ||
(config->feature_fraction < 1.0f && config->feature_fraction > 0.0f));
} else {
CHECK_EQ(config->data_sample_strategy, std::string("goss"));
}
GBDT::Init(config, train_data, objective_function, training_metrics);
if (num_init_iteration_ > 0) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
MultiplyScore(cur_tree_id, 1.0f / num_init_iteration_);
}
} else {
CHECK_EQ(train_data->metadata().init_score(), nullptr);
}
CHECK_EQ(num_tree_per_iteration_, num_class_);
// not shrinkage rate for the RF
shrinkage_rate_ = 1.0f;
// only boosting one time
Boosting();
if (data_sample_strategy_->is_use_subset() && data_sample_strategy_->bag_data_cnt() < num_data_) {
tmp_grad_.resize(num_data_);
tmp_hess_.resize(num_data_);
}
}
void ResetConfig(const Config* config) override {
if (config->data_sample_strategy == std::string("bagging")) {
CHECK((config->bagging_freq > 0 && config->bagging_fraction < 1.0f && config->bagging_fraction > 0.0f) ||
(config->feature_fraction < 1.0f && config->feature_fraction > 0.0f));
} else {
CHECK_EQ(config->data_sample_strategy, std::string("goss"));
}
GBDT::ResetConfig(config);
// not shrinkage rate for the RF
shrinkage_rate_ = 1.0f;
}
void ResetTrainingData(const Dataset* train_data, const ObjectiveFunction* objective_function,
const std::vector<const Metric*>& training_metrics) override {
GBDT::ResetTrainingData(train_data, objective_function, training_metrics);
if (iter_ + num_init_iteration_ > 0) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
train_score_updater_->MultiplyScore(1.0f / (iter_ + num_init_iteration_), cur_tree_id);
}
}
CHECK_EQ(num_tree_per_iteration_, num_class_);
// only boosting one time
Boosting();
if (data_sample_strategy_->is_use_subset() && data_sample_strategy_->bag_data_cnt() < num_data_) {
tmp_grad_.resize(num_data_);
tmp_hess_.resize(num_data_);
}
}
void Boosting() override {
if (objective_function_ == nullptr) {
Log::Fatal("RF mode do not support custom objective function, please use built-in objectives.");
}
init_scores_.resize(num_tree_per_iteration_, 0.0);
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
init_scores_[cur_tree_id] = BoostFromAverage(cur_tree_id, false);
}
size_t total_size = static_cast<size_t>(num_data_) * num_tree_per_iteration_;
std::vector<double> tmp_scores(total_size, 0.0f);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int j = 0; j < num_tree_per_iteration_; ++j) {
size_t offset = static_cast<size_t>(j)* num_data_;
for (data_size_t i = 0; i < num_data_; ++i) {
tmp_scores[offset + i] = init_scores_[j];
}
}
objective_function_->
GetGradients(tmp_scores.data(), gradients_.data(), hessians_.data());
}
bool TrainOneIter(const score_t* gradients, const score_t* hessians) override {
// bagging logic
data_sample_strategy_ ->Bagging(iter_, tree_learner_.get(), gradients_.data(), hessians_.data());
const bool is_use_subset = data_sample_strategy_->is_use_subset();
const data_size_t bag_data_cnt = data_sample_strategy_->bag_data_cnt();
const std::vector<data_size_t, Common::AlignmentAllocator<data_size_t, kAlignedSize>>& bag_data_indices = data_sample_strategy_->bag_data_indices();
// GOSSStrategy->Bagging may modify value of bag_data_cnt_
if (is_use_subset && bag_data_cnt < num_data_) {
tmp_grad_.resize(num_data_);
tmp_hess_.resize(num_data_);
}
CHECK_EQ(gradients, nullptr);
CHECK_EQ(hessians, nullptr);
gradients = gradients_.data();
hessians = hessians_.data();
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
std::unique_ptr<Tree> new_tree(new Tree(2, false, false));
size_t offset = static_cast<size_t>(cur_tree_id)* num_data_;
if (class_need_train_[cur_tree_id]) {
auto grad = gradients + offset;
auto hess = hessians + offset;
if (is_use_subset && bag_data_cnt < num_data_ && !boosting_on_gpu_) {
for (int i = 0; i < bag_data_cnt; ++i) {
tmp_grad_[i] = grad[bag_data_indices[i]];
tmp_hess_[i] = hess[bag_data_indices[i]];
}
grad = tmp_grad_.data();
hess = tmp_hess_.data();
}
new_tree.reset(tree_learner_->Train(grad, hess, false));
}
if (new_tree->num_leaves() > 1) {
double pred = init_scores_[cur_tree_id];
auto residual_getter = [pred](const label_t* label, int i) {return static_cast<double>(label[i]) - pred; };
tree_learner_->RenewTreeOutput(new_tree.get(), objective_function_, residual_getter,
num_data_, bag_data_indices.data(), bag_data_cnt, train_score_updater_->score());
if (std::fabs(init_scores_[cur_tree_id]) > kEpsilon) {
new_tree->AddBias(init_scores_[cur_tree_id]);
}
// update score
MultiplyScore(cur_tree_id, (iter_ + num_init_iteration_));
UpdateScore(new_tree.get(), cur_tree_id);
MultiplyScore(cur_tree_id, 1.0 / (iter_ + num_init_iteration_ + 1));
} else {
// only add default score one-time
if (models_.size() < static_cast<size_t>(num_tree_per_iteration_)) {
double output = 0.0;
if (!class_need_train_[cur_tree_id]) {
if (objective_function_ != nullptr) {
output = objective_function_->BoostFromScore(cur_tree_id);
} else {
output = init_scores_[cur_tree_id];
}
}
new_tree->AsConstantTree(output, num_data_);
MultiplyScore(cur_tree_id, (iter_ + num_init_iteration_));
UpdateScore(new_tree.get(), cur_tree_id);
MultiplyScore(cur_tree_id, 1.0 / (iter_ + num_init_iteration_ + 1));
}
}
// add model
models_.push_back(std::move(new_tree));
}
++iter_;
return false;
}
void RollbackOneIter() override {
if (iter_ <= 0) {
return;
}
int cur_iter = iter_ + num_init_iteration_ - 1;
// reset score
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
auto curr_tree = cur_iter * num_tree_per_iteration_ + cur_tree_id;
models_[curr_tree]->Shrinkage(-1.0);
MultiplyScore(cur_tree_id, (iter_ + num_init_iteration_));
train_score_updater_->AddScore(models_[curr_tree].get(), cur_tree_id);
for (auto& score_updater : valid_score_updater_) {
score_updater->AddScore(models_[curr_tree].get(), cur_tree_id);
}
MultiplyScore(cur_tree_id, 1.0f / (iter_ + num_init_iteration_ - 1));
}
// remove model
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
models_.pop_back();
}
--iter_;
}
void MultiplyScore(const int cur_tree_id, double val) {
train_score_updater_->MultiplyScore(val, cur_tree_id);
for (auto& score_updater : valid_score_updater_) {
score_updater->MultiplyScore(val, cur_tree_id);
}
}
void AddValidDataset(const Dataset* valid_data,
const std::vector<const Metric*>& valid_metrics) override {
GBDT::AddValidDataset(valid_data, valid_metrics);
if (iter_ + num_init_iteration_ > 0) {
for (int cur_tree_id = 0; cur_tree_id < num_tree_per_iteration_; ++cur_tree_id) {
valid_score_updater_.back()->MultiplyScore(1.0f / (iter_ + num_init_iteration_), cur_tree_id);
}
}
}
bool NeedAccuratePrediction() const override {
// No early stopping for prediction
return true;
};
private:
std::vector<score_t> tmp_grad_;
std::vector<score_t> tmp_hess_;
std::vector<double> init_scores_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_BOOSTING_RF_HPP_
src/boosting/sample_strategy.cpp
+28
View File
@@ -0,0 +1,28 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/sample_strategy.h>
#include <string>
#include "goss.hpp"
#include "bagging.hpp"
namespace LightGBM {
SampleStrategy* SampleStrategy::CreateSampleStrategy(
const Config* config,
const Dataset* train_data,
const ObjectiveFunction* objective_function,
int num_tree_per_iteration) {
if (config->data_sample_strategy == std::string("goss")) {
return new GOSSStrategy(config, train_data, num_tree_per_iteration);
} else {
return new BaggingSampleStrategy(config, train_data, objective_function, num_tree_per_iteration);
}
}
} // namespace LightGBM
src/boosting/score_updater.hpp
+129
View File
@@ -0,0 +1,129 @@
/*!
* Copyright (c) 2017-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2017-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_BOOSTING_SCORE_UPDATER_HPP_
#define LIGHTGBM_SRC_BOOSTING_SCORE_UPDATER_HPP_
#include <LightGBM/dataset.h>
#include <LightGBM/meta.h>
#include <LightGBM/tree.h>
#include <LightGBM/tree_learner.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <cstring>
#include <vector>
namespace LightGBM {
/*!
* \brief Used to store and update score for data
*/
class ScoreUpdater {
public:
/*!
* \brief Constructor, will pass a const pointer of dataset
* \param data This class will bind with this data set
*/
ScoreUpdater(const Dataset* data, int num_tree_per_iteration) : data_(data) {
num_data_ = data->num_data();
int64_t total_size = static_cast<int64_t>(num_data_) * num_tree_per_iteration;
score_.resize(total_size);
// default start score is zero
std::memset(score_.data(), 0, total_size * sizeof(double));
has_init_score_ = false;
const double* init_score = data->metadata().init_score();
// if exists initial score, will start from it
if (init_score != nullptr) {
if ((data->metadata().num_init_score() % num_data_) != 0
|| (data->metadata().num_init_score() / num_data_) != num_tree_per_iteration) {
Log::Fatal("Number of class for initial score error");
}
has_init_score_ = true;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (total_size >= 1024)
for (int64_t i = 0; i < total_size; ++i) {
score_[i] = init_score[i];
}
}
}
/*! \brief Destructor */
virtual ~ScoreUpdater() {
}
inline bool has_init_score() const { return has_init_score_; }
virtual inline void AddScore(double val, int cur_tree_id) {
Common::FunctionTimer fun_timer("ScoreUpdater::AddScore", global_timer);
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (num_data_ >= 1024)
for (int i = 0; i < num_data_; ++i) {
score_[offset + i] += val;
}
}
virtual inline void MultiplyScore(double val, int cur_tree_id) {
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (num_data_ >= 1024)
for (int i = 0; i < num_data_; ++i) {
score_[offset + i] *= val;
}
}
/*!
* \brief Using tree model to get prediction number, then adding to scores for all data
* Note: this function generally will be used on validation data too.
* \param tree Trained tree model
* \param cur_tree_id Current tree for multiclass training
*/
virtual inline void AddScore(const Tree* tree, int cur_tree_id) {
Common::FunctionTimer fun_timer("ScoreUpdater::AddScore", global_timer);
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
tree->AddPredictionToScore(data_, num_data_, score_.data() + offset);
}
/*!
* \brief Adding prediction score, only used for training data.
* The training data is partitioned into tree leaves after training
* Based on which We can get prediction quickly.
* \param tree_learner
* \param cur_tree_id Current tree for multiclass training
*/
virtual inline void AddScore(const TreeLearner* tree_learner, const Tree* tree, int cur_tree_id) {
Common::FunctionTimer fun_timer("ScoreUpdater::AddScore", global_timer);
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
tree_learner->AddPredictionToScore(tree, score_.data() + offset);
}
/*!
* \brief Using tree model to get prediction number, then adding to scores for parts of data
* Used for prediction of training out-of-bag data
* \param tree Trained tree model
* \param data_indices Indices of data that will be processed
* \param data_cnt Number of data that will be processed
* \param cur_tree_id Current tree for multiclass training
*/
virtual inline void AddScore(const Tree* tree, const data_size_t* data_indices,
data_size_t data_cnt, int cur_tree_id) {
Common::FunctionTimer fun_timer("ScoreUpdater::AddScore", global_timer);
const size_t offset = static_cast<size_t>(num_data_) * cur_tree_id;
tree->AddPredictionToScore(data_, data_indices, data_cnt, score_.data() + offset);
}
/*! \brief Pointer of score */
virtual inline const double* score() const { return score_.data(); }
inline data_size_t num_data() const { return num_data_; }
/*! \brief Disable copy */
ScoreUpdater& operator=(const ScoreUpdater&) = delete;
/*! \brief Disable copy */
ScoreUpdater(const ScoreUpdater&) = delete;
protected:
/*! \brief Number of total data */
data_size_t num_data_;
/*! \brief Pointer of data set */
const Dataset* data_;
/*! \brief Scores for data set */
std::vector<double, Common::AlignmentAllocator<double, kAlignedSize>> score_;
bool has_init_score_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_BOOSTING_SCORE_UPDATER_HPP_
src/c_api.cpp
+3080
View File
File diff suppressed because it is too large Load Diff
src/cuda/cuda_algorithms.cu
+451
View File
@@ -0,0 +1,451 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
* Modifications Copyright(C) 2023 Advanced Micro Devices, Inc. All rights reserved.
*/
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_algorithms.hpp>
#include <LightGBM/cuda/cuda_rocm_interop.h>
#include <algorithm>
namespace LightGBM {
template <typename T>
__global__ void ShufflePrefixSumGlobalKernel(T* values, size_t len, T* block_prefix_sum_buffer) {
__shared__ T shared_mem_buffer[WARPSIZE];
const size_t index = static_cast<size_t>(threadIdx.x + blockIdx.x * blockDim.x);
T value = 0;
if (index < len) {
value = values[index];
}
const T prefix_sum_value = ShufflePrefixSum<T>(value, shared_mem_buffer);
values[index] = prefix_sum_value;
if (threadIdx.x == blockDim.x - 1) {
block_prefix_sum_buffer[blockIdx.x] = prefix_sum_value;
}
}
template <typename T>
__global__ void ShufflePrefixSumGlobalReduceBlockKernel(T* block_prefix_sum_buffer, int num_blocks) {
__shared__ T shared_mem_buffer[WARPSIZE];
const int num_blocks_per_thread = (num_blocks + GLOBAL_PREFIX_SUM_BLOCK_SIZE - 2) / (GLOBAL_PREFIX_SUM_BLOCK_SIZE - 1);
int thread_block_start = threadIdx.x == 0 ? 0 : (threadIdx.x - 1) * num_blocks_per_thread;
int thread_block_end = threadIdx.x == 0 ? 0 : min(thread_block_start + num_blocks_per_thread, num_blocks);
T base = 0;
for (int block_index = thread_block_start; block_index < thread_block_end; ++block_index) {
base += block_prefix_sum_buffer[block_index];
}
base = ShufflePrefixSum<T>(base, shared_mem_buffer);
thread_block_start = threadIdx.x == blockDim.x - 1 ? 0 : threadIdx.x * num_blocks_per_thread;
thread_block_end = threadIdx.x == blockDim.x - 1 ? 0 : min(thread_block_start + num_blocks_per_thread, num_blocks);
for (int block_index = thread_block_start + 1; block_index < thread_block_end; ++block_index) {
block_prefix_sum_buffer[block_index] += block_prefix_sum_buffer[block_index - 1];
}
for (int block_index = thread_block_start; block_index < thread_block_end; ++block_index) {
block_prefix_sum_buffer[block_index] += base;
}
}
template <typename T>
__global__ void ShufflePrefixSumGlobalAddBase(size_t len, const T* block_prefix_sum_buffer, T* values) {
const T base = blockIdx.x == 0 ? 0 : block_prefix_sum_buffer[blockIdx.x - 1];
const size_t index = static_cast<size_t>(threadIdx.x + blockIdx.x * blockDim.x);
if (index < len) {
values[index] += base;
}
}
template <typename T>
void ShufflePrefixSumGlobal(T* values, size_t len, T* block_prefix_sum_buffer) {
const int num_blocks = (static_cast<int>(len) + GLOBAL_PREFIX_SUM_BLOCK_SIZE - 1) / GLOBAL_PREFIX_SUM_BLOCK_SIZE;
ShufflePrefixSumGlobalKernel<<<num_blocks, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(values, len, block_prefix_sum_buffer);
ShufflePrefixSumGlobalReduceBlockKernel<<<1, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(block_prefix_sum_buffer, num_blocks);
ShufflePrefixSumGlobalAddBase<<<num_blocks, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(len, block_prefix_sum_buffer, values);
}
template void ShufflePrefixSumGlobal<uint16_t>(uint16_t* values, size_t len, uint16_t* block_prefix_sum_buffer);
template void ShufflePrefixSumGlobal<uint32_t>(uint32_t* values, size_t len, uint32_t* block_prefix_sum_buffer);
template void ShufflePrefixSumGlobal<uint64_t>(uint64_t* values, size_t len, uint64_t* block_prefix_sum_buffer);
__global__ void BitonicArgSortItemsGlobalKernel(const double* scores,
const int num_queries,
const data_size_t* cuda_query_boundaries,
data_size_t* out_indices) {
const int query_index_start = static_cast<int>(blockIdx.x) * BITONIC_SORT_QUERY_ITEM_BLOCK_SIZE;
const int query_index_end = min(query_index_start + BITONIC_SORT_QUERY_ITEM_BLOCK_SIZE, num_queries);
for (int query_index = query_index_start; query_index < query_index_end; ++query_index) {
const data_size_t query_item_start = cuda_query_boundaries[query_index];
const data_size_t query_item_end = cuda_query_boundaries[query_index + 1];
const data_size_t num_items_in_query = query_item_end - query_item_start;
BitonicArgSortDevice<double, data_size_t, false, BITONIC_SORT_NUM_ELEMENTS, 11>(scores + query_item_start,
out_indices + query_item_start,
num_items_in_query);
__syncthreads();
}
}
void BitonicArgSortItemsGlobal(
const double* scores,
const int num_queries,
const data_size_t* cuda_query_boundaries,
data_size_t* out_indices) {
const int num_blocks = (num_queries + BITONIC_SORT_QUERY_ITEM_BLOCK_SIZE - 1) / BITONIC_SORT_QUERY_ITEM_BLOCK_SIZE;
BitonicArgSortItemsGlobalKernel<<<num_blocks, BITONIC_SORT_NUM_ELEMENTS>>>(
scores, num_queries, cuda_query_boundaries, out_indices);
SynchronizeCUDADevice(__FILE__, __LINE__);
}
template <typename T>
__global__ void BlockReduceSum(T* block_buffer, const data_size_t num_blocks) {
__shared__ T shared_buffer[WARPSIZE];
T thread_sum = 0;
for (data_size_t block_index = static_cast<data_size_t>(threadIdx.x); block_index < num_blocks; block_index += static_cast<data_size_t>(blockDim.x)) {
thread_sum += block_buffer[block_index];
}
thread_sum = ShuffleReduceSum<T>(thread_sum, shared_buffer, blockDim.x);
if (threadIdx.x == 0) {
block_buffer[0] = thread_sum;
}
}
template <typename VAL_T, typename REDUCE_T>
__global__ void ShuffleReduceSumGlobalKernel(const VAL_T* values, const data_size_t num_value, REDUCE_T* block_buffer) {
__shared__ REDUCE_T shared_buffer[WARPSIZE];
const data_size_t data_index = static_cast<data_size_t>(blockIdx.x * blockDim.x + threadIdx.x);
const REDUCE_T value = (data_index < num_value ? static_cast<REDUCE_T>(values[data_index]) : 0.0f);
const REDUCE_T reduce_value = ShuffleReduceSum<REDUCE_T>(value, shared_buffer, blockDim.x);
if (threadIdx.x == 0) {
block_buffer[blockIdx.x] = reduce_value;
}
}
template <typename VAL_T, typename REDUCE_T>
void ShuffleReduceSumGlobal(const VAL_T* values, size_t n, REDUCE_T* block_buffer) {
const data_size_t num_value = static_cast<data_size_t>(n);
const data_size_t num_blocks = (num_value + GLOBAL_PREFIX_SUM_BLOCK_SIZE - 1) / GLOBAL_PREFIX_SUM_BLOCK_SIZE;
ShuffleReduceSumGlobalKernel<VAL_T, REDUCE_T><<<num_blocks, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(values, num_value, block_buffer);
BlockReduceSum<REDUCE_T><<<1, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(block_buffer, num_blocks);
}
template void ShuffleReduceSumGlobal<label_t, double>(const label_t* values, size_t n, double* block_buffer);
template void ShuffleReduceSumGlobal<double, double>(const double* values, size_t n, double* block_buffer);
template <typename VAL_T, typename REDUCE_T>
__global__ void ShuffleReduceMinGlobalKernel(const VAL_T* values, const data_size_t num_value, REDUCE_T* block_buffer) {
__shared__ REDUCE_T shared_buffer[WARPSIZE];
const data_size_t data_index = static_cast<data_size_t>(blockIdx.x * blockDim.x + threadIdx.x);
const REDUCE_T value = (data_index < num_value ? static_cast<REDUCE_T>(values[data_index]) : 0.0f);
const REDUCE_T reduce_value = ShuffleReduceMin<REDUCE_T>(value, shared_buffer, blockDim.x);
if (threadIdx.x == 0) {
block_buffer[blockIdx.x] = reduce_value;
}
}
template <typename T>
__global__ void ShuffleBlockReduceMin(T* block_buffer, const data_size_t num_blocks) {
__shared__ T shared_buffer[WARPSIZE];
T thread_min = 0;
for (data_size_t block_index = static_cast<data_size_t>(threadIdx.x); block_index < num_blocks; block_index += static_cast<data_size_t>(blockDim.x)) {
const T value = block_buffer[block_index];
if (value < thread_min) {
thread_min = value;
}
}
thread_min = ShuffleReduceMin<T>(thread_min, shared_buffer, blockDim.x);
if (threadIdx.x == 0) {
block_buffer[0] = thread_min;
}
}
template <typename VAL_T, typename REDUCE_T>
void ShuffleReduceMinGlobal(const VAL_T* values, size_t n, REDUCE_T* block_buffer) {
const data_size_t num_value = static_cast<data_size_t>(n);
const data_size_t num_blocks = (num_value + GLOBAL_PREFIX_SUM_BLOCK_SIZE - 1) / GLOBAL_PREFIX_SUM_BLOCK_SIZE;
ShuffleReduceMinGlobalKernel<VAL_T, REDUCE_T><<<num_blocks, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(values, num_value, block_buffer);
ShuffleBlockReduceMin<REDUCE_T><<<1, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(block_buffer, num_blocks);
}
template void ShuffleReduceMinGlobal<label_t, double>(const label_t* values, size_t n, double* block_buffer);
template <typename VAL_T, typename REDUCE_T>
__global__ void ShuffleReduceDotProdGlobalKernel(const VAL_T* values1, const VAL_T* values2, const data_size_t num_value, REDUCE_T* block_buffer) {
__shared__ REDUCE_T shared_buffer[WARPSIZE];
const data_size_t data_index = static_cast<data_size_t>(blockIdx.x * blockDim.x + threadIdx.x);
const REDUCE_T value1 = (data_index < num_value ? static_cast<REDUCE_T>(values1[data_index]) : 0.0f);
const REDUCE_T value2 = (data_index < num_value ? static_cast<REDUCE_T>(values2[data_index]) : 0.0f);
const REDUCE_T reduce_value = ShuffleReduceSum<REDUCE_T>(value1 * value2, shared_buffer, blockDim.x);
if (threadIdx.x == 0) {
block_buffer[blockIdx.x] = reduce_value;
}
}
template <typename VAL_T, typename REDUCE_T>
void ShuffleReduceDotProdGlobal(const VAL_T* values1, const VAL_T* values2, size_t n, REDUCE_T* block_buffer) {
const data_size_t num_value = static_cast<data_size_t>(n);
const data_size_t num_blocks = (num_value + GLOBAL_PREFIX_SUM_BLOCK_SIZE - 1) / GLOBAL_PREFIX_SUM_BLOCK_SIZE;
ShuffleReduceDotProdGlobalKernel<VAL_T, REDUCE_T><<<num_blocks, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(values1, values2, num_value, block_buffer);
BlockReduceSum<REDUCE_T><<<1, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(block_buffer, num_blocks);
}
template void ShuffleReduceDotProdGlobal<label_t, double>(const label_t* values1, const label_t* values2, size_t n, double* block_buffer);
template <typename INDEX_T, typename VAL_T, typename REDUCE_T>
__global__ void GlobalInclusiveArgPrefixSumKernel(
const INDEX_T* sorted_indices, const VAL_T* in_values, REDUCE_T* out_values, REDUCE_T* block_buffer, data_size_t num_data) {
__shared__ REDUCE_T shared_buffer[WARPSIZE];
const data_size_t data_index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
REDUCE_T value = static_cast<REDUCE_T>(data_index < num_data ? in_values[sorted_indices[data_index]] : 0);
__syncthreads();
value = ShufflePrefixSum<REDUCE_T>(value, shared_buffer);
if (data_index < num_data) {
out_values[data_index] = value;
}
if (threadIdx.x == blockDim.x - 1) {
block_buffer[blockIdx.x + 1] = value;
}
}
template <typename T>
__global__ void GlobalInclusivePrefixSumReduceBlockKernel(T* block_buffer, data_size_t num_blocks) {
__shared__ T shared_buffer[WARPSIZE];
T thread_sum = 0;
const data_size_t num_blocks_per_thread = (num_blocks + static_cast<data_size_t>(blockDim.x)) / static_cast<data_size_t>(blockDim.x);
const data_size_t thread_start_block_index = static_cast<data_size_t>(threadIdx.x) * num_blocks_per_thread;
const data_size_t thread_end_block_index = min(thread_start_block_index + num_blocks_per_thread, num_blocks + 1);
for (data_size_t block_index = thread_start_block_index; block_index < thread_end_block_index; ++block_index) {
thread_sum += block_buffer[block_index];
}
ShufflePrefixSumExclusive<T>(thread_sum, shared_buffer);
for (data_size_t block_index = thread_start_block_index; block_index < thread_end_block_index; ++block_index) {
block_buffer[block_index] += thread_sum;
}
}
template <typename T>
__global__ void GlobalInclusivePrefixSumAddBlockBaseKernel(const T* block_buffer, T* values, data_size_t num_data) {
const T block_sum_base = block_buffer[blockIdx.x];
const data_size_t data_index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
if (data_index < num_data) {
values[data_index] += block_sum_base;
}
}
template <typename VAL_T, typename REDUCE_T, typename INDEX_T>
void GlobalInclusiveArgPrefixSum(const INDEX_T* sorted_indices, const VAL_T* in_values, REDUCE_T* out_values, REDUCE_T* block_buffer, size_t n) {
const data_size_t num_data = static_cast<data_size_t>(n);
const data_size_t num_blocks = (num_data + GLOBAL_PREFIX_SUM_BLOCK_SIZE - 1) / GLOBAL_PREFIX_SUM_BLOCK_SIZE;
GlobalInclusiveArgPrefixSumKernel<INDEX_T, VAL_T, REDUCE_T><<<num_blocks, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(
sorted_indices, in_values, out_values, block_buffer, num_data);
SynchronizeCUDADevice(__FILE__, __LINE__);
GlobalInclusivePrefixSumReduceBlockKernel<REDUCE_T><<<1, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(
block_buffer, num_blocks);
SynchronizeCUDADevice(__FILE__, __LINE__);
GlobalInclusivePrefixSumAddBlockBaseKernel<REDUCE_T><<<num_blocks, GLOBAL_PREFIX_SUM_BLOCK_SIZE>>>(
block_buffer, out_values, num_data);
SynchronizeCUDADevice(__FILE__, __LINE__);
}
template void GlobalInclusiveArgPrefixSum<label_t, double, data_size_t>(const data_size_t* sorted_indices, const label_t* in_values, double* out_values, double* block_buffer, size_t n);
template <typename VAL_T, typename INDEX_T, bool ASCENDING>
__global__ void BitonicArgSortGlobalKernel(const VAL_T* values, INDEX_T* indices, const int num_total_data) {
const int thread_index = static_cast<int>(threadIdx.x);
const int low = static_cast<int>(blockIdx.x * BITONIC_SORT_NUM_ELEMENTS);
const bool outer_ascending = ASCENDING ? (blockIdx.x % 2 == 0) : (blockIdx.x % 2 == 1);
const VAL_T* values_pointer = values + low;
INDEX_T* indices_pointer = indices + low;
const int num_data = min(BITONIC_SORT_NUM_ELEMENTS, num_total_data - low);
__shared__ VAL_T shared_values[BITONIC_SORT_NUM_ELEMENTS];
__shared__ INDEX_T shared_indices[BITONIC_SORT_NUM_ELEMENTS];
if (thread_index < num_data) {
shared_values[thread_index] = values_pointer[thread_index];
shared_indices[thread_index] = static_cast<INDEX_T>(thread_index + blockIdx.x * blockDim.x);
}
__syncthreads();
for (int depth = BITONIC_SORT_DEPTH - 1; depth >= 1; --depth) {
const int segment_length = 1 << (BITONIC_SORT_DEPTH - depth);
const int segment_index = thread_index / segment_length;
const bool ascending = outer_ascending ? (segment_index % 2 == 0) : (segment_index % 2 == 1);
const int num_total_segment = (num_data + segment_length - 1) / segment_length;
{
const int inner_depth = depth;
const int inner_segment_length_half = 1 << (BITONIC_SORT_DEPTH - 1 - inner_depth);
const int inner_segment_index_half = thread_index / inner_segment_length_half;
const int offset = ((inner_segment_index_half >> 1) == num_total_segment - 1 && ascending == outer_ascending) ?
(num_total_segment * segment_length - num_data) : 0;
const int segment_start = segment_index * segment_length;
if (inner_segment_index_half % 2 == 0) {
if (thread_index >= offset + segment_start) {
const int index_to_compare = thread_index + inner_segment_length_half - offset;
const INDEX_T this_index = shared_indices[thread_index];
const INDEX_T other_index = shared_indices[index_to_compare];
const VAL_T this_value = shared_values[thread_index];
const VAL_T other_value = shared_values[index_to_compare];
if (index_to_compare < num_data && (this_value > other_value) == ascending) {
shared_indices[thread_index] = other_index;
shared_indices[index_to_compare] = this_index;
shared_values[thread_index] = other_value;
shared_values[index_to_compare] = this_value;
}
}
}
__syncthreads();
}
for (int inner_depth = depth + 1; inner_depth < BITONIC_SORT_DEPTH; ++inner_depth) {
const int inner_segment_length_half = 1 << (BITONIC_SORT_DEPTH - 1 - inner_depth);
const int inner_segment_index_half = thread_index / inner_segment_length_half;
if (inner_segment_index_half % 2 == 0) {
const int index_to_compare = thread_index + inner_segment_length_half;
const INDEX_T this_index = shared_indices[thread_index];
const INDEX_T other_index = shared_indices[index_to_compare];
const VAL_T this_value = shared_values[thread_index];
const VAL_T other_value = shared_values[index_to_compare];
if (index_to_compare < num_data && (this_value > other_value) == ascending) {
shared_indices[thread_index] = other_index;
shared_indices[index_to_compare] = this_index;
shared_values[thread_index] = other_value;
shared_values[index_to_compare] = this_value;
}
}
__syncthreads();
}
}
if (thread_index < num_data) {
indices_pointer[thread_index] = shared_indices[thread_index];
}
}
template <typename VAL_T, typename INDEX_T, bool ASCENDING>
__global__ void BitonicArgSortMergeKernel(const VAL_T* values, INDEX_T* indices, const int segment_length, const int len) {
const int thread_index = static_cast<int>(threadIdx.x + blockIdx.x * blockDim.x);
const int segment_index = thread_index / segment_length;
const bool ascending = ASCENDING ? (segment_index % 2 == 0) : (segment_index % 2 == 1);
__shared__ VAL_T shared_values[BITONIC_SORT_NUM_ELEMENTS];
__shared__ INDEX_T shared_indices[BITONIC_SORT_NUM_ELEMENTS];
const int offset = static_cast<int>(blockIdx.x * blockDim.x);
const int local_len = min(BITONIC_SORT_NUM_ELEMENTS, len - offset);
if (thread_index < len) {
const INDEX_T index = indices[thread_index];
shared_values[threadIdx.x] = values[index];
shared_indices[threadIdx.x] = index;
}
__syncthreads();
int half_segment_length = BITONIC_SORT_NUM_ELEMENTS / 2;
while (half_segment_length >= 1) {
const int half_segment_index = static_cast<int>(threadIdx.x) / half_segment_length;
if (half_segment_index % 2 == 0) {
const int index_to_compare = static_cast<int>(threadIdx.x) + half_segment_length;
const INDEX_T this_index = shared_indices[threadIdx.x];
const INDEX_T other_index = shared_indices[index_to_compare];
const VAL_T this_value = shared_values[threadIdx.x];
const VAL_T other_value = shared_values[index_to_compare];
if (index_to_compare < local_len && ((this_value > other_value) == ascending)) {
shared_indices[threadIdx.x] = other_index;
shared_indices[index_to_compare] = this_index;
shared_values[threadIdx.x] = other_value;
shared_values[index_to_compare] = this_value;
}
}
__syncthreads();
half_segment_length >>= 1;
}
if (thread_index < len) {
indices[thread_index] = shared_indices[threadIdx.x];
}
}
template <typename VAL_T, typename INDEX_T, bool ASCENDING, bool BEGIN>
__global__ void BitonicArgCompareKernel(const VAL_T* values, INDEX_T* indices, const int half_segment_length, const int outer_segment_length, const int len) {
const int thread_index = static_cast<int>(threadIdx.x + blockIdx.x * blockDim.x);
const int segment_index = thread_index / outer_segment_length;
const int half_segment_index = thread_index / half_segment_length;
const bool ascending = ASCENDING ? (segment_index % 2 == 0) : (segment_index % 2 == 1);
if (half_segment_index % 2 == 0) {
const int num_total_segment = (len + outer_segment_length - 1) / outer_segment_length;
if (BEGIN && (half_segment_index >> 1) == num_total_segment - 1 && ascending == ASCENDING) {
const int offset = num_total_segment * outer_segment_length - len;
const int segment_start = segment_index * outer_segment_length;
if (thread_index >= offset + segment_start) {
const int index_to_compare = thread_index + half_segment_length - offset;
if (index_to_compare < len) {
const INDEX_T this_index = indices[thread_index];
const INDEX_T other_index = indices[index_to_compare];
if ((values[this_index] > values[other_index]) == ascending) {
indices[thread_index] = other_index;
indices[index_to_compare] = this_index;
}
}
}
} else {
const int index_to_compare = thread_index + half_segment_length;
if (index_to_compare < len) {
const INDEX_T this_index = indices[thread_index];
const INDEX_T other_index = indices[index_to_compare];
if ((values[this_index] > values[other_index]) == ascending) {
indices[thread_index] = other_index;
indices[index_to_compare] = this_index;
}
}
}
}
}
template <typename VAL_T, typename INDEX_T, bool ASCENDING>
void BitonicArgSortGlobalHelper(const VAL_T* values, INDEX_T* indices, const size_t len) {
int max_depth = 1;
int len_to_shift = static_cast<int>(len) - 1;
while (len_to_shift > 0) {
++max_depth;
len_to_shift >>= 1;
}
const int num_blocks = (static_cast<int>(len) + BITONIC_SORT_NUM_ELEMENTS - 1) / BITONIC_SORT_NUM_ELEMENTS;
BitonicArgSortGlobalKernel<VAL_T, INDEX_T, ASCENDING><<<num_blocks, BITONIC_SORT_NUM_ELEMENTS>>>(values, indices, static_cast<int>(len));
SynchronizeCUDADevice(__FILE__, __LINE__);
for (int depth = max_depth - 11; depth >= 1; --depth) {
const int segment_length = (1 << (max_depth - depth));
int half_segment_length = (segment_length >> 1);
{
BitonicArgCompareKernel<VAL_T, INDEX_T, ASCENDING, true><<<num_blocks, BITONIC_SORT_NUM_ELEMENTS>>>(
values, indices, half_segment_length, segment_length, static_cast<int>(len));
SynchronizeCUDADevice(__FILE__, __LINE__);
half_segment_length >>= 1;
}
for (int inner_depth = depth + 1; inner_depth <= max_depth - 11; ++inner_depth) {
BitonicArgCompareKernel<VAL_T, INDEX_T, ASCENDING, false><<<num_blocks, BITONIC_SORT_NUM_ELEMENTS>>>(
values, indices, half_segment_length, segment_length, static_cast<int>(len));
SynchronizeCUDADevice(__FILE__, __LINE__);
half_segment_length >>= 1;
}
BitonicArgSortMergeKernel<VAL_T, INDEX_T, ASCENDING><<<num_blocks, BITONIC_SORT_NUM_ELEMENTS>>>(
values, indices, segment_length, static_cast<int>(len));
SynchronizeCUDADevice(__FILE__, __LINE__);
}
}
template <>
void BitonicArgSortGlobal<double, data_size_t, false>(const double* values, data_size_t* indices, const size_t len) {
BitonicArgSortGlobalHelper<double, data_size_t, false>(values, indices, len);
}
template <>
void BitonicArgSortGlobal<double, data_size_t, true>(const double* values, data_size_t* indices, const size_t len) {
BitonicArgSortGlobalHelper<double, data_size_t, true>(values, indices, len);
}
template <>
void BitonicArgSortGlobal<label_t, data_size_t, false>(const label_t* values, data_size_t* indices, const size_t len) {
BitonicArgSortGlobalHelper<label_t, data_size_t, false>(values, indices, len);
}
template <>
void BitonicArgSortGlobal<data_size_t, int, true>(const data_size_t* values, int* indices, const size_t len) {
BitonicArgSortGlobalHelper<data_size_t, int, true>(values, indices, len);
}
} // namespace LightGBM
#endif // USE_CUDA
src/cuda/cuda_utils.cpp
+61
View File
@@ -0,0 +1,61 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_rocm_interop.h>
#include <LightGBM/cuda/cuda_utils.hu>
namespace LightGBM {
void SynchronizeCUDADevice(const char* file, const int line) {
gpuAssert(cudaDeviceSynchronize(), file, line);
}
void SynchronizeCUDAStream(cudaStream_t cuda_stream, const char* file, const int line) {
gpuAssert(cudaStreamSynchronize(cuda_stream), file, line);
}
void PrintLastCUDAError() {
const char* error_name = cudaGetErrorName(cudaGetLastError());
Log::Fatal(error_name);
}
void SetCUDADevice(int gpu_device_id, const char* file, int line) {
int cur_gpu_device_id = 0;
CUDASUCCESS_OR_FATAL_OUTER(cudaGetDevice(&cur_gpu_device_id));
if (cur_gpu_device_id != gpu_device_id) {
CUDASUCCESS_OR_FATAL_OUTER(cudaSetDevice(gpu_device_id));
}
}
int GetCUDADevice(const char* file, int line) {
int cur_gpu_device_id = 0;
CUDASUCCESS_OR_FATAL_OUTER(cudaGetDevice(&cur_gpu_device_id));
return cur_gpu_device_id;
}
cudaStream_t CUDAStreamCreate() {
cudaStream_t cuda_stream;
CUDASUCCESS_OR_FATAL(cudaStreamCreate(&cuda_stream));
return cuda_stream;
}
void CUDAStreamDestroy(cudaStream_t cuda_stream) {
CUDASUCCESS_OR_FATAL(cudaStreamDestroy(cuda_stream));
}
void NCCLGroupStart() {
NCCLCHECK(ncclGroupStart());
}
void NCCLGroupEnd() {
NCCLCHECK(ncclGroupEnd());
}
} // namespace LightGBM
#endif // USE_CUDA
src/io/bin.cpp
+1077
View File
File diff suppressed because it is too large Load Diff
src/io/config.cpp
+519
View File
@@ -0,0 +1,519 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/config.h>
#include <LightGBM/cuda/vector_cudahost.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/log.h>
#include <LightGBM/utils/random.h>
#include <algorithm>
#include <cctype>
#include <limits>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace LightGBM {
void Config::KV2Map(std::unordered_map<std::string, std::vector<std::string>>* params, const char* kv) {
std::vector<std::string> tmp_strs = Common::Split(kv, '=');
if (tmp_strs.size() == 2 || tmp_strs.size() == 1) {
std::string key = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[0]));
std::string value = "";
if (tmp_strs.size() == 2) {
value = Common::RemoveQuotationSymbol(Common::Trim(tmp_strs[1]));
}
if (key.size() > 0) {
params->operator[](key).emplace_back(value);
}
} else {
Log::Warning("Unknown parameter %s", kv);
}
}
void GetFirstValueAsInt(const std::unordered_map<std::string, std::vector<std::string>>& params, std::string key, int* out) {
const auto pair = params.find(key);
if (pair != params.end()) {
auto candidate = pair->second[0].c_str();
if (!Common::AtoiAndCheck(candidate, out)) {
Log::Fatal("Parameter %s should be of type int, got \"%s\"", key.c_str(), candidate);
}
}
}
void Config::SetVerbosity(const std::unordered_map<std::string, std::vector<std::string>>& params) {
int verbosity = 1;
// if "verbosity" was found in params, prefer that to any other aliases
const auto verbosity_iter = params.find("verbosity");
if (verbosity_iter != params.end()) {
GetFirstValueAsInt(params, "verbosity", &verbosity);
} else {
// if "verbose" was found in params and "verbosity" was not, use that value
const auto verbose_iter = params.find("verbose");
if (verbose_iter != params.end()) {
GetFirstValueAsInt(params, "verbose", &verbosity);
} else {
// if "verbosity" and "verbose" were both missing from params, don't modify LightGBM's log level
return;
}
}
// otherwise, update LightGBM's log level based on the passed-in value
if (verbosity < 0) {
LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Fatal);
} else if (verbosity == 0) {
LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Warning);
} else if (verbosity == 1) {
LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Info);
} else {
LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Debug);
}
}
void Config::KeepFirstValues(const std::unordered_map<std::string, std::vector<std::string>>& params, std::unordered_map<std::string, std::string>* out) {
for (auto pair = params.begin(); pair != params.end(); ++pair) {
auto name = pair->first.c_str();
auto values = pair->second;
out->emplace(name, values[0]);
for (size_t i = 1; i < pair->second.size(); ++i) {
Log::Warning("%s is set=%s, %s=%s will be ignored. Current value: %s=%s",
name, values[0].c_str(),
name, values[i].c_str(),
name, values[0].c_str());
}
}
}
std::unordered_map<std::string, std::string> Config::Str2Map(const char* parameters) {
std::unordered_map<std::string, std::vector<std::string>> all_params;
std::unordered_map<std::string, std::string> params;
auto args = Common::Split(parameters, " \t\n\r");
for (auto arg : args) {
KV2Map(&all_params, Common::Trim(arg).c_str());
}
SetVerbosity(all_params);
KeepFirstValues(all_params, &params);
ParameterAlias::KeyAliasTransform(&params);
return params;
}
void GetBoostingType(const std::unordered_map<std::string, std::string>& params, std::string* boosting) {
std::string value;
if (Config::GetString(params, "boosting", &value)) {
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c){ return std::tolower(c); });
if (value == std::string("gbdt") || value == std::string("gbrt")) {
*boosting = "gbdt";
} else if (value == std::string("dart")) {
*boosting = "dart";
} else if (value == std::string("goss")) {
*boosting = "goss";
} else if (value == std::string("rf") || value == std::string("random_forest")) {
*boosting = "rf";
} else {
Log::Fatal("Unknown boosting type %s", value.c_str());
}
}
}
void GetDataSampleStrategy(const std::unordered_map<std::string, std::string>& params, std::string* strategy) {
std::string value;
if (Config::GetString(params, "data_sample_strategy", &value)) {
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c){ return std::tolower(c); });
if (value == std::string("goss")) {
*strategy = "goss";
} else if (value == std::string("bagging")) {
*strategy = "bagging";
} else {
Log::Fatal("Unknown sample strategy %s", value.c_str());
}
}
}
void ParseMetrics(const std::string& value, std::vector<std::string>* out_metric) {
std::unordered_set<std::string> metric_sets;
out_metric->clear();
std::vector<std::string> metrics = Common::Split(value.c_str(), ',');
for (auto& met : metrics) {
auto type = ParseMetricAlias(met);
if (metric_sets.count(type) <= 0) {
out_metric->push_back(type);
metric_sets.insert(type);
}
}
}
void GetObjectiveType(const std::unordered_map<std::string, std::string>& params, std::string* objective) {
std::string value;
if (Config::GetString(params, "objective", &value)) {
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c){ return std::tolower(c); });
*objective = ParseObjectiveAlias(value);
}
}
void GetMetricType(const std::unordered_map<std::string, std::string>& params, const std::string& objective, std::vector<std::string>* metric) {
std::string value;
if (Config::GetString(params, "metric", &value)) {
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c){ return std::tolower(c); });
ParseMetrics(value, metric);
}
// add names of objective function if not providing metric
if (metric->empty() && value.size() == 0) {
ParseMetrics(objective, metric);
}
}
void GetTaskType(const std::unordered_map<std::string, std::string>& params, TaskType* task) {
std::string value;
if (Config::GetString(params, "task", &value)) {
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c){ return std::tolower(c); });
if (value == std::string("train") || value == std::string("training")) {
*task = TaskType::kTrain;
} else if (value == std::string("predict") || value == std::string("prediction")
|| value == std::string("test")) {
*task = TaskType::kPredict;
} else if (value == std::string("convert_model")) {
*task = TaskType::kConvertModel;
} else if (value == std::string("refit") || value == std::string("refit_tree")) {
*task = TaskType::KRefitTree;
} else if (value == std::string("save_binary")) {
*task = TaskType::kSaveBinary;
} else {
Log::Fatal("Unknown task type %s", value.c_str());
}
}
}
void GetDeviceType(const std::unordered_map<std::string, std::string>& params, std::string* device_type) {
std::string value;
if (Config::GetString(params, "device_type", &value)) {
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c){ return std::tolower(c); });
if (value == std::string("cpu")) {
*device_type = "cpu";
} else if (value == std::string("gpu")) {
*device_type = "gpu";
} else if (value == std::string("cuda")) {
*device_type = "cuda";
} else {
Log::Fatal("Unknown device type %s", value.c_str());
}
}
}
void GetTreeLearnerType(const std::unordered_map<std::string, std::string>& params, std::string* tree_learner) {
std::string value;
if (Config::GetString(params, "tree_learner", &value)) {
std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c){ return std::tolower(c); });
if (value == std::string("serial")) {
*tree_learner = "serial";
} else if (value == std::string("feature") || value == std::string("feature_parallel")) {
*tree_learner = "feature";
} else if (value == std::string("data") || value == std::string("data_parallel")) {
*tree_learner = "data";
} else if (value == std::string("voting") || value == std::string("voting_parallel")) {
*tree_learner = "voting";
} else {
Log::Fatal("Unknown tree learner type %s", value.c_str());
}
}
}
void Config::GetAucMuWeights() {
if (auc_mu_weights.empty()) {
// equal weights for all classes
auc_mu_weights_matrix = std::vector<std::vector<double>> (num_class, std::vector<double>(num_class, 1));
for (size_t i = 0; i < static_cast<size_t>(num_class); ++i) {
auc_mu_weights_matrix[i][i] = 0;
}
} else {
auc_mu_weights_matrix = std::vector<std::vector<double>> (num_class, std::vector<double>(num_class, 0));
if (auc_mu_weights.size() != static_cast<size_t>(num_class * num_class)) {
Log::Fatal("auc_mu_weights must have %d elements, but found %zu", num_class * num_class, auc_mu_weights.size());
}
for (size_t i = 0; i < static_cast<size_t>(num_class); ++i) {
for (size_t j = 0; j < static_cast<size_t>(num_class); ++j) {
if (i == j) {
auc_mu_weights_matrix[i][j] = 0;
if (std::fabs(auc_mu_weights[i * num_class + j]) > kZeroThreshold) {
Log::Info("AUC-mu matrix must have zeros on diagonal. Overwriting value in position %zu of auc_mu_weights with 0.", i * num_class + j);
}
} else {
if (std::fabs(auc_mu_weights[i * num_class + j]) < kZeroThreshold) {
Log::Fatal("AUC-mu matrix must have non-zero values for non-diagonal entries. Found zero value in position %zu of auc_mu_weights.", i * num_class + j);
}
auc_mu_weights_matrix[i][j] = auc_mu_weights[i * num_class + j];
}
}
}
}
}
void Config::GetInteractionConstraints() {
if (interaction_constraints == "") {
interaction_constraints_vector = std::vector<std::vector<int>>();
} else {
interaction_constraints_vector = Common::StringToArrayofArrays<int>(interaction_constraints, '[', ']', ',');
}
}
void Config::Set(const std::unordered_map<std::string, std::string>& params) {
// generate seeds by seed.
if (GetInt(params, "seed", &seed)) {
Random rand(seed);
int int_max = std::numeric_limits<int16_t>::max();
data_random_seed = static_cast<int>(rand.NextShort(0, int_max));
bagging_seed = static_cast<int>(rand.NextShort(0, int_max));
drop_seed = static_cast<int>(rand.NextShort(0, int_max));
feature_fraction_seed = static_cast<int>(rand.NextShort(0, int_max));
objective_seed = static_cast<int>(rand.NextShort(0, int_max));
extra_seed = static_cast<int>(rand.NextShort(0, int_max));
}
GetTaskType(params, &task);
GetBoostingType(params, &boosting);
GetDataSampleStrategy(params, &data_sample_strategy);
GetObjectiveType(params, &objective);
GetMetricType(params, objective, &metric);
GetDeviceType(params, &device_type);
if (device_type == std::string("cuda")) {
LGBM_config_::current_device = lgbm_device_cuda;
}
GetTreeLearnerType(params, &tree_learner);
GetMembersFromString(params);
GetAucMuWeights();
GetInteractionConstraints();
// sort eval_at
std::sort(eval_at.begin(), eval_at.end());
std::vector<std::string> new_valid;
for (size_t i = 0; i < valid.size(); ++i) {
if (valid[i] != data) {
// Only push the non-training data
new_valid.push_back(valid[i]);
} else {
is_provide_training_metric = true;
}
}
valid = new_valid;
if ((task == TaskType::kSaveBinary) && !save_binary) {
Log::Info("save_binary parameter set to true because task is save_binary");
save_binary = true;
}
// check for conflicts
CheckParamConflict(params);
}
bool CheckMultiClassObjective(const std::string& objective) {
return (objective == std::string("multiclass") || objective == std::string("multiclassova"));
}
void Config::CheckParamConflict(const std::unordered_map<std::string, std::string>& params) {
// check if objective, metric, and num_class match
int num_class_check = num_class;
bool objective_type_multiclass = CheckMultiClassObjective(objective) || (objective == std::string("custom") && num_class_check > 1);
if (objective_type_multiclass) {
if (num_class_check <= 1) {
Log::Fatal("Number of classes should be specified and greater than 1 for multiclass training");
}
} else {
if (task == TaskType::kTrain && num_class_check != 1) {
Log::Fatal("Number of classes must be 1 for non-multiclass training");
}
}
for (std::string metric_type : metric) {
bool metric_type_multiclass = (CheckMultiClassObjective(metric_type)
|| metric_type == std::string("multi_logloss")
|| metric_type == std::string("multi_error")
|| metric_type == std::string("auc_mu")
|| (metric_type == std::string("custom") && num_class_check > 1));
if ((objective_type_multiclass && !metric_type_multiclass)
|| (!objective_type_multiclass && metric_type_multiclass)) {
Log::Fatal("Multiclass objective and metrics don't match");
}
}
if (num_machines > 1) {
is_parallel = true;
} else {
is_parallel = false;
tree_learner = "serial";
}
bool is_single_tree_learner = tree_learner == std::string("serial");
if (is_single_tree_learner) {
is_parallel = false;
num_machines = 1;
}
if (is_single_tree_learner || tree_learner == std::string("feature")) {
is_data_based_parallel = false;
} else if (tree_learner == std::string("data")
|| tree_learner == std::string("voting")) {
is_data_based_parallel = true;
if (histogram_pool_size >= 0
&& tree_learner == std::string("data")) {
Log::Warning("Histogram LRU queue was enabled (histogram_pool_size=%f).\n"
"Will disable this to reduce communication costs",
histogram_pool_size);
// Change pool size to -1 (no limit) when using data parallel to reduce communication costs
histogram_pool_size = -1;
}
}
if (is_data_based_parallel) {
if (!forcedsplits_filename.empty()) {
Log::Fatal("Don't support forcedsplits in %s tree learner",
tree_learner.c_str());
}
}
// max_depth defaults to -1, so max_depth>0 implies "you explicitly overrode the default"
//
// Changing max_depth while leaving num_leaves at its default (31) can lead to 2 undesirable situations:
//
// * (0 <= max_depth <= 4) it's not possible to produce a tree with 31 leaves
// - this block reduces num_leaves to 2^max_depth
// * (max_depth > 4) 31 leaves is less than a full depth-wise tree, which might lead to underfitting
// - this block warns about that
// ref: https://github.com/lightgbm-org/LightGBM/issues/2898#issuecomment-1002860601
if (max_depth > 0 && (params.count("num_leaves") == 0 || params.at("num_leaves").empty())) {
double full_num_leaves = std::pow(2, max_depth);
if (full_num_leaves > num_leaves) {
Log::Warning("Provided parameters constrain tree depth (max_depth=%d) without explicitly setting 'num_leaves'. "
"This can lead to underfitting. To resolve this warning, pass 'num_leaves' (<=%.0f) in params. "
"Alternatively, pass (max_depth=-1) and just use 'num_leaves' to constrain model complexity.",
max_depth,
full_num_leaves);
}
if (full_num_leaves < num_leaves) {
// Fits in an int, and is more restrictive than the current num_leaves
num_leaves = static_cast<int>(full_num_leaves);
}
}
if (device_type == std::string("gpu")) {
// force col-wise for gpu version
force_col_wise = true;
force_row_wise = false;
if (deterministic) {
Log::Warning("Although \"deterministic\" is set, the results ran by GPU may be non-deterministic.");
}
if (use_quantized_grad) {
Log::Warning("Quantized training is not supported by GPU tree learner. Switch to full precision training.");
use_quantized_grad = false;
}
} else if (device_type == std::string("cuda")) {
// force row-wise for cuda version
force_col_wise = false;
force_row_wise = true;
if (deterministic) {
Log::Warning("Although \"deterministic\" is set, the results ran by GPU may be non-deterministic.");
}
}
// linear tree learner must be serial type and run on CPU device
if (linear_tree) {
if (device_type != std::string("cpu") && device_type != std::string("gpu")) {
device_type = "cpu";
Log::Warning("Linear tree learner only works with CPU and GPU. Falling back to CPU now.");
}
if (tree_learner != std::string("serial")) {
tree_learner = "serial";
Log::Warning("Linear tree learner must be serial.");
}
if (zero_as_missing) {
Log::Fatal("zero_as_missing must be false when fitting linear trees.");
}
if (objective == std::string("regression_l1")) {
Log::Fatal("Cannot use regression_l1 objective when fitting linear trees.");
}
}
// min_data_in_leaf must be at least 2 if path smoothing is active. This is because when the split is calculated
// the count is calculated using the proportion of hessian in the leaf which is rounded up to nearest int, so it can
// be 1 when there is actually no data in the leaf. In rare cases this can cause a bug because with path smoothing the
// calculated split gain can be positive even with zero gradient and hessian.
if (path_smooth > kEpsilon && min_data_in_leaf < 2) {
min_data_in_leaf = 2;
Log::Warning("min_data_in_leaf has been increased to 2 because this is required when path smoothing is active.");
}
if (is_parallel && (monotone_constraints_method == std::string("intermediate") || monotone_constraints_method == std::string("advanced"))) {
// In distributed mode, local node doesn't have histograms on all features, cannot perform "intermediate" monotone constraints.
Log::Warning("Cannot use \"intermediate\" or \"advanced\" monotone constraints in distributed learning, auto set to \"basic\" method.");
monotone_constraints_method = "basic";
}
if (feature_fraction_bynode != 1.0 && (monotone_constraints_method == std::string("intermediate") || monotone_constraints_method == std::string("advanced"))) {
// "intermediate" monotone constraints need to recompute splits. If the features are sampled when computing the
// split initially, then the sampling needs to be recorded or done once again, which is currently not supported
Log::Warning("Cannot use \"intermediate\" or \"advanced\" monotone constraints with feature fraction different from 1, auto set monotone constraints to \"basic\" method.");
monotone_constraints_method = "basic";
}
if (max_depth > 0 && monotone_penalty >= max_depth) {
Log::Warning("Monotone penalty greater than tree depth. Monotone features won't be used.");
}
if (min_data_in_leaf <= 0 && min_sum_hessian_in_leaf <= kEpsilon) {
Log::Warning(
"Cannot set both min_data_in_leaf and min_sum_hessian_in_leaf to 0. "
"Will set min_data_in_leaf to 1.");
min_data_in_leaf = 1;
}
if (boosting == std::string("goss")) {
boosting = std::string("gbdt");
data_sample_strategy = std::string("goss");
Log::Warning("Found boosting=goss. For backwards compatibility reasons, LightGBM interprets this as boosting=gbdt, data_sample_strategy=goss."
"To suppress this warning, set data_sample_strategy=goss instead.");
}
if (bagging_by_query && data_sample_strategy != std::string("bagging")) {
Log::Warning("bagging_by_query=true is only compatible with data_sample_strategy=bagging. Setting bagging_by_query=false.");
bagging_by_query = false;
}
}
std::string Config::ToString() const {
std::stringstream str_buf;
str_buf << "[boosting: " << boosting << "]\n";
str_buf << "[objective: " << objective << "]\n";
str_buf << "[metric: " << Common::Join(metric, ",") << "]\n";
str_buf << "[tree_learner: " << tree_learner << "]\n";
str_buf << "[device_type: " << device_type << "]\n";
str_buf << SaveMembersToString();
return str_buf.str();
}
const std::string Config::DumpAliases() {
auto map = Config::parameter2aliases();
for (auto& pair : map) {
std::sort(pair.second.begin(), pair.second.end(), SortAlias);
}
std::stringstream str_buf;
str_buf << "{\n";
bool first = true;
for (const auto& pair : map) {
if (first) {
str_buf << " \"";
first = false;
} else {
str_buf << " , \"";
}
str_buf << pair.first << "\": [";
if (pair.second.size() > 0) {
str_buf << "\"" << CommonC::Join(pair.second, "\", \"") << "\"";
}
str_buf << "]\n";
}
str_buf << "}\n";
return str_buf.str();
}
} // namespace LightGBM
src/io/config_auto.cpp
+1096
View File
File diff suppressed because it is too large Load Diff
src/io/cuda/cuda_column_data.cpp
+227
View File
@@ -0,0 +1,227 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_column_data.hpp>
#include <cstdint>
#include <vector>
namespace LightGBM {
CUDAColumnData::CUDAColumnData(const data_size_t num_data, const int gpu_device_id) {
num_threads_ = OMP_NUM_THREADS();
num_data_ = num_data;
gpu_device_id_ = gpu_device_id >= 0 ? gpu_device_id : 0;
SetCUDADevice(gpu_device_id_, __FILE__, __LINE__);
data_by_column_.clear();
}
CUDAColumnData::~CUDAColumnData() {}
template <bool IS_SPARSE, bool IS_4BIT, typename BIN_TYPE>
void CUDAColumnData::InitOneColumnData(const void* in_column_data, BinIterator* bin_iterator, CUDAVector<uint8_t>* out_column_data_pointer) {
CUDAVector<BIN_TYPE> cuda_column_data;
if (!IS_SPARSE) {
if (IS_4BIT) {
std::vector<BIN_TYPE> expanded_column_data(num_data_, 0);
const BIN_TYPE* in_column_data_reintrepreted = reinterpret_cast<const BIN_TYPE*>(in_column_data);
for (data_size_t i = 0; i < num_data_; ++i) {
expanded_column_data[i] = static_cast<BIN_TYPE>((in_column_data_reintrepreted[i >> 1] >> ((i & 1) << 2)) & 0xf);
}
cuda_column_data.InitFromHostVector(expanded_column_data);
} else {
cuda_column_data.InitFromHostMemory(reinterpret_cast<const BIN_TYPE*>(in_column_data), static_cast<size_t>(num_data_));
}
} else {
// need to iterate bin iterator
std::vector<BIN_TYPE> expanded_column_data(num_data_, 0);
for (data_size_t i = 0; i < num_data_; ++i) {
expanded_column_data[i] = static_cast<BIN_TYPE>(bin_iterator->RawGet(i));
}
cuda_column_data.InitFromHostVector(expanded_column_data);
}
out_column_data_pointer->MoveFrom(cuda_column_data, sizeof(BIN_TYPE) * cuda_column_data.Size());
}
void CUDAColumnData::Init(const int num_columns,
const std::vector<const void*>& column_data,
const std::vector<BinIterator*>& column_bin_iterator,
const std::vector<uint8_t>& column_bit_type,
const std::vector<uint32_t>& feature_max_bin,
const std::vector<uint32_t>& feature_min_bin,
const std::vector<uint32_t>& feature_offset,
const std::vector<uint32_t>& feature_most_freq_bin,
const std::vector<uint32_t>& feature_default_bin,
const std::vector<uint8_t>& feature_missing_is_zero,
const std::vector<uint8_t>& feature_missing_is_na,
const std::vector<uint8_t>& feature_mfb_is_zero,
const std::vector<uint8_t>& feature_mfb_is_na,
const std::vector<int>& feature_to_column) {
num_columns_ = num_columns;
column_bit_type_ = column_bit_type;
feature_max_bin_ = feature_max_bin;
feature_min_bin_ = feature_min_bin;
feature_offset_ = feature_offset;
feature_most_freq_bin_ = feature_most_freq_bin;
feature_default_bin_ = feature_default_bin;
feature_missing_is_zero_ = feature_missing_is_zero;
feature_missing_is_na_ = feature_missing_is_na;
feature_mfb_is_zero_ = feature_mfb_is_zero;
feature_mfb_is_na_ = feature_mfb_is_na;
for (int column_index = 0; column_index < num_columns_; ++column_index) {
data_by_column_.emplace_back(new CUDAVector<uint8_t>());
}
OMP_INIT_EX();
#pragma omp parallel num_threads(num_threads_)
{
SetCUDADevice(gpu_device_id_, __FILE__, __LINE__);
#pragma omp for schedule(static)
for (int column_index = 0; column_index < num_columns_; ++column_index) {
OMP_LOOP_EX_BEGIN();
const int8_t bit_type = column_bit_type[column_index];
if (column_data[column_index] != nullptr) {
// is dense column
if (bit_type == 4) {
column_bit_type_[column_index] = 8;
InitOneColumnData<false, true, uint8_t>(column_data[column_index], nullptr, data_by_column_[column_index].get());
} else if (bit_type == 8) {
InitOneColumnData<false, false, uint8_t>(column_data[column_index], nullptr, data_by_column_[column_index].get());
} else if (bit_type == 16) {
InitOneColumnData<false, false, uint16_t>(column_data[column_index], nullptr, data_by_column_[column_index].get());
} else if (bit_type == 32) {
InitOneColumnData<false, false, uint32_t>(column_data[column_index], nullptr, data_by_column_[column_index].get());
} else {
Log::Fatal("Unknown column bit type %d", bit_type);
}
} else {
// is sparse column
if (bit_type == 8) {
InitOneColumnData<true, false, uint8_t>(nullptr, column_bin_iterator[column_index], data_by_column_[column_index].get());
} else if (bit_type == 16) {
InitOneColumnData<true, false, uint16_t>(nullptr, column_bin_iterator[column_index], data_by_column_[column_index].get());
} else if (bit_type == 32) {
InitOneColumnData<true, false, uint32_t>(nullptr, column_bin_iterator[column_index], data_by_column_[column_index].get());
} else {
Log::Fatal("Unknown column bit type %d", bit_type);
}
}
OMP_LOOP_EX_END();
}
}
OMP_THROW_EX();
feature_to_column_ = feature_to_column;
cuda_data_by_column_.InitFromHostVector(GetDataByColumnPointers(data_by_column_));
InitColumnMetaInfo();
}
void CUDAColumnData::CopySubrow(
const CUDAColumnData* full_set,
const data_size_t* used_indices,
const data_size_t num_used_indices) {
num_threads_ = full_set->num_threads_;
num_columns_ = full_set->num_columns_;
column_bit_type_ = full_set->column_bit_type_;
feature_min_bin_ = full_set->feature_min_bin_;
feature_max_bin_ = full_set->feature_max_bin_;
feature_offset_ = full_set->feature_offset_;
feature_most_freq_bin_ = full_set->feature_most_freq_bin_;
feature_default_bin_ = full_set->feature_default_bin_;
feature_missing_is_zero_ = full_set->feature_missing_is_zero_;
feature_missing_is_na_ = full_set->feature_missing_is_na_;
feature_mfb_is_zero_ = full_set->feature_mfb_is_zero_;
feature_mfb_is_na_ = full_set->feature_mfb_is_na_;
feature_to_column_ = full_set->feature_to_column_;
if (cuda_used_indices_.Size() == 0) {
// initialize the subset cuda column data
const size_t num_used_indices_size = static_cast<size_t>(num_used_indices);
cuda_used_indices_.Resize(num_used_indices_size);
for (int column_index = 0; column_index < num_columns_; ++column_index) {
data_by_column_.emplace_back(new CUDAVector<uint8_t>());
}
OMP_INIT_EX();
#pragma omp parallel num_threads(num_threads_)
{
SetCUDADevice(gpu_device_id_, __FILE__, __LINE__);
#pragma omp for schedule(static)
for (int column_index = 0; column_index < num_columns_; ++column_index) {
OMP_LOOP_EX_BEGIN();
const uint8_t bit_type = column_bit_type_[column_index];
if (bit_type == 8) {
CUDAVector<uint8_t> column_data;
column_data.Resize(num_used_indices_size);
data_by_column_[column_index]->MoveFrom(column_data, sizeof(uint8_t) * column_data.Size());
} else if (bit_type == 16) {
CUDAVector<uint16_t> column_data;
column_data.Resize(num_used_indices_size);
data_by_column_[column_index]->MoveFrom(column_data, sizeof(uint16_t) * column_data.Size());
} else if (bit_type == 32) {
CUDAVector<uint32_t> column_data;
column_data.Resize(num_used_indices_size);
data_by_column_[column_index]->MoveFrom(column_data, sizeof(uint32_t) * column_data.Size());
}
OMP_LOOP_EX_END();
}
}
OMP_THROW_EX();
cuda_data_by_column_.InitFromHostVector(GetDataByColumnPointers(data_by_column_));
InitColumnMetaInfo();
cur_subset_buffer_size_ = num_used_indices;
} else {
if (num_used_indices > cur_subset_buffer_size_) {
ResizeWhenCopySubrow(num_used_indices);
cur_subset_buffer_size_ = num_used_indices;
}
}
cuda_used_indices_.InitFromHostMemory(used_indices, static_cast<size_t>(num_used_indices));
num_used_indices_ = num_used_indices;
LaunchCopySubrowKernel(full_set->cuda_data_by_column());
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDAColumnData::ResizeWhenCopySubrow(const data_size_t num_used_indices) {
const size_t num_used_indices_size = static_cast<size_t>(num_used_indices);
cuda_used_indices_.Resize(num_used_indices_size);
OMP_INIT_EX();
#pragma omp parallel num_threads(num_threads_)
{
SetCUDADevice(gpu_device_id_, __FILE__, __LINE__);
#pragma omp for schedule(static)
for (int column_index = 0; column_index < num_columns_; ++column_index) {
OMP_LOOP_EX_BEGIN();
const uint8_t bit_type = column_bit_type_[column_index];
if (bit_type == 8) {
data_by_column_[column_index]->Resize(sizeof(uint8_t) * num_used_indices_size);
} else if (bit_type == 16) {
data_by_column_[column_index]->Resize(sizeof(uint16_t) * num_used_indices_size);
} else if (bit_type == 32) {
data_by_column_[column_index]->Resize(sizeof(uint32_t) * num_used_indices_size);
}
OMP_LOOP_EX_END();
}
}
OMP_THROW_EX();
cuda_data_by_column_.InitFromHostVector(GetDataByColumnPointers(data_by_column_));
}
void CUDAColumnData::InitColumnMetaInfo() {
cuda_column_bit_type_.InitFromHostVector(column_bit_type_);
cuda_feature_max_bin_.InitFromHostVector(feature_max_bin_);
cuda_feature_min_bin_.InitFromHostVector(feature_min_bin_);
cuda_feature_offset_.InitFromHostVector(feature_offset_);
cuda_feature_most_freq_bin_.InitFromHostVector(feature_most_freq_bin_);
cuda_feature_default_bin_.InitFromHostVector(feature_default_bin_);
cuda_feature_missing_is_zero_.InitFromHostVector(feature_missing_is_zero_);
cuda_feature_missing_is_na_.InitFromHostVector(feature_missing_is_na_);
cuda_feature_mfb_is_zero_.InitFromHostVector(feature_mfb_is_zero_);
cuda_feature_mfb_is_na_.InitFromHostVector(feature_mfb_is_na_);
cuda_feature_to_column_.InitFromHostVector(feature_to_column_);
}
} // namespace LightGBM
#endif // USE_CUDA
src/io/cuda/cuda_column_data.cu
+62
View File
@@ -0,0 +1,62 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_column_data.hpp>
#define COPY_SUBROW_BLOCK_SIZE_COLUMN_DATA (1024)
namespace LightGBM {
__global__ void CopySubrowKernel_ColumnData(
uint8_t* const* in_cuda_data_by_column,
const uint8_t* cuda_column_bit_type,
const data_size_t* cuda_used_indices,
const data_size_t num_used_indices,
const int num_column,
uint8_t** out_cuda_data_by_column) {
const data_size_t local_data_index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
if (local_data_index < num_used_indices) {
for (int column_index = 0; column_index < num_column; ++column_index) {
const uint8_t* in_column_data = in_cuda_data_by_column[column_index];
uint8_t* out_column_data = out_cuda_data_by_column[column_index];
const uint8_t bit_type = cuda_column_bit_type[column_index];
if (bit_type == 8) {
const uint8_t* true_in_column_data = reinterpret_cast<const uint8_t*>(in_column_data);
uint8_t* true_out_column_data = reinterpret_cast<uint8_t*>(out_column_data);
const data_size_t global_data_index = cuda_used_indices[local_data_index];
true_out_column_data[local_data_index] = true_in_column_data[global_data_index];
} else if (bit_type == 16) {
const uint16_t* true_in_column_data = reinterpret_cast<const uint16_t*>(in_column_data);
uint16_t* true_out_column_data = reinterpret_cast<uint16_t*>(out_column_data);
const data_size_t global_data_index = cuda_used_indices[local_data_index];
true_out_column_data[local_data_index] = true_in_column_data[global_data_index];
} else if (bit_type == 32) {
const uint32_t* true_in_column_data = reinterpret_cast<const uint32_t*>(in_column_data);
uint32_t* true_out_column_data = reinterpret_cast<uint32_t*>(out_column_data);
const data_size_t global_data_index = cuda_used_indices[local_data_index];
true_out_column_data[local_data_index] = true_in_column_data[global_data_index];
}
}
}
}
void CUDAColumnData::LaunchCopySubrowKernel(uint8_t* const* in_cuda_data_by_column) {
const int num_blocks = (num_used_indices_ + COPY_SUBROW_BLOCK_SIZE_COLUMN_DATA - 1) / COPY_SUBROW_BLOCK_SIZE_COLUMN_DATA;
CopySubrowKernel_ColumnData<<<num_blocks, COPY_SUBROW_BLOCK_SIZE_COLUMN_DATA>>>(
in_cuda_data_by_column,
cuda_column_bit_type_.RawData(),
cuda_used_indices_.RawData(),
num_used_indices_,
num_columns_,
cuda_data_by_column_.RawData());
}
} // namespace LightGBM
#endif // USE_CUDA
src/io/cuda/cuda_metadata.cpp
+79
View File
@@ -0,0 +1,79 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_metadata.hpp>
#include <vector>
namespace LightGBM {
CUDAMetadata::CUDAMetadata(const int gpu_device_id) {
if (gpu_device_id >= 0) {
SetCUDADevice(gpu_device_id, __FILE__, __LINE__);
} else {
SetCUDADevice(0, __FILE__, __LINE__);
}
}
CUDAMetadata::~CUDAMetadata() {}
void CUDAMetadata::Init(const std::vector<label_t>& label,
const std::vector<label_t>& weight,
const std::vector<data_size_t>& query_boundaries,
const std::vector<label_t>& query_weights,
const std::vector<double>& init_score) {
if (label.size() == 0) {
cuda_label_.Clear();
} else {
cuda_label_.InitFromHostVector(label);
}
if (weight.size() == 0) {
cuda_weights_.Clear();
} else {
cuda_weights_.InitFromHostVector(weight);
}
if (query_boundaries.size() == 0) {
cuda_query_boundaries_.Clear();
} else {
cuda_query_boundaries_.InitFromHostVector(query_boundaries);
}
if (query_weights.size() == 0) {
cuda_query_weights_.Clear();
} else {
cuda_query_weights_.InitFromHostVector(query_weights);
}
if (init_score.size() == 0) {
cuda_init_score_.Clear();
} else {
cuda_init_score_.InitFromHostVector(init_score);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDAMetadata::SetLabel(const label_t* label, data_size_t len) {
cuda_label_.InitFromHostMemory(label, static_cast<size_t>(len));
}
void CUDAMetadata::SetWeights(const label_t* weights, data_size_t len) {
cuda_weights_.InitFromHostMemory(weights, static_cast<size_t>(len));
}
void CUDAMetadata::SetQuery(const data_size_t* query_boundaries, const label_t* query_weights, data_size_t num_queries) {
cuda_query_boundaries_.InitFromHostMemory(query_boundaries, static_cast<size_t>(num_queries) + 1);
if (query_weights != nullptr) {
cuda_query_weights_.InitFromHostMemory(query_weights, static_cast<size_t>(num_queries));
}
}
void CUDAMetadata::SetInitScore(const double* init_score, data_size_t len) {
cuda_init_score_.InitFromHostMemory(init_score, len);
}
} // namespace LightGBM
#endif // USE_CUDA
src/io/cuda/cuda_row_data.cpp
+438
View File
@@ -0,0 +1,438 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_row_data.hpp>
#include <vector>
namespace LightGBM {
CUDARowData::CUDARowData(const Dataset* train_data,
const TrainingShareStates* train_share_state,
const int gpu_device_id,
const bool gpu_use_dp):
gpu_device_id_(gpu_device_id),
gpu_use_dp_(gpu_use_dp) {
num_threads_ = OMP_NUM_THREADS();
num_data_ = train_data->num_data();
const auto& feature_hist_offsets = train_share_state->feature_hist_offsets();
if (gpu_use_dp_) {
shared_hist_size_ = DP_SHARED_HIST_SIZE;
} else {
shared_hist_size_ = SP_SHARED_HIST_SIZE;
}
if (feature_hist_offsets.empty()) {
num_total_bin_ = 0;
} else {
num_total_bin_ = static_cast<int>(feature_hist_offsets.back());
}
num_feature_group_ = train_data->num_feature_groups();
num_feature_ = train_data->num_features();
if (gpu_device_id >= 0) {
SetCUDADevice(gpu_device_id, __FILE__, __LINE__);
} else {
SetCUDADevice(0, __FILE__, __LINE__);
}
}
CUDARowData::~CUDARowData() {}
void CUDARowData::Init(const Dataset* train_data, TrainingShareStates* train_share_state) {
if (num_feature_ == 0) {
return;
}
DivideCUDAFeatureGroups(train_data, train_share_state);
bit_type_ = 0;
size_t total_size = 0;
const void* host_row_ptr = nullptr;
row_ptr_bit_type_ = 0;
const void* host_data = train_share_state->GetRowWiseData(&bit_type_, &total_size, &is_sparse_, &host_row_ptr, &row_ptr_bit_type_);
if (bit_type_ == 8) {
if (!is_sparse_) {
std::vector<uint8_t> partitioned_data;
GetDenseDataPartitioned<uint8_t>(reinterpret_cast<const uint8_t*>(host_data), &partitioned_data);
cuda_data_uint8_t_.InitFromHostVector(partitioned_data);
} else {
if (row_ptr_bit_type_ == 16) {
InitSparseData<uint8_t, uint16_t>(
reinterpret_cast<const uint8_t*>(host_data),
reinterpret_cast<const uint16_t*>(host_row_ptr),
&cuda_data_uint8_t_,
&cuda_row_ptr_uint16_t_,
&cuda_partition_ptr_uint16_t_);
} else if (row_ptr_bit_type_ == 32) {
InitSparseData<uint8_t, uint32_t>(
reinterpret_cast<const uint8_t*>(host_data),
reinterpret_cast<const uint32_t*>(host_row_ptr),
&cuda_data_uint8_t_,
&cuda_row_ptr_uint32_t_,
&cuda_partition_ptr_uint32_t_);
} else if (row_ptr_bit_type_ == 64) {
InitSparseData<uint8_t, uint64_t>(
reinterpret_cast<const uint8_t*>(host_data),
reinterpret_cast<const uint64_t*>(host_row_ptr),
&cuda_data_uint8_t_,
&cuda_row_ptr_uint64_t_,
&cuda_partition_ptr_uint64_t_);
} else {
Log::Fatal("Unknown data ptr bit type %d", row_ptr_bit_type_);
}
}
} else if (bit_type_ == 16) {
if (!is_sparse_) {
std::vector<uint16_t> partitioned_data;
GetDenseDataPartitioned<uint16_t>(reinterpret_cast<const uint16_t*>(host_data), &partitioned_data);
cuda_data_uint16_t_.InitFromHostVector(partitioned_data);
} else {
if (row_ptr_bit_type_ == 16) {
InitSparseData<uint16_t, uint16_t>(
reinterpret_cast<const uint16_t*>(host_data),
reinterpret_cast<const uint16_t*>(host_row_ptr),
&cuda_data_uint16_t_,
&cuda_row_ptr_uint16_t_,
&cuda_partition_ptr_uint16_t_);
} else if (row_ptr_bit_type_ == 32) {
InitSparseData<uint16_t, uint32_t>(
reinterpret_cast<const uint16_t*>(host_data),
reinterpret_cast<const uint32_t*>(host_row_ptr),
&cuda_data_uint16_t_,
&cuda_row_ptr_uint32_t_,
&cuda_partition_ptr_uint32_t_);
} else if (row_ptr_bit_type_ == 64) {
InitSparseData<uint16_t, uint64_t>(
reinterpret_cast<const uint16_t*>(host_data),
reinterpret_cast<const uint64_t*>(host_row_ptr),
&cuda_data_uint16_t_,
&cuda_row_ptr_uint64_t_,
&cuda_partition_ptr_uint64_t_);
} else {
Log::Fatal("Unknown data ptr bit type %d", row_ptr_bit_type_);
}
}
} else if (bit_type_ == 32) {
if (!is_sparse_) {
std::vector<uint32_t> partitioned_data;
GetDenseDataPartitioned<uint32_t>(reinterpret_cast<const uint32_t*>(host_data), &partitioned_data);
cuda_data_uint32_t_.InitFromHostVector(partitioned_data);
} else {
if (row_ptr_bit_type_ == 16) {
InitSparseData<uint32_t, uint16_t>(
reinterpret_cast<const uint32_t*>(host_data),
reinterpret_cast<const uint16_t*>(host_row_ptr),
&cuda_data_uint32_t_,
&cuda_row_ptr_uint16_t_,
&cuda_partition_ptr_uint16_t_);
} else if (row_ptr_bit_type_ == 32) {
InitSparseData<uint32_t, uint32_t>(
reinterpret_cast<const uint32_t*>(host_data),
reinterpret_cast<const uint32_t*>(host_row_ptr),
&cuda_data_uint32_t_,
&cuda_row_ptr_uint32_t_,
&cuda_partition_ptr_uint32_t_);
} else if (row_ptr_bit_type_ == 64) {
InitSparseData<uint32_t, uint64_t>(
reinterpret_cast<const uint32_t*>(host_data),
reinterpret_cast<const uint64_t*>(host_row_ptr),
&cuda_data_uint32_t_,
&cuda_row_ptr_uint64_t_,
&cuda_partition_ptr_uint64_t_);
} else {
Log::Fatal("Unknown data ptr bit type %d", row_ptr_bit_type_);
}
}
} else {
Log::Fatal("Unknown bit type = %d", bit_type_);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDARowData::DivideCUDAFeatureGroups(const Dataset* train_data, TrainingShareStates* share_state) {
const uint32_t max_num_bin_per_partition = shared_hist_size_ / 2;
const std::vector<uint32_t>& column_hist_offsets = share_state->column_hist_offsets();
std::vector<int> feature_group_num_feature_offsets;
int offsets = 0;
int prev_group_index = -1;
for (int feature_index = 0; feature_index < num_feature_; ++feature_index) {
const int feature_group_index = train_data->Feature2Group(feature_index);
if (prev_group_index == -1 || feature_group_index != prev_group_index) {
feature_group_num_feature_offsets.emplace_back(offsets);
prev_group_index = feature_group_index;
}
++offsets;
}
CHECK_EQ(offsets, num_feature_);
feature_group_num_feature_offsets.emplace_back(offsets);
uint32_t start_hist_offset = 0;
feature_partition_column_index_offsets_.clear();
column_hist_offsets_.clear();
partition_hist_offsets_.clear();
feature_partition_column_index_offsets_.emplace_back(0);
partition_hist_offsets_.emplace_back(0);
const int num_feature_groups = train_data->num_feature_groups();
int column_index = 0;
num_feature_partitions_ = 0;
large_bin_partitions_.clear();
small_bin_partitions_.clear();
for (int feature_group_index = 0; feature_group_index < num_feature_groups; ++feature_group_index) {
if (!train_data->IsMultiGroup(feature_group_index)) {
const uint32_t column_feature_hist_start = column_hist_offsets[column_index];
const uint32_t column_feature_hist_end = column_hist_offsets[column_index + 1];
const uint32_t num_bin_in_dense_group = column_feature_hist_end - column_feature_hist_start;
// if one column has too many bins, use a separate partition for that column
if (num_bin_in_dense_group > max_num_bin_per_partition) {
feature_partition_column_index_offsets_.emplace_back(column_index + 1);
start_hist_offset = column_feature_hist_end;
partition_hist_offsets_.emplace_back(start_hist_offset);
large_bin_partitions_.emplace_back(num_feature_partitions_);
++num_feature_partitions_;
column_hist_offsets_.emplace_back(0);
++column_index;
continue;
}
// try if adding this column exceed the maximum number per partition
const uint32_t cur_hist_num_bin = column_feature_hist_end - start_hist_offset;
if (cur_hist_num_bin > max_num_bin_per_partition) {
feature_partition_column_index_offsets_.emplace_back(column_index);
start_hist_offset = column_feature_hist_start;
partition_hist_offsets_.emplace_back(start_hist_offset);
small_bin_partitions_.emplace_back(num_feature_partitions_);
++num_feature_partitions_;
}
column_hist_offsets_.emplace_back(column_hist_offsets[column_index] - start_hist_offset);
if (feature_group_index == num_feature_groups - 1) {
feature_partition_column_index_offsets_.emplace_back(column_index + 1);
partition_hist_offsets_.emplace_back(column_hist_offsets.back());
small_bin_partitions_.emplace_back(num_feature_partitions_);
++num_feature_partitions_;
}
++column_index;
} else {
const int group_feature_index_start = feature_group_num_feature_offsets[feature_group_index];
const int num_feature_in_group = feature_group_num_feature_offsets[feature_group_index + 1] - group_feature_index_start;
for (int sub_feature_index = 0; sub_feature_index < num_feature_in_group; ++sub_feature_index) {
const int feature_index = group_feature_index_start + sub_feature_index;
const uint32_t column_feature_hist_start = column_hist_offsets[column_index];
const uint32_t column_feature_hist_end = column_hist_offsets[column_index + 1];
const uint32_t num_bin_in_dense_group = column_feature_hist_end - column_feature_hist_start;
// if one column has too many bins, use a separate partition for that column
if (num_bin_in_dense_group > max_num_bin_per_partition) {
feature_partition_column_index_offsets_.emplace_back(column_index + 1);
start_hist_offset = column_feature_hist_end;
partition_hist_offsets_.emplace_back(start_hist_offset);
large_bin_partitions_.emplace_back(num_feature_partitions_);
++num_feature_partitions_;
column_hist_offsets_.emplace_back(0);
++column_index;
continue;
}
// try if adding this column exceed the maximum number per partition
const uint32_t cur_hist_num_bin = column_feature_hist_end - start_hist_offset;
if (cur_hist_num_bin > max_num_bin_per_partition) {
feature_partition_column_index_offsets_.emplace_back(column_index);
start_hist_offset = column_feature_hist_start;
partition_hist_offsets_.emplace_back(start_hist_offset);
small_bin_partitions_.emplace_back(num_feature_partitions_);
++num_feature_partitions_;
}
column_hist_offsets_.emplace_back(column_hist_offsets[column_index] - start_hist_offset);
if (feature_group_index == num_feature_groups - 1 && sub_feature_index == num_feature_in_group - 1) {
CHECK_EQ(feature_index, num_feature_ - 1);
feature_partition_column_index_offsets_.emplace_back(column_index + 1);
partition_hist_offsets_.emplace_back(column_hist_offsets.back());
small_bin_partitions_.emplace_back(num_feature_partitions_);
++num_feature_partitions_;
}
++column_index;
}
}
}
column_hist_offsets_.emplace_back(column_hist_offsets.back() - start_hist_offset);
max_num_column_per_partition_ = 0;
for (size_t i = 0; i < feature_partition_column_index_offsets_.size() - 1; ++i) {
const int num_column = feature_partition_column_index_offsets_[i + 1] - feature_partition_column_index_offsets_[i];
if (num_column > max_num_column_per_partition_) {
max_num_column_per_partition_ = num_column;
}
}
cuda_feature_partition_column_index_offsets_.InitFromHostVector(feature_partition_column_index_offsets_);
cuda_column_hist_offsets_.InitFromHostVector(column_hist_offsets_);
cuda_partition_hist_offsets_.InitFromHostVector(partition_hist_offsets_);
}
template <typename BIN_TYPE>
void CUDARowData::GetDenseDataPartitioned(const BIN_TYPE* row_wise_data, std::vector<BIN_TYPE>* partitioned_data) {
const int num_total_columns = feature_partition_column_index_offsets_.back();
partitioned_data->resize(static_cast<size_t>(num_total_columns) * static_cast<size_t>(num_data_), 0);
BIN_TYPE* out_data = partitioned_data->data();
Threading::For<data_size_t>(0, num_data_, 512,
[this, num_total_columns, row_wise_data, out_data] (int /*thread_index*/, data_size_t start, data_size_t end) {
for (size_t i = 0; i < feature_partition_column_index_offsets_.size() - 1; ++i) {
const int num_prev_columns = static_cast<int>(feature_partition_column_index_offsets_[i]);
const size_t offset = static_cast<size_t>(num_data_) * static_cast<size_t>(num_prev_columns);
const int partition_column_start = feature_partition_column_index_offsets_[i];
const int partition_column_end = feature_partition_column_index_offsets_[i + 1];
const int num_columns_in_cur_partition = partition_column_end - partition_column_start;
for (data_size_t data_index = start; data_index < end; ++data_index) {
const size_t data_offset = offset + static_cast<size_t>(data_index) * num_columns_in_cur_partition;
const size_t read_data_offset = static_cast<size_t>(data_index) * num_total_columns;
for (int column_index = 0; column_index < num_columns_in_cur_partition; ++column_index) {
const size_t true_column_index = read_data_offset + column_index + partition_column_start;
const BIN_TYPE bin = row_wise_data[true_column_index];
out_data[data_offset + column_index] = bin;
}
}
}
});
}
template <typename BIN_TYPE, typename DATA_PTR_TYPE>
void CUDARowData::GetSparseDataPartitioned(
const BIN_TYPE* row_wise_data,
const DATA_PTR_TYPE* row_ptr,
std::vector<std::vector<BIN_TYPE>>* partitioned_data,
std::vector<std::vector<DATA_PTR_TYPE>>* partitioned_row_ptr,
std::vector<DATA_PTR_TYPE>* partition_ptr) {
const int num_partitions = static_cast<int>(feature_partition_column_index_offsets_.size()) - 1;
partitioned_data->resize(num_partitions);
partitioned_row_ptr->resize(num_partitions);
std::vector<int> thread_max_elements_per_row(num_threads_, 0);
Threading::For<int>(0, num_partitions, 1,
[partitioned_data, partitioned_row_ptr, row_ptr, row_wise_data, &thread_max_elements_per_row, this] (int thread_index, int start, int end) {
for (int partition_index = start; partition_index < end; ++partition_index) {
std::vector<BIN_TYPE>& data_for_this_partition = partitioned_data->at(partition_index);
std::vector<DATA_PTR_TYPE>& row_ptr_for_this_partition = partitioned_row_ptr->at(partition_index);
const int partition_hist_start = partition_hist_offsets_[partition_index];
const int partition_hist_end = partition_hist_offsets_[partition_index + 1];
DATA_PTR_TYPE offset = 0;
row_ptr_for_this_partition.clear();
data_for_this_partition.clear();
row_ptr_for_this_partition.emplace_back(offset);
for (data_size_t data_index = 0; data_index < num_data_; ++data_index) {
const DATA_PTR_TYPE row_start = row_ptr[data_index];
const DATA_PTR_TYPE row_end = row_ptr[data_index + 1];
const BIN_TYPE* row_data_start = row_wise_data + row_start;
const BIN_TYPE* row_data_end = row_wise_data + row_end;
const size_t partition_start_in_row = std::lower_bound(row_data_start, row_data_end, partition_hist_start) - row_data_start;
const size_t partition_end_in_row = std::lower_bound(row_data_start, row_data_end, partition_hist_end) - row_data_start;
for (size_t pos = partition_start_in_row; pos < partition_end_in_row; ++pos) {
const BIN_TYPE bin = row_data_start[pos];
CHECK_GE(bin, static_cast<BIN_TYPE>(partition_hist_start));
data_for_this_partition.emplace_back(bin - partition_hist_start);
}
CHECK_GE(partition_end_in_row, partition_start_in_row);
const data_size_t num_elements_in_row = partition_end_in_row - partition_start_in_row;
offset += static_cast<DATA_PTR_TYPE>(num_elements_in_row);
row_ptr_for_this_partition.emplace_back(offset);
if (num_elements_in_row > thread_max_elements_per_row[thread_index]) {
thread_max_elements_per_row[thread_index] = num_elements_in_row;
}
}
}
});
partition_ptr->clear();
DATA_PTR_TYPE offset = 0;
partition_ptr->emplace_back(offset);
for (size_t i = 0; i < partitioned_row_ptr->size(); ++i) {
offset += partitioned_row_ptr->at(i).back();
partition_ptr->emplace_back(offset);
}
max_num_column_per_partition_ = 0;
for (int thread_index = 0; thread_index < num_threads_; ++thread_index) {
if (thread_max_elements_per_row[thread_index] > max_num_column_per_partition_) {
max_num_column_per_partition_ = thread_max_elements_per_row[thread_index];
}
}
}
template <typename BIN_TYPE, typename ROW_PTR_TYPE>
void CUDARowData::InitSparseData(const BIN_TYPE* host_data,
const ROW_PTR_TYPE* host_row_ptr,
CUDAVector<BIN_TYPE>* cuda_data,
CUDAVector<ROW_PTR_TYPE>* cuda_row_ptr,
CUDAVector<ROW_PTR_TYPE>* cuda_partition_ptr) {
std::vector<std::vector<BIN_TYPE>> partitioned_data;
std::vector<std::vector<ROW_PTR_TYPE>> partitioned_data_ptr;
std::vector<ROW_PTR_TYPE> partition_ptr;
GetSparseDataPartitioned<BIN_TYPE, ROW_PTR_TYPE>(host_data, host_row_ptr, &partitioned_data, &partitioned_data_ptr, &partition_ptr);
cuda_partition_ptr->InitFromHostVector(partition_ptr);
cuda_data->Resize(partition_ptr.back());
cuda_row_ptr->Resize((num_data_ + 1) * partitioned_data_ptr.size());
for (size_t i = 0; i < partitioned_data.size(); ++i) {
const std::vector<ROW_PTR_TYPE>& data_ptr_for_this_partition = partitioned_data_ptr[i];
const std::vector<BIN_TYPE>& data_for_this_partition = partitioned_data[i];
CopyFromHostToCUDADevice<BIN_TYPE>(cuda_data->RawData() + partition_ptr[i], data_for_this_partition.data(), data_for_this_partition.size(), __FILE__, __LINE__);
CopyFromHostToCUDADevice<ROW_PTR_TYPE>(cuda_row_ptr->RawData() + i * (num_data_ + 1), data_ptr_for_this_partition.data(), data_ptr_for_this_partition.size(), __FILE__, __LINE__);
}
}
template <typename BIN_TYPE>
const BIN_TYPE* CUDARowData::GetBin() const {
if (bit_type_ == 8) {
return reinterpret_cast<const BIN_TYPE*>(cuda_data_uint8_t_.RawData());
} else if (bit_type_ == 16) {
return reinterpret_cast<const BIN_TYPE*>(cuda_data_uint16_t_.RawData());
} else if (bit_type_ == 32) {
return reinterpret_cast<const BIN_TYPE*>(cuda_data_uint32_t_.RawData());
} else {
Log::Fatal("Unknown bit_type %d for GetBin.", bit_type_);
}
}
template const uint8_t* CUDARowData::GetBin<uint8_t>() const;
template const uint16_t* CUDARowData::GetBin<uint16_t>() const;
template const uint32_t* CUDARowData::GetBin<uint32_t>() const;
template <typename PTR_TYPE>
const PTR_TYPE* CUDARowData::GetRowPtr() const {
if (row_ptr_bit_type_ == 16) {
return reinterpret_cast<const PTR_TYPE*>(cuda_row_ptr_uint16_t_.RawData());
} else if (row_ptr_bit_type_ == 32) {
return reinterpret_cast<const PTR_TYPE*>(cuda_row_ptr_uint32_t_.RawData());
} else if (row_ptr_bit_type_ == 64) {
return reinterpret_cast<const PTR_TYPE*>(cuda_row_ptr_uint64_t_.RawData());
} else {
Log::Fatal("Unknown row_ptr_bit_type = %d for GetRowPtr.", row_ptr_bit_type_);
}
}
template const uint16_t* CUDARowData::GetRowPtr<uint16_t>() const;
template const uint32_t* CUDARowData::GetRowPtr<uint32_t>() const;
template const uint64_t* CUDARowData::GetRowPtr<uint64_t>() const;
template <typename PTR_TYPE>
const PTR_TYPE* CUDARowData::GetPartitionPtr() const {
if (row_ptr_bit_type_ == 16) {
return reinterpret_cast<const PTR_TYPE*>(cuda_partition_ptr_uint16_t_.RawData());
} else if (row_ptr_bit_type_ == 32) {
return reinterpret_cast<const PTR_TYPE*>(cuda_partition_ptr_uint32_t_.RawData());
} else if (row_ptr_bit_type_ == 64) {
return reinterpret_cast<const PTR_TYPE*>(cuda_partition_ptr_uint64_t_.RawData());
} else {
Log::Fatal("Unknown row_ptr_bit_type = %d for GetPartitionPtr.", row_ptr_bit_type_);
}
}
template const uint16_t* CUDARowData::GetPartitionPtr<uint16_t>() const;
template const uint32_t* CUDARowData::GetPartitionPtr<uint32_t>() const;
template const uint64_t* CUDARowData::GetPartitionPtr<uint64_t>() const;
} // namespace LightGBM
#endif // USE_CUDA
src/io/cuda/cuda_tree.cpp
+214
View File
@@ -0,0 +1,214 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_tree.hpp>
namespace LightGBM {
CUDATree::CUDATree(int max_leaves, bool track_branch_features, bool is_linear,
const int gpu_device_id, const bool has_categorical_feature):
Tree(max_leaves, track_branch_features, is_linear),
num_threads_per_block_add_prediction_to_score_(1024) {
is_cuda_tree_ = true;
if (gpu_device_id >= 0) {
SetCUDADevice(gpu_device_id, __FILE__, __LINE__);
} else {
SetCUDADevice(0, __FILE__, __LINE__);
}
if (has_categorical_feature) {
cuda_cat_boundaries_.Resize(max_leaves);
cuda_cat_boundaries_inner_.Resize(max_leaves);
}
InitCUDAMemory();
}
CUDATree::CUDATree(const Tree* host_tree):
Tree(*host_tree),
num_threads_per_block_add_prediction_to_score_(1024) {
is_cuda_tree_ = true;
InitCUDA();
}
CUDATree::~CUDATree() {
gpuAssert(cudaStreamDestroy(cuda_stream_), __FILE__, __LINE__);
}
void CUDATree::InitCUDAMemory() {
cuda_left_child_.Resize(static_cast<size_t>(max_leaves_));
cuda_right_child_.Resize(static_cast<size_t>(max_leaves_));
cuda_split_feature_inner_.Resize(static_cast<size_t>(max_leaves_));
cuda_split_feature_.Resize(static_cast<size_t>(max_leaves_));
cuda_leaf_depth_.Resize(static_cast<size_t>(max_leaves_));
cuda_leaf_parent_.Resize(static_cast<size_t>(max_leaves_));
cuda_threshold_in_bin_.Resize(static_cast<size_t>(max_leaves_));
cuda_threshold_.Resize(static_cast<size_t>(max_leaves_));
cuda_decision_type_.Resize(static_cast<size_t>(max_leaves_));
cuda_leaf_value_.Resize(static_cast<size_t>(max_leaves_));
cuda_internal_weight_.Resize(static_cast<size_t>(max_leaves_));
cuda_internal_value_.Resize(static_cast<size_t>(max_leaves_));
cuda_leaf_weight_.Resize(static_cast<size_t>(max_leaves_));
cuda_leaf_count_.Resize(static_cast<size_t>(max_leaves_));
cuda_internal_count_.Resize(static_cast<size_t>(max_leaves_));
cuda_split_gain_.Resize(static_cast<size_t>(max_leaves_));
SetCUDAMemory<double>(cuda_leaf_value_.RawData(), 0.0f, 1, __FILE__, __LINE__);
SetCUDAMemory<double>(cuda_leaf_weight_.RawData(), 0.0f, 1, __FILE__, __LINE__);
SetCUDAMemory<int>(cuda_leaf_parent_.RawData(), -1, 1, __FILE__, __LINE__);
CUDASUCCESS_OR_FATAL(cudaStreamCreate(&cuda_stream_));
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDATree::InitCUDA() {
cuda_left_child_.InitFromHostVector(left_child_);
cuda_right_child_.InitFromHostVector(right_child_);
cuda_split_feature_inner_.InitFromHostVector(split_feature_inner_);
cuda_split_feature_.InitFromHostVector(split_feature_);
cuda_threshold_in_bin_.InitFromHostVector(threshold_in_bin_);
cuda_threshold_.InitFromHostVector(threshold_);
cuda_leaf_depth_.InitFromHostVector(leaf_depth_);
cuda_decision_type_.InitFromHostVector(decision_type_);
cuda_internal_weight_.InitFromHostVector(internal_weight_);
cuda_internal_value_.InitFromHostVector(internal_value_);
cuda_internal_count_.InitFromHostVector(internal_count_);
cuda_leaf_count_.InitFromHostVector(leaf_count_);
cuda_split_gain_.InitFromHostVector(split_gain_);
cuda_leaf_value_.InitFromHostVector(leaf_value_);
cuda_leaf_weight_.InitFromHostVector(leaf_weight_);
cuda_leaf_parent_.InitFromHostVector(leaf_parent_);
CUDASUCCESS_OR_FATAL(cudaStreamCreate(&cuda_stream_));
SynchronizeCUDADevice(__FILE__, __LINE__);
}
int CUDATree::Split(const int leaf_index,
const int real_feature_index,
const double real_threshold,
const MissingType missing_type,
const CUDASplitInfo* cuda_split_info) {
LaunchSplitKernel(leaf_index, real_feature_index, real_threshold, missing_type, cuda_split_info);
RecordBranchFeatures(leaf_index, num_leaves_, real_feature_index);
++num_leaves_;
return num_leaves_ - 1;
}
int CUDATree::SplitCategorical(const int leaf_index,
const int real_feature_index,
const MissingType missing_type,
const CUDASplitInfo* cuda_split_info,
uint32_t* cuda_bitset,
size_t cuda_bitset_len,
uint32_t* cuda_bitset_inner,
size_t cuda_bitset_inner_len) {
LaunchSplitCategoricalKernel(leaf_index, real_feature_index,
missing_type, cuda_split_info,
cuda_bitset_len, cuda_bitset_inner_len);
cuda_bitset_.PushBack(cuda_bitset, cuda_bitset_len);
cuda_bitset_inner_.PushBack(cuda_bitset_inner, cuda_bitset_inner_len);
++num_leaves_;
++num_cat_;
RecordBranchFeatures(leaf_index, num_leaves_, real_feature_index);
return num_leaves_ - 1;
}
void CUDATree::RecordBranchFeatures(const int left_leaf_index,
const int right_leaf_index,
const int real_feature_index) {
if (track_branch_features_) {
branch_features_[right_leaf_index] = branch_features_[left_leaf_index];
branch_features_[right_leaf_index].push_back(real_feature_index);
branch_features_[left_leaf_index].push_back(real_feature_index);
}
}
void CUDATree::AddPredictionToScore(const Dataset* data,
data_size_t num_data,
double* score) const {
LaunchAddPredictionToScoreKernel(data, nullptr, num_data, score);
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDATree::AddPredictionToScore(const Dataset* data,
const data_size_t* used_data_indices,
data_size_t num_data, double* score) const {
LaunchAddPredictionToScoreKernel(data, used_data_indices, num_data, score);
SynchronizeCUDADevice(__FILE__, __LINE__);
}
inline void CUDATree::Shrinkage(double rate) {
Tree::Shrinkage(rate);
LaunchShrinkageKernel(rate);
}
inline void CUDATree::AddBias(double val) {
Tree::AddBias(val);
LaunchAddBiasKernel(val);
}
void CUDATree::ToHost() {
left_child_.resize(max_leaves_ - 1);
right_child_.resize(max_leaves_ - 1);
split_feature_inner_.resize(max_leaves_ - 1);
split_feature_.resize(max_leaves_ - 1);
threshold_in_bin_.resize(max_leaves_ - 1);
threshold_.resize(max_leaves_ - 1);
decision_type_.resize(max_leaves_ - 1, 0);
split_gain_.resize(max_leaves_ - 1);
leaf_parent_.resize(max_leaves_);
leaf_value_.resize(max_leaves_);
leaf_weight_.resize(max_leaves_);
leaf_count_.resize(max_leaves_);
internal_value_.resize(max_leaves_ - 1);
internal_weight_.resize(max_leaves_ - 1);
internal_count_.resize(max_leaves_ - 1);
leaf_depth_.resize(max_leaves_);
const size_t num_leaves_size = static_cast<size_t>(num_leaves_);
CopyFromCUDADeviceToHost<int>(left_child_.data(), cuda_left_child_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<int>(right_child_.data(), cuda_right_child_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<int>(split_feature_inner_.data(), cuda_split_feature_inner_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<int>(split_feature_.data(), cuda_split_feature_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<uint32_t>(threshold_in_bin_.data(), cuda_threshold_in_bin_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<double>(threshold_.data(), cuda_threshold_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<int8_t>(decision_type_.data(), cuda_decision_type_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<float>(split_gain_.data(), cuda_split_gain_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<int>(leaf_parent_.data(), cuda_leaf_parent_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<double>(leaf_value_.data(), cuda_leaf_value_.RawData(), num_leaves_size, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<double>(leaf_weight_.data(), cuda_leaf_weight_.RawData(), num_leaves_size, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<data_size_t>(leaf_count_.data(), cuda_leaf_count_.RawData(), num_leaves_size, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<double>(internal_value_.data(), cuda_internal_value_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<double>(internal_weight_.data(), cuda_internal_weight_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<data_size_t>(internal_count_.data(), cuda_internal_count_.RawData(), num_leaves_size - 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<int>(leaf_depth_.data(), cuda_leaf_depth_.RawData(), num_leaves_size, __FILE__, __LINE__);
if (num_cat_ > 0) {
cuda_cat_boundaries_inner_.Resize(num_cat_ + 1);
cuda_cat_boundaries_.Resize(num_cat_ + 1);
cat_boundaries_ = cuda_cat_boundaries_.ToHost();
cat_boundaries_inner_ = cuda_cat_boundaries_inner_.ToHost();
cat_threshold_ = cuda_bitset_.ToHost();
cat_threshold_inner_ = cuda_bitset_inner_.ToHost();
}
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDATree::SyncLeafOutputFromHostToCUDA() {
CopyFromHostToCUDADevice<double>(cuda_leaf_value_.RawData(), leaf_value_.data(), leaf_value_.size(), __FILE__, __LINE__);
}
void CUDATree::SyncLeafOutputFromCUDAToHost() {
CopyFromCUDADeviceToHost<double>(leaf_value_.data(), cuda_leaf_value_.RawData(), leaf_value_.size(), __FILE__, __LINE__);
}
void CUDATree::AsConstantTree(double val, int count) {
Tree::AsConstantTree(val, count);
CopyFromHostToCUDADevice<double>(cuda_leaf_value_.RawData(), &val, 1, __FILE__, __LINE__);
CopyFromHostToCUDADevice<int>(cuda_leaf_count_.RawData(), &count, 1, __FILE__, __LINE__);
}
} // namespace LightGBM
#endif // USE_CUDA
src/io/cuda/cuda_tree.cu
+460
View File
@@ -0,0 +1,460 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_tree.hpp>
namespace LightGBM {
__device__ void SetDecisionTypeCUDA(int8_t* decision_type, bool input, int8_t mask) {
if (input) {
(*decision_type) |= mask;
} else {
(*decision_type) &= (127 - mask);
}
}
__device__ void SetMissingTypeCUDA(int8_t* decision_type, int8_t input) {
(*decision_type) &= 3;
(*decision_type) |= (input << 2);
}
__device__ bool GetDecisionTypeCUDA(int8_t decision_type, int8_t mask) {
return (decision_type & mask) > 0;
}
__device__ int8_t GetMissingTypeCUDA(int8_t decision_type) {
return (decision_type >> 2) & 3;
}
__device__ bool IsZeroCUDA(double fval) {
return (fval >= -kZeroThreshold && fval <= kZeroThreshold);
}
template<typename T>
__device__ bool FindInBitsetCUDA(const uint32_t* bits, int n, T pos) {
int i1 = pos / 32;
if (i1 >= n) {
return false;
}
int i2 = pos % 32;
return (bits[i1] >> i2) & 1;
}
__global__ void SplitKernel( // split information
const int leaf_index,
const int real_feature_index,
const double real_threshold,
const MissingType missing_type,
const CUDASplitInfo* cuda_split_info,
// tree structure
const int num_leaves,
int* leaf_parent,
int* leaf_depth,
int* left_child,
int* right_child,
int* split_feature_inner,
int* split_feature,
float* split_gain,
double* internal_weight,
double* internal_value,
data_size_t* internal_count,
double* leaf_weight,
double* leaf_value,
data_size_t* leaf_count,
int8_t* decision_type,
uint32_t* threshold_in_bin,
double* threshold) {
const int new_node_index = num_leaves - 1;
const int thread_index = static_cast<int>(threadIdx.x + blockIdx.x * blockDim.x);
const int parent_index = leaf_parent[leaf_index];
if (thread_index == 0) {
if (parent_index >= 0) {
// if cur node is left child
if (left_child[parent_index] == ~leaf_index) {
left_child[parent_index] = new_node_index;
} else {
right_child[parent_index] = new_node_index;
}
}
left_child[new_node_index] = ~leaf_index;
right_child[new_node_index] = ~num_leaves;
leaf_parent[leaf_index] = new_node_index;
leaf_parent[num_leaves] = new_node_index;
} else if (thread_index == 1) {
// add new node
split_feature_inner[new_node_index] = cuda_split_info->inner_feature_index;
} else if (thread_index == 2) {
split_feature[new_node_index] = real_feature_index;
} else if (thread_index == 3) {
split_gain[new_node_index] = static_cast<float>(cuda_split_info->gain);
} else if (thread_index == 4) {
// save current leaf value to internal node before change
internal_weight[new_node_index] = cuda_split_info->left_sum_hessians + cuda_split_info->right_sum_hessians;
leaf_weight[leaf_index] = cuda_split_info->left_sum_hessians;
} else if (thread_index == 5) {
internal_value[new_node_index] = leaf_value[leaf_index];
leaf_value[leaf_index] = isnan(cuda_split_info->left_value) ? 0.0f : cuda_split_info->left_value;
} else if (thread_index == 6) {
internal_count[new_node_index] = cuda_split_info->left_count + cuda_split_info->right_count;
} else if (thread_index == 7) {
leaf_count[leaf_index] = cuda_split_info->left_count;
} else if (thread_index == 8) {
leaf_value[num_leaves] = isnan(cuda_split_info->right_value) ? 0.0f : cuda_split_info->right_value;
} else if (thread_index == 9) {
leaf_weight[num_leaves] = cuda_split_info->right_sum_hessians;
} else if (thread_index == 10) {
leaf_count[num_leaves] = cuda_split_info->right_count;
} else if (thread_index == 11) {
// update leaf depth
leaf_depth[num_leaves] = leaf_depth[leaf_index] + 1;
leaf_depth[leaf_index]++;
} else if (thread_index == 12) {
decision_type[new_node_index] = 0;
SetDecisionTypeCUDA(&decision_type[new_node_index], false, kCategoricalMask);
SetDecisionTypeCUDA(&decision_type[new_node_index], cuda_split_info->default_left, kDefaultLeftMask);
SetMissingTypeCUDA(&decision_type[new_node_index], static_cast<int8_t>(missing_type));
} else if (thread_index == 13) {
threshold_in_bin[new_node_index] = cuda_split_info->threshold;
} else if (thread_index == 14) {
threshold[new_node_index] = real_threshold;
}
}
void CUDATree::LaunchSplitKernel(const int leaf_index,
const int real_feature_index,
const double real_threshold,
const MissingType missing_type,
const CUDASplitInfo* cuda_split_info) {
SplitKernel<<<3, 5, 0, cuda_stream_>>>(
// split information
leaf_index,
real_feature_index,
real_threshold,
missing_type,
cuda_split_info,
// tree structure
num_leaves_,
cuda_leaf_parent_.RawData(),
cuda_leaf_depth_.RawData(),
cuda_left_child_.RawData(),
cuda_right_child_.RawData(),
cuda_split_feature_inner_.RawData(),
cuda_split_feature_.RawData(),
cuda_split_gain_.RawData(),
cuda_internal_weight_.RawData(),
cuda_internal_value_.RawData(),
cuda_internal_count_.RawData(),
cuda_leaf_weight_.RawData(),
cuda_leaf_value_.RawData(),
cuda_leaf_count_.RawData(),
cuda_decision_type_.RawData(),
cuda_threshold_in_bin_.RawData(),
cuda_threshold_.RawData());
}
__global__ void SplitCategoricalKernel( // split information
const int leaf_index,
const int real_feature_index,
const MissingType missing_type,
const CUDASplitInfo* cuda_split_info,
// tree structure
const int num_leaves,
int* leaf_parent,
int* leaf_depth,
int* left_child,
int* right_child,
int* split_feature_inner,
int* split_feature,
float* split_gain,
double* internal_weight,
double* internal_value,
data_size_t* internal_count,
double* leaf_weight,
double* leaf_value,
data_size_t* leaf_count,
int8_t* decision_type,
uint32_t* threshold_in_bin,
double* threshold,
size_t cuda_bitset_len,
size_t cuda_bitset_inner_len,
int num_cat,
int* cuda_cat_boundaries,
int* cuda_cat_boundaries_inner) {
const int new_node_index = num_leaves - 1;
const int thread_index = static_cast<int>(threadIdx.x + blockIdx.x * blockDim.x);
const int parent_index = leaf_parent[leaf_index];
if (thread_index == 0) {
if (parent_index >= 0) {
// if cur node is left child
if (left_child[parent_index] == ~leaf_index) {
left_child[parent_index] = new_node_index;
} else {
right_child[parent_index] = new_node_index;
}
}
left_child[new_node_index] = ~leaf_index;
right_child[new_node_index] = ~num_leaves;
leaf_parent[leaf_index] = new_node_index;
leaf_parent[num_leaves] = new_node_index;
} else if (thread_index == 1) {
// add new node
split_feature_inner[new_node_index] = cuda_split_info->inner_feature_index;
} else if (thread_index == 2) {
split_feature[new_node_index] = real_feature_index;
} else if (thread_index == 3) {
split_gain[new_node_index] = static_cast<float>(cuda_split_info->gain);
} else if (thread_index == 4) {
// save current leaf value to internal node before change
internal_weight[new_node_index] = cuda_split_info->left_sum_hessians + cuda_split_info->right_sum_hessians;
leaf_weight[leaf_index] = cuda_split_info->left_sum_hessians;
} else if (thread_index == 5) {
internal_value[new_node_index] = leaf_value[leaf_index];
leaf_value[leaf_index] = isnan(cuda_split_info->left_value) ? 0.0f : cuda_split_info->left_value;
} else if (thread_index == 6) {
internal_count[new_node_index] = cuda_split_info->left_count + cuda_split_info->right_count;
} else if (thread_index == 7) {
leaf_count[leaf_index] = cuda_split_info->left_count;
} else if (thread_index == 8) {
leaf_value[num_leaves] = isnan(cuda_split_info->right_value) ? 0.0f : cuda_split_info->right_value;
} else if (thread_index == 9) {
leaf_weight[num_leaves] = cuda_split_info->right_sum_hessians;
} else if (thread_index == 10) {
leaf_count[num_leaves] = cuda_split_info->right_count;
} else if (thread_index == 11) {
// update leaf depth
leaf_depth[num_leaves] = leaf_depth[leaf_index] + 1;
leaf_depth[leaf_index]++;
} else if (thread_index == 12) {
decision_type[new_node_index] = 0;
SetDecisionTypeCUDA(&decision_type[new_node_index], true, kCategoricalMask);
SetMissingTypeCUDA(&decision_type[new_node_index], static_cast<int8_t>(missing_type));
} else if (thread_index == 13) {
threshold_in_bin[new_node_index] = num_cat;
} else if (thread_index == 14) {
threshold[new_node_index] = num_cat;
} else if (thread_index == 15) {
if (num_cat == 0) {
cuda_cat_boundaries[num_cat] = 0;
}
cuda_cat_boundaries[num_cat + 1] = cuda_cat_boundaries[num_cat] + cuda_bitset_len;
} else if (thread_index == 16) {
if (num_cat == 0) {
cuda_cat_boundaries_inner[num_cat] = 0;
}
cuda_cat_boundaries_inner[num_cat + 1] = cuda_cat_boundaries_inner[num_cat] + cuda_bitset_inner_len;
}
}
void CUDATree::LaunchSplitCategoricalKernel(const int leaf_index,
const int real_feature_index,
const MissingType missing_type,
const CUDASplitInfo* cuda_split_info,
size_t cuda_bitset_len,
size_t cuda_bitset_inner_len) {
SplitCategoricalKernel<<<3, 6, 0, cuda_stream_>>>(
// split information
leaf_index,
real_feature_index,
missing_type,
cuda_split_info,
// tree structure
num_leaves_,
cuda_leaf_parent_.RawData(),
cuda_leaf_depth_.RawData(),
cuda_left_child_.RawData(),
cuda_right_child_.RawData(),
cuda_split_feature_inner_.RawData(),
cuda_split_feature_.RawData(),
cuda_split_gain_.RawData(),
cuda_internal_weight_.RawData(),
cuda_internal_value_.RawData(),
cuda_internal_count_.RawData(),
cuda_leaf_weight_.RawData(),
cuda_leaf_value_.RawData(),
cuda_leaf_count_.RawData(),
cuda_decision_type_.RawData(),
cuda_threshold_in_bin_.RawData(),
cuda_threshold_.RawData(),
cuda_bitset_len,
cuda_bitset_inner_len,
num_cat_,
cuda_cat_boundaries_.RawData(),
cuda_cat_boundaries_inner_.RawData());
}
__global__ void ShrinkageKernel(const double rate, double* cuda_leaf_value, const int num_leaves) {
const int leaf_index = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
if (leaf_index < num_leaves) {
cuda_leaf_value[leaf_index] *= rate;
}
}
void CUDATree::LaunchShrinkageKernel(const double rate) {
const int num_threads_per_block = 1024;
const int num_blocks = (num_leaves_ + num_threads_per_block - 1) / num_threads_per_block;
ShrinkageKernel<<<num_blocks, num_threads_per_block>>>(rate, cuda_leaf_value_.RawData(), num_leaves_);
}
__global__ void AddBiasKernel(const double val, double* cuda_leaf_value, const int num_leaves) {
const int leaf_index = static_cast<int>(blockIdx.x * blockDim.x + threadIdx.x);
if (leaf_index < num_leaves) {
cuda_leaf_value[leaf_index] += val;
}
}
void CUDATree::LaunchAddBiasKernel(const double val) {
const int num_threads_per_block = 1024;
const int num_blocks = (num_leaves_ + num_threads_per_block - 1) / num_threads_per_block;
AddBiasKernel<<<num_blocks, num_threads_per_block>>>(val, cuda_leaf_value_.RawData(), num_leaves_);
}
template <bool USE_INDICES>
__global__ void AddPredictionToScoreKernel(
// dataset information
const data_size_t num_data,
uint8_t* const* cuda_data_by_column,
const uint8_t* cuda_column_bit_type,
const uint32_t* cuda_feature_min_bin,
const uint32_t* cuda_feature_max_bin,
const uint32_t* cuda_feature_offset,
const uint32_t* cuda_feature_default_bin,
const uint32_t* cuda_feature_most_freq_bin,
const int* cuda_feature_to_column,
const data_size_t* cuda_used_indices,
// tree information
const uint32_t* cuda_threshold_in_bin,
const int8_t* cuda_decision_type,
const int* cuda_split_feature_inner,
const int* cuda_left_child,
const int* cuda_right_child,
const double* cuda_leaf_value,
const uint32_t* cuda_bitset_inner,
const int* cuda_cat_boundaries_inner,
// output
double* score) {
const data_size_t inner_data_index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
if (inner_data_index < num_data) {
const data_size_t data_index = USE_INDICES ? cuda_used_indices[inner_data_index] : inner_data_index;
int node = 0;
while (node >= 0) {
const int split_feature_inner = cuda_split_feature_inner[node];
const int column = cuda_feature_to_column[split_feature_inner];
const uint32_t default_bin = cuda_feature_default_bin[split_feature_inner];
const uint32_t most_freq_bin = cuda_feature_most_freq_bin[split_feature_inner];
const uint32_t max_bin = cuda_feature_max_bin[split_feature_inner];
const uint32_t min_bin = cuda_feature_min_bin[split_feature_inner];
const uint32_t offset = cuda_feature_offset[split_feature_inner];
const uint8_t column_bit_type = cuda_column_bit_type[column];
uint32_t bin = 0;
if (column_bit_type == 8) {
bin = static_cast<uint32_t>((reinterpret_cast<const uint8_t*>(cuda_data_by_column[column]))[data_index]);
} else if (column_bit_type == 16) {
bin = static_cast<uint32_t>((reinterpret_cast<const uint16_t*>(cuda_data_by_column[column]))[data_index]);
} else if (column_bit_type == 32) {
bin = static_cast<uint32_t>((reinterpret_cast<const uint32_t*>(cuda_data_by_column[column]))[data_index]);
}
if (bin >= min_bin && bin <= max_bin) {
bin = bin - min_bin + offset;
} else {
bin = most_freq_bin;
}
const int8_t decision_type = cuda_decision_type[node];
if (GetDecisionTypeCUDA(decision_type, kCategoricalMask)) {
int cat_idx = static_cast<int>(cuda_threshold_in_bin[node]);
if (FindInBitsetCUDA(cuda_bitset_inner + cuda_cat_boundaries_inner[cat_idx],
cuda_cat_boundaries_inner[cat_idx + 1] - cuda_cat_boundaries_inner[cat_idx], bin)) {
node = cuda_left_child[node];
} else {
node = cuda_right_child[node];
}
} else {
const uint32_t threshold_in_bin = cuda_threshold_in_bin[node];
const int8_t missing_type = GetMissingTypeCUDA(decision_type);
const bool default_left = ((decision_type & kDefaultLeftMask) > 0);
if ((missing_type == 1 && bin == default_bin) || (missing_type == 2 && bin == max_bin)) {
if (default_left) {
node = cuda_left_child[node];
} else {
node = cuda_right_child[node];
}
} else {
if (bin <= threshold_in_bin) {
node = cuda_left_child[node];
} else {
node = cuda_right_child[node];
}
}
}
}
score[data_index] += cuda_leaf_value[~node];
}
}
void CUDATree::LaunchAddPredictionToScoreKernel(
const Dataset* data,
const data_size_t* used_data_indices,
data_size_t num_data,
double* score) const {
const CUDAColumnData* cuda_column_data = data->cuda_column_data();
const int num_blocks = (num_data + num_threads_per_block_add_prediction_to_score_ - 1) / num_threads_per_block_add_prediction_to_score_;
if (used_data_indices == nullptr) {
AddPredictionToScoreKernel<false><<<num_blocks, num_threads_per_block_add_prediction_to_score_>>>(
// dataset information
num_data,
cuda_column_data->cuda_data_by_column(),
cuda_column_data->cuda_column_bit_type(),
cuda_column_data->cuda_feature_min_bin(),
cuda_column_data->cuda_feature_max_bin(),
cuda_column_data->cuda_feature_offset(),
cuda_column_data->cuda_feature_default_bin(),
cuda_column_data->cuda_feature_most_freq_bin(),
cuda_column_data->cuda_feature_to_column(),
nullptr,
// tree information
cuda_threshold_in_bin_.RawData(),
cuda_decision_type_.RawData(),
cuda_split_feature_inner_.RawData(),
cuda_left_child_.RawData(),
cuda_right_child_.RawData(),
cuda_leaf_value_.RawData(),
cuda_bitset_inner_.RawDataReadOnly(),
cuda_cat_boundaries_inner_.RawDataReadOnly(),
// output
score);
} else {
AddPredictionToScoreKernel<true><<<num_blocks, num_threads_per_block_add_prediction_to_score_>>>(
// dataset information
num_data,
cuda_column_data->cuda_data_by_column(),
cuda_column_data->cuda_column_bit_type(),
cuda_column_data->cuda_feature_min_bin(),
cuda_column_data->cuda_feature_max_bin(),
cuda_column_data->cuda_feature_offset(),
cuda_column_data->cuda_feature_default_bin(),
cuda_column_data->cuda_feature_most_freq_bin(),
cuda_column_data->cuda_feature_to_column(),
used_data_indices,
// tree information
cuda_threshold_in_bin_.RawData(),
cuda_decision_type_.RawData(),
cuda_split_feature_inner_.RawData(),
cuda_left_child_.RawData(),
cuda_right_child_.RawData(),
cuda_leaf_value_.RawData(),
cuda_bitset_inner_.RawDataReadOnly(),
cuda_cat_boundaries_inner_.RawDataReadOnly(),
// output
score);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
}
} // namespace LightGBM
#endif // USE_CUDA
src/io/dataset.cpp
+1955
View File
File diff suppressed because it is too large Load Diff
src/io/dataset_loader.cpp
+1587
View File
File diff suppressed because it is too large Load Diff
src/io/dense_bin.hpp
+650
View File
@@ -0,0 +1,650 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_IO_DENSE_BIN_HPP_
#define LIGHTGBM_SRC_IO_DENSE_BIN_HPP_
#include <LightGBM/bin.h>
#include <LightGBM/cuda/vector_cudahost.h>
#include <cstdint>
#include <cstring>
#include <vector>
namespace LightGBM {
template <typename VAL_T, bool IS_4BIT>
class DenseBin;
template <typename VAL_T, bool IS_4BIT>
class DenseBinIterator : public BinIterator {
public:
explicit DenseBinIterator(const DenseBin<VAL_T, IS_4BIT>* bin_data,
uint32_t min_bin, uint32_t max_bin,
uint32_t most_freq_bin)
: bin_data_(bin_data),
min_bin_(static_cast<VAL_T>(min_bin)),
max_bin_(static_cast<VAL_T>(max_bin)),
most_freq_bin_(static_cast<VAL_T>(most_freq_bin)) {
if (most_freq_bin_ == 0) {
offset_ = 1;
} else {
offset_ = 0;
}
}
inline uint32_t RawGet(data_size_t idx) override;
inline uint32_t Get(data_size_t idx) override;
inline void Reset(data_size_t) override {}
private:
const DenseBin<VAL_T, IS_4BIT>* bin_data_;
VAL_T min_bin_;
VAL_T max_bin_;
VAL_T most_freq_bin_;
uint8_t offset_;
};
/*!
* \brief Used to store bins for dense feature
* Use template to reduce memory cost
*/
template <typename VAL_T, bool IS_4BIT>
class DenseBin : public Bin {
public:
friend DenseBinIterator<VAL_T, IS_4BIT>;
explicit DenseBin(data_size_t num_data)
: num_data_(num_data) {
if (IS_4BIT) {
CHECK_EQ(sizeof(VAL_T), 1);
data_.resize((num_data_ + 1) / 2, static_cast<uint8_t>(0));
buf_.resize((num_data_ + 1) / 2, static_cast<uint8_t>(0));
} else {
data_.resize(num_data_, static_cast<VAL_T>(0));
}
}
~DenseBin() {}
void Push(int, data_size_t idx, uint32_t value) override {
if (IS_4BIT) {
const int i1 = idx >> 1;
const int i2 = (idx & 1) << 2;
const uint8_t val = static_cast<uint8_t>(value) << i2;
if (i2 == 0) {
data_[i1] = val;
} else {
buf_[i1] = val;
}
} else {
data_[idx] = static_cast<VAL_T>(value);
}
}
void ReSize(data_size_t num_data) override {
if (num_data_ != num_data) {
num_data_ = num_data;
if (IS_4BIT) {
data_.resize((num_data_ + 1) / 2, static_cast<VAL_T>(0));
} else {
data_.resize(num_data_);
}
}
}
BinIterator* GetIterator(uint32_t min_bin, uint32_t max_bin,
uint32_t most_freq_bin) const override;
template <bool USE_INDICES, bool USE_PREFETCH, bool USE_HESSIAN>
void ConstructHistogramInner(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* ordered_gradients,
const score_t* ordered_hessians,
hist_t* out) const {
data_size_t i = start;
hist_t* grad = out;
hist_t* hess = out + 1;
hist_cnt_t* cnt = reinterpret_cast<hist_cnt_t*>(hess);
if (USE_PREFETCH) {
const data_size_t pf_offset = 64 / sizeof(VAL_T);
const data_size_t pf_end = end - pf_offset;
for (; i < pf_end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto pf_idx =
USE_INDICES ? data_indices[i + pf_offset] : i + pf_offset;
if (IS_4BIT) {
PREFETCH_T0(data_.data() + (pf_idx >> 1));
} else {
PREFETCH_T0(data_.data() + pf_idx);
}
const auto ti = static_cast<uint32_t>(data(idx)) << 1;
if (USE_HESSIAN) {
grad[ti] += ordered_gradients[i];
hess[ti] += ordered_hessians[i];
} else {
grad[ti] += ordered_gradients[i];
++cnt[ti];
}
}
}
for (; i < end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto ti = static_cast<uint32_t>(data(idx)) << 1;
if (USE_HESSIAN) {
grad[ti] += ordered_gradients[i];
hess[ti] += ordered_hessians[i];
} else {
grad[ti] += ordered_gradients[i];
++cnt[ti];
}
}
}
void ConstructHistogram(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
const score_t* ordered_hessians,
hist_t* out) const override {
ConstructHistogramInner<true, true, true>(
data_indices, start, end, ordered_gradients, ordered_hessians, out);
}
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
const score_t* ordered_hessians,
hist_t* out) const override {
ConstructHistogramInner<false, false, true>(
nullptr, start, end, ordered_gradients, ordered_hessians, out);
}
void ConstructHistogram(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
hist_t* out) const override {
ConstructHistogramInner<true, true, false>(data_indices, start, end,
ordered_gradients, nullptr, out);
}
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
hist_t* out) const override {
ConstructHistogramInner<false, false, false>(
nullptr, start, end, ordered_gradients, nullptr, out);
}
template <bool USE_INDICES, bool USE_PREFETCH, bool USE_HESSIAN, typename PACKED_HIST_T, int HIST_BITS>
void ConstructHistogramIntInner(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* ordered_gradients,
hist_t* out) const {
data_size_t i = start;
PACKED_HIST_T* out_ptr = reinterpret_cast<PACKED_HIST_T*>(out);
const int16_t* gradients_ptr = reinterpret_cast<const int16_t*>(ordered_gradients);
const VAL_T* data_ptr_base = data_.data();
if (USE_PREFETCH) {
const data_size_t pf_offset = 64 / sizeof(VAL_T);
const data_size_t pf_end = end - pf_offset;
for (; i < pf_end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto pf_idx =
USE_INDICES ? data_indices[i + pf_offset] : i + pf_offset;
if (IS_4BIT) {
PREFETCH_T0(data_ptr_base + (pf_idx >> 1));
} else {
PREFETCH_T0(data_ptr_base + pf_idx);
}
const auto ti = static_cast<uint32_t>(data(idx));
const int16_t gradient_16 = gradients_ptr[i];
if (USE_HESSIAN) {
const PACKED_HIST_T gradient_packed = HIST_BITS == 8 ? gradient_16 :
(static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) | (gradient_16 & 0xff);
out_ptr[ti] += gradient_packed;
} else {
const PACKED_HIST_T gradient_packed = HIST_BITS == 8 ? gradient_16 :
(static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) | (1);
out_ptr[ti] += gradient_packed;
}
}
}
for (; i < end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto ti = static_cast<uint32_t>(data(idx));
const int16_t gradient_16 = gradients_ptr[i];
if (USE_HESSIAN) {
const PACKED_HIST_T gradient_packed = HIST_BITS == 8 ? gradient_16 :
(static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) | (gradient_16 & 0xff);
out_ptr[ti] += gradient_packed;
} else {
const PACKED_HIST_T gradient_packed = HIST_BITS == 8 ? gradient_16 :
(static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) | (1);
out_ptr[ti] += gradient_packed;
}
}
}
void ConstructHistogramInt8(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, true, int16_t, 8>(
data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt8(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, true, int16_t, 8>(
nullptr, start, end, ordered_gradients, out);
}
void ConstructHistogramInt8(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, false, int16_t, 8>(
data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt8(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, false, int16_t, 8>(
nullptr, start, end, ordered_gradients, out);
}
void ConstructHistogramInt16(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, true, int32_t, 16>(
data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt16(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, true, int32_t, 16>(
nullptr, start, end, ordered_gradients, out);
}
void ConstructHistogramInt16(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, false, int32_t, 16>(
data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt16(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, false, int32_t, 16>(
nullptr, start, end, ordered_gradients, out);
}
void ConstructHistogramInt32(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, true, int64_t, 32>(
data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt32(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, true, int64_t, 32>(
nullptr, start, end, ordered_gradients, out);
}
void ConstructHistogramInt32(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, false, int64_t, 32>(
data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt32(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, false, int64_t, 32>(
nullptr, start, end, ordered_gradients, out);
}
template <bool MISS_IS_ZERO, bool MISS_IS_NA, bool MFB_IS_ZERO,
bool MFB_IS_NA, bool USE_MIN_BIN>
data_size_t SplitInner(uint32_t min_bin, uint32_t max_bin,
uint32_t default_bin, uint32_t most_freq_bin,
bool default_left, uint32_t threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const {
auto th = static_cast<VAL_T>(threshold + min_bin);
auto t_zero_bin = static_cast<VAL_T>(min_bin + default_bin);
if (most_freq_bin == 0) {
--th;
--t_zero_bin;
}
const auto minb = static_cast<VAL_T>(min_bin);
const auto maxb = static_cast<VAL_T>(max_bin);
data_size_t lte_count = 0;
data_size_t gt_count = 0;
data_size_t* default_indices = gt_indices;
data_size_t* default_count = &gt_count;
data_size_t* missing_default_indices = gt_indices;
data_size_t* missing_default_count = &gt_count;
if (most_freq_bin <= threshold) {
default_indices = lte_indices;
default_count = &lte_count;
}
if (MISS_IS_ZERO || MISS_IS_NA) {
if (default_left) {
missing_default_indices = lte_indices;
missing_default_count = &lte_count;
}
}
if (min_bin < max_bin) {
for (data_size_t i = 0; i < cnt; ++i) {
const data_size_t idx = data_indices[i];
const auto bin = data(idx);
if ((MISS_IS_ZERO && !MFB_IS_ZERO && bin == t_zero_bin) ||
(MISS_IS_NA && !MFB_IS_NA && bin == maxb)) {
missing_default_indices[(*missing_default_count)++] = idx;
} else if ((USE_MIN_BIN && (bin < minb || bin > maxb)) ||
(!USE_MIN_BIN && bin == 0)) {
if ((MISS_IS_NA && MFB_IS_NA) || (MISS_IS_ZERO && MFB_IS_ZERO)) {
missing_default_indices[(*missing_default_count)++] = idx;
} else {
default_indices[(*default_count)++] = idx;
}
} else if (bin > th) {
gt_indices[gt_count++] = idx;
} else {
lte_indices[lte_count++] = idx;
}
}
} else {
data_size_t* max_bin_indices = gt_indices;
data_size_t* max_bin_count = &gt_count;
if (maxb <= th) {
max_bin_indices = lte_indices;
max_bin_count = &lte_count;
}
for (data_size_t i = 0; i < cnt; ++i) {
const data_size_t idx = data_indices[i];
const auto bin = data(idx);
if (MISS_IS_ZERO && !MFB_IS_ZERO && bin == t_zero_bin) {
missing_default_indices[(*missing_default_count)++] = idx;
} else if (bin != maxb) {
if ((MISS_IS_NA && MFB_IS_NA) || (MISS_IS_ZERO && MFB_IS_ZERO)) {
missing_default_indices[(*missing_default_count)++] = idx;
} else {
default_indices[(*default_count)++] = idx;
}
} else {
if (MISS_IS_NA && !MFB_IS_NA) {
missing_default_indices[(*missing_default_count)++] = idx;
} else {
max_bin_indices[(*max_bin_count)++] = idx;
}
}
}
}
return lte_count;
}
data_size_t Split(uint32_t min_bin, uint32_t max_bin, uint32_t default_bin,
uint32_t most_freq_bin, MissingType missing_type,
bool default_left, uint32_t threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const override {
#define ARGUMENTS \
min_bin, max_bin, default_bin, most_freq_bin, default_left, threshold, \
data_indices, cnt, lte_indices, gt_indices
if (missing_type == MissingType::None) {
return SplitInner<false, false, false, false, true>(ARGUMENTS);
} else if (missing_type == MissingType::Zero) {
if (default_bin == most_freq_bin) {
return SplitInner<true, false, true, false, true>(ARGUMENTS);
} else {
return SplitInner<true, false, false, false, true>(ARGUMENTS);
}
} else {
if (max_bin == most_freq_bin + min_bin && most_freq_bin > 0) {
return SplitInner<false, true, false, true, true>(ARGUMENTS);
} else {
return SplitInner<false, true, false, false, true>(ARGUMENTS);
}
}
#undef ARGUMENTS
}
data_size_t Split(uint32_t max_bin, uint32_t default_bin,
uint32_t most_freq_bin, MissingType missing_type,
bool default_left, uint32_t threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const override {
#define ARGUMENTS \
1, max_bin, default_bin, most_freq_bin, default_left, threshold, \
data_indices, cnt, lte_indices, gt_indices
if (missing_type == MissingType::None) {
return SplitInner<false, false, false, false, false>(ARGUMENTS);
} else if (missing_type == MissingType::Zero) {
if (default_bin == most_freq_bin) {
return SplitInner<true, false, true, false, false>(ARGUMENTS);
} else {
return SplitInner<true, false, false, false, false>(ARGUMENTS);
}
} else {
if (max_bin == most_freq_bin + 1 && most_freq_bin > 0) {
return SplitInner<false, true, false, true, false>(ARGUMENTS);
} else {
return SplitInner<false, true, false, false, false>(ARGUMENTS);
}
}
#undef ARGUMENTS
}
template <bool USE_MIN_BIN>
data_size_t SplitCategoricalInner(uint32_t min_bin, uint32_t max_bin,
uint32_t most_freq_bin,
const uint32_t* threshold,
int num_threshold,
const data_size_t* data_indices,
data_size_t cnt, data_size_t* lte_indices,
data_size_t* gt_indices) const {
data_size_t lte_count = 0;
data_size_t gt_count = 0;
data_size_t* default_indices = gt_indices;
data_size_t* default_count = &gt_count;
int8_t offset = most_freq_bin == 0 ? 1 : 0;
if (most_freq_bin > 0 &&
Common::FindInBitset(threshold, num_threshold, most_freq_bin)) {
default_indices = lte_indices;
default_count = &lte_count;
}
for (data_size_t i = 0; i < cnt; ++i) {
const data_size_t idx = data_indices[i];
const uint32_t bin = data(idx);
if (USE_MIN_BIN && (bin < min_bin || bin > max_bin)) {
default_indices[(*default_count)++] = idx;
} else if (!USE_MIN_BIN && bin == 0) {
default_indices[(*default_count)++] = idx;
} else if (Common::FindInBitset(threshold, num_threshold,
bin - min_bin + offset)) {
lte_indices[lte_count++] = idx;
} else {
gt_indices[gt_count++] = idx;
}
}
return lte_count;
}
data_size_t SplitCategorical(uint32_t min_bin, uint32_t max_bin,
uint32_t most_freq_bin,
const uint32_t* threshold, int num_threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const override {
return SplitCategoricalInner<true>(min_bin, max_bin, most_freq_bin,
threshold, num_threshold, data_indices,
cnt, lte_indices, gt_indices);
}
data_size_t SplitCategorical(uint32_t max_bin, uint32_t most_freq_bin,
const uint32_t* threshold, int num_threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const override {
return SplitCategoricalInner<false>(1, max_bin, most_freq_bin, threshold,
num_threshold, data_indices, cnt,
lte_indices, gt_indices);
}
data_size_t num_data() const override { return num_data_; }
void* get_data() override { return data_.data(); }
void FinishLoad() override {
if (IS_4BIT) {
if (buf_.empty()) {
return;
}
int len = (num_data_ + 1) / 2;
for (int i = 0; i < len; ++i) {
data_[i] |= buf_[i];
}
buf_.clear();
}
}
void LoadFromMemory(
const void* memory,
const std::vector<data_size_t>& local_used_indices) override {
const VAL_T* mem_data = reinterpret_cast<const VAL_T*>(memory);
if (!local_used_indices.empty()) {
if (IS_4BIT) {
const data_size_t rest = num_data_ & 1;
for (int i = 0; i < num_data_ - rest; i += 2) {
// get old bins
data_size_t idx = local_used_indices[i];
const auto bin1 = static_cast<uint8_t>(
(mem_data[idx >> 1] >> ((idx & 1) << 2)) & 0xf);
idx = local_used_indices[i + 1];
const auto bin2 = static_cast<uint8_t>(
(mem_data[idx >> 1] >> ((idx & 1) << 2)) & 0xf);
// add
const int i1 = i >> 1;
data_[i1] = (bin1 | (bin2 << 4));
}
if (rest) {
data_size_t idx = local_used_indices[num_data_ - 1];
data_[num_data_ >> 1] =
(mem_data[idx >> 1] >> ((idx & 1) << 2)) & 0xf;
}
} else {
for (int i = 0; i < num_data_; ++i) {
data_[i] = mem_data[local_used_indices[i]];
}
}
} else {
for (size_t i = 0; i < data_.size(); ++i) {
data_[i] = mem_data[i];
}
}
}
inline VAL_T data(data_size_t idx) const {
if (IS_4BIT) {
return (data_[idx >> 1] >> ((idx & 1) << 2)) & 0xf;
} else {
return data_[idx];
}
}
void CopySubrow(const Bin* full_bin, const data_size_t* used_indices,
data_size_t num_used_indices) override {
auto other_bin = dynamic_cast<const DenseBin<VAL_T, IS_4BIT>*>(full_bin);
if (IS_4BIT) {
const data_size_t rest = num_used_indices & 1;
for (int i = 0; i < num_used_indices - rest; i += 2) {
data_size_t idx = used_indices[i];
const auto bin1 = static_cast<uint8_t>(
(other_bin->data_[idx >> 1] >> ((idx & 1) << 2)) & 0xf);
idx = used_indices[i + 1];
const auto bin2 = static_cast<uint8_t>(
(other_bin->data_[idx >> 1] >> ((idx & 1) << 2)) & 0xf);
const int i1 = i >> 1;
data_[i1] = (bin1 | (bin2 << 4));
}
if (rest) {
data_size_t idx = used_indices[num_used_indices - 1];
data_[num_used_indices >> 1] =
(other_bin->data_[idx >> 1] >> ((idx & 1) << 2)) & 0xf;
}
} else {
for (int i = 0; i < num_used_indices; ++i) {
data_[i] = other_bin->data_[used_indices[i]];
}
}
}
void SaveBinaryToFile(BinaryWriter* writer) const override {
writer->AlignedWrite(data_.data(), sizeof(VAL_T) * data_.size());
}
size_t SizesInByte() const override {
return VirtualFileWriter::AlignedSize(sizeof(VAL_T) * data_.size());
}
DenseBin<VAL_T, IS_4BIT>* Clone() override;
const void* GetColWiseData(uint8_t* bit_type, bool* is_sparse, std::vector<BinIterator*>* bin_iterator, const int num_threads) const override;
const void* GetColWiseData(uint8_t* bit_type, bool* is_sparse, BinIterator** bin_iterator) const override;
private:
data_size_t num_data_;
#ifdef USE_CUDA
std::vector<VAL_T, CHAllocator<VAL_T>> data_;
#else
std::vector<VAL_T, Common::AlignmentAllocator<VAL_T, kAlignedSize>> data_;
#endif
std::vector<uint8_t> buf_;
DenseBin(const DenseBin<VAL_T, IS_4BIT>& other)
: num_data_(other.num_data_), data_(other.data_) {}
};
template <typename VAL_T, bool IS_4BIT>
DenseBin<VAL_T, IS_4BIT>* DenseBin<VAL_T, IS_4BIT>::Clone() {
return new DenseBin<VAL_T, IS_4BIT>(*this);
}
template <typename VAL_T, bool IS_4BIT>
uint32_t DenseBinIterator<VAL_T, IS_4BIT>::Get(data_size_t idx) {
auto ret = bin_data_->data(idx);
if (ret >= min_bin_ && ret <= max_bin_) {
return ret - min_bin_ + offset_;
} else {
return most_freq_bin_;
}
}
template <typename VAL_T, bool IS_4BIT>
inline uint32_t DenseBinIterator<VAL_T, IS_4BIT>::RawGet(data_size_t idx) {
return bin_data_->data(idx);
}
template <typename VAL_T, bool IS_4BIT>
BinIterator* DenseBin<VAL_T, IS_4BIT>::GetIterator(
uint32_t min_bin, uint32_t max_bin, uint32_t most_freq_bin) const {
return new DenseBinIterator<VAL_T, IS_4BIT>(this, min_bin, max_bin,
most_freq_bin);
}
} // namespace LightGBM
#endif // LIGHTGBM_SRC_IO_DENSE_BIN_HPP_
src/io/file_io.cpp
+80
View File
@@ -0,0 +1,80 @@
/*!
* Copyright (c) 2018-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2018-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#include <LightGBM/utils/file_io.h>
#include <LightGBM/utils/log.h>
#include <algorithm>
#include <cstdio>
#include <memory>
#include <sstream>
#include <string>
#include <unordered_map>
namespace LightGBM {
struct LocalFile : VirtualFileReader, VirtualFileWriter {
LocalFile(const std::string& filename, const std::string& mode)
: filename_(filename), mode_(mode) {}
virtual ~LocalFile() {
if (file_ != NULL) {
fclose(file_);
}
}
bool Init() {
if (file_ == NULL) {
#if _MSC_VER
fopen_s(&file_, filename_.c_str(), mode_.c_str());
#else
file_ = fopen(filename_.c_str(), mode_.c_str());
#endif
}
return file_ != NULL;
}
bool Exists() const {
LocalFile file(filename_, "rb");
return file.Init();
}
size_t Read(void* buffer, size_t bytes) const {
return fread(buffer, 1, bytes, file_);
}
size_t Write(const void* buffer, size_t bytes) {
size_t bytes_written = fwrite(buffer, 1, bytes, file_);
if (bytes_written != bytes) {
Log::Fatal(
"Cannot write binary data to %s, wrote %zu of %zu bytes",
filename_.c_str(), bytes_written, bytes);
}
return bytes_written;
}
private:
FILE* file_ = NULL;
const std::string filename_;
const std::string mode_;
};
std::unique_ptr<VirtualFileReader> VirtualFileReader::Make(
const std::string& filename) {
return std::unique_ptr<VirtualFileReader>(new LocalFile(filename, "rb"));
}
std::unique_ptr<VirtualFileWriter> VirtualFileWriter::Make(
const std::string& filename) {
return std::unique_ptr<VirtualFileWriter>(new LocalFile(filename, "wb"));
}
bool VirtualFileWriter::Exists(const std::string& filename) {
LocalFile file(filename, "rb");
return file.Exists();
}
} // namespace LightGBM
src/io/json11.cpp
+785
View File
@@ -0,0 +1,785 @@
/* Copyright (c) 2013 Dropbox, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <LightGBM/utils/json11.h>
#include <LightGBM/utils/log.h>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <limits>
#include <memory>
#include <string>
#include <utility>
#include <vector>
namespace json11_internal_lightgbm {
static const int max_depth = 200;
using std::initializer_list;
using std::make_shared;
using std::map;
using std::string;
using std::vector;
using LightGBM::Log;
/* Helper for representing null - just a do-nothing struct, plus comparison
* operators so the helpers in JsonValue work. We can't use nullptr_t because
* it may not be orderable.
*/
struct NullStruct {
bool operator==(NullStruct) const { return true; }
bool operator<(NullStruct) const { return false; }
};
/* * * * * * * * * * * * * * * * * * * *
* Serialization
*/
static void dump(NullStruct, string *out) { *out += "null"; }
static void dump(double value, string *out) {
if (std::isfinite(value)) {
char buf[32];
snprintf(buf, sizeof buf, "%.17g", value);
*out += buf;
} else {
*out += "null";
}
}
static void dump(int value, string *out) {
char buf[32];
snprintf(buf, sizeof buf, "%d", value);
*out += buf;
}
static void dump(bool value, string *out) { *out += value ? "true" : "false"; }
static void dump(const string &value, string *out) {
*out += '"';
for (size_t i = 0; i < value.length(); i++) {
const char ch = value[i];
if (ch == '\\') {
*out += "\\\\";
} else if (ch == '"') {
*out += "\\\"";
} else if (ch == '\b') {
*out += "\\b";
} else if (ch == '\f') {
*out += "\\f";
} else if (ch == '\n') {
*out += "\\n";
} else if (ch == '\r') {
*out += "\\r";
} else if (ch == '\t') {
*out += "\\t";
} else if (static_cast<uint8_t>(ch) <= 0x1f) {
char buf[8];
snprintf(buf, sizeof buf, "\\u%04x", ch);
*out += buf;
} else if (static_cast<uint8_t>(ch) == 0xe2 &&
static_cast<uint8_t>(value[i + 1]) == 0x80 &&
static_cast<uint8_t>(value[i + 2]) == 0xa8) {
*out += "\\u2028";
i += 2;
} else if (static_cast<uint8_t>(ch) == 0xe2 &&
static_cast<uint8_t>(value[i + 1]) == 0x80 &&
static_cast<uint8_t>(value[i + 2]) == 0xa9) {
*out += "\\u2029";
i += 2;
} else {
*out += ch;
}
}
*out += '"';
}
static void dump(const Json::array &values, string *out) {
bool first = true;
*out += "[";
for (const auto &value : values) {
if (!first) *out += ", ";
value.dump(out);
first = false;
}
*out += "]";
}
static void dump(const Json::object &values, string *out) {
bool first = true;
*out += "{";
for (const auto &kv : values) {
if (!first) *out += ", ";
dump(kv.first, out);
*out += ": ";
kv.second.dump(out);
first = false;
}
*out += "}";
}
void Json::dump(string *out) const { m_ptr->dump(out); }
/* * * * * * * * * * * * * * * * * * * *
* Value wrappers
*/
template <Json::Type tag, typename T>
class Value : public JsonValue {
protected:
// Constructors
explicit Value(const T &value) : m_value(value) {}
explicit Value(T &&value) : m_value(std::move(value)) {}
// Get type tag
Json::Type type() const override { return tag; }
// Comparisons
bool equals(const JsonValue *other) const override {
return m_value == static_cast<const Value<tag, T> *>(other)->m_value;
}
bool less(const JsonValue *other) const override {
return m_value < (static_cast<const Value<tag, T> *>(other)->m_value);
}
const T m_value;
void dump(string *out) const override { json11_internal_lightgbm::dump(m_value, out); }
};
class JsonDouble final : public Value<Json::NUMBER, double> {
double number_value() const override { return m_value; }
int int_value() const override { return static_cast<int>(m_value); }
bool equals(const JsonValue *other) const override {
return m_value == other->number_value();
}
bool less(const JsonValue *other) const override {
return m_value < other->number_value();
}
public:
explicit JsonDouble(double value) : Value(value) {}
};
class JsonInt final : public Value<Json::NUMBER, int> {
double number_value() const override { return m_value; }
int int_value() const override { return m_value; }
bool equals(const JsonValue *other) const override {
return m_value == other->number_value();
}
bool less(const JsonValue *other) const override {
return m_value < other->number_value();
}
public:
explicit JsonInt(int value) : Value(value) {}
};
class JsonBoolean final : public Value<Json::BOOL, bool> {
bool bool_value() const override { return m_value; }
public:
explicit JsonBoolean(bool value) : Value(value) {}
};
class JsonString final : public Value<Json::STRING, string> {
const string &string_value() const override { return m_value; }
public:
explicit JsonString(const string &value) : Value(value) {}
explicit JsonString(string &&value) : Value(std::move(value)) {}
};
class JsonArray final : public Value<Json::ARRAY, Json::array> {
const Json::array &array_items() const override { return m_value; }
const Json &operator[](size_t i) const override;
public:
explicit JsonArray(const Json::array &value) : Value(value) {}
explicit JsonArray(Json::array &&value) : Value(std::move(value)) {}
};
class JsonObject final : public Value<Json::OBJECT, Json::object> {
const Json::object &object_items() const override { return m_value; }
const Json &operator[](const string &key) const override;
public:
explicit JsonObject(const Json::object &value) : Value(value) {}
explicit JsonObject(Json::object &&value) : Value(std::move(value)) {}
};
class JsonNull final : public Value<Json::NUL, NullStruct> {
public:
JsonNull() : Value({}) {}
};
/* * * * * * * * * * * * * * * * * * * *
* Static globals - static-init-safe
*/
struct Statics {
const std::shared_ptr<JsonValue> null = make_shared<JsonNull>();
const std::shared_ptr<JsonValue> t = make_shared<JsonBoolean>(true);
const std::shared_ptr<JsonValue> f = make_shared<JsonBoolean>(false);
const string empty_string;
const vector<Json> empty_vector;
const map<string, Json> empty_map;
Statics() {}
};
static const Statics &statics() {
static const Statics s{};
return s;
}
static const Json &static_null() {
// This has to be separate, not in Statics, because Json() accesses
// statics().null.
static const Json json_null;
return json_null;
}
/* * * * * * * * * * * * * * * * * * * *
* Constructors
*/
Json::Json() noexcept : m_ptr(statics().null) {}
Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {}
Json::Json(double value) : m_ptr(make_shared<JsonDouble>(value)) {}
Json::Json(int value) : m_ptr(make_shared<JsonInt>(value)) {}
Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {}
Json::Json(const string &value) : m_ptr(make_shared<JsonString>(value)) {}
Json::Json(string &&value) : m_ptr(make_shared<JsonString>(std::move(value))) {}
Json::Json(const char *value) : m_ptr(make_shared<JsonString>(value)) {}
Json::Json(const Json::array &values) : m_ptr(make_shared<JsonArray>(values)) {}
Json::Json(Json::array &&values)
: m_ptr(make_shared<JsonArray>(std::move(values))) {}
Json::Json(const Json::object &values)
: m_ptr(make_shared<JsonObject>(values)) {}
Json::Json(Json::object &&values)
: m_ptr(make_shared<JsonObject>(std::move(values))) {}
/* * * * * * * * * * * * * * * * * * * *
* Accessors
*/
Json::Type Json::type() const { return m_ptr->type(); }
double Json::number_value() const { return m_ptr->number_value(); }
int Json::int_value() const { return m_ptr->int_value(); }
bool Json::bool_value() const { return m_ptr->bool_value(); }
const string &Json::string_value() const { return m_ptr->string_value(); }
const vector<Json> &Json::array_items() const { return m_ptr->array_items(); }
const map<string, Json> &Json::object_items() const {
return m_ptr->object_items();
}
const Json &Json::operator[](size_t i) const { return (*m_ptr)[i]; }
const Json &Json::operator[](const string &key) const { return (*m_ptr)[key]; }
double JsonValue::number_value() const { return 0; }
int JsonValue::int_value() const { return 0; }
bool JsonValue::bool_value() const { return false; }
const string &JsonValue::string_value() const { return statics().empty_string; }
const vector<Json> &JsonValue::array_items() const {
return statics().empty_vector;
}
const map<string, Json> &JsonValue::object_items() const {
return statics().empty_map;
}
const Json &JsonValue::operator[](size_t) const { return static_null(); }
const Json &JsonValue::operator[](const string &) const {
return static_null();
}
const Json &JsonObject::operator[](const string &key) const {
auto iter = m_value.find(key);
return (iter == m_value.end()) ? static_null() : iter->second;
}
const Json &JsonArray::operator[](size_t i) const {
if (i >= m_value.size())
return static_null();
else
return m_value[i];
}
/* * * * * * * * * * * * * * * * * * * *
* Comparison
*/
bool Json::operator==(const Json &other) const {
if (m_ptr == other.m_ptr) return true;
if (m_ptr->type() != other.m_ptr->type()) return false;
return m_ptr->equals(other.m_ptr.get());
}
bool Json::operator<(const Json &other) const {
if (m_ptr == other.m_ptr) return false;
if (m_ptr->type() != other.m_ptr->type())
return m_ptr->type() < other.m_ptr->type();
return m_ptr->less(other.m_ptr.get());
}
/* * * * * * * * * * * * * * * * * * * *
* Parsing
*/
/* esc(c)
*
* Format char c suitable for printing in an error message.
*/
static inline string esc(char c) {
char buf[12];
if (static_cast<uint8_t>(c) >= 0x20 && static_cast<uint8_t>(c) <= 0x7f) {
snprintf(buf, sizeof buf, "'%c' (%d)", c, c);
} else {
snprintf(buf, sizeof buf, "(%d)", c);
}
return string(buf);
}
template <typename T>
static inline bool in_range(T x, T lower, T upper) {
return (x >= lower && x <= upper);
}
namespace {
/* JsonParser
*
* Object that tracks all state of an in-progress parse.
*/
struct JsonParser final {
/* State
*/
const char *str;
const size_t str_len;
size_t i;
string *err;
bool failed;
const JsonParse strategy;
/* fail(msg, err_ret = Json())
*
* Mark this parse as failed.
*/
Json fail(string &&msg) { return fail(std::move(msg), Json()); }
template <typename T>
T fail(string &&msg, const T err_ret) {
if (!failed) *err = std::move(msg);
failed = true;
return err_ret;
}
/* consume_whitespace()
*
* Advance until the current character is non-whitespace.
*/
void consume_whitespace() {
while (str[i] == ' ' || str[i] == '\r' || str[i] == '\n' || str[i] == '\t')
i++;
}
/* consume_comment()
*
* Advance comments (c-style inline and multiline).
*/
bool consume_comment() {
bool comment_found = false;
if (str[i] == '/') {
i++;
if (i == str_len)
return fail("Unexpected end of input after start of comment", false);
if (str[i] == '/') { // inline comment
i++;
// advance until next line, or end of input
while (i < str_len && str[i] != '\n') {
i++;
}
comment_found = true;
} else if (str[i] == '*') { // multiline comment
i++;
if (i > str_len - 2)
return fail("Unexpected end of input inside multi-line comment",
false);
// advance until closing tokens
while (!(str[i] == '*' && str[i + 1] == '/')) {
i++;
if (i > str_len - 2)
return fail("Unexpected end of input inside multi-line comment",
false);
}
i += 2;
comment_found = true;
} else {
return fail("Malformed comment", false);
}
}
return comment_found;
}
/* consume_garbage()
*
* Advance until the current character is non-whitespace and non-comment.
*/
void consume_garbage() {
consume_whitespace();
if (strategy == JsonParse::COMMENTS) {
bool comment_found = false;
do {
comment_found = consume_comment();
if (failed) return;
consume_whitespace();
} while (comment_found);
}
}
/* get_next_token()
*
* Return the next non-whitespace character. If the end of the input is
* reached, flag an error and return 0.
*/
char get_next_token() {
consume_garbage();
if (failed) return char{0};
if (i == str_len) return fail("Unexpected end of input", char{0});
return str[i++];
}
/* encode_utf8(pt, out)
*
* Encode pt as UTF-8 and add it to out.
*/
void encode_utf8(int64_t pt, string* out) {
if (pt < 0) return;
if (pt < 0x80) {
*out += static_cast<char>(pt);
} else if (pt < 0x800) {
*out += static_cast<char>((pt >> 6) | 0xC0);
*out += static_cast<char>((pt & 0x3F) | 0x80);
} else if (pt < 0x10000) {
*out += static_cast<char>((pt >> 12) | 0xE0);
*out += static_cast<char>(((pt >> 6) & 0x3F) | 0x80);
*out += static_cast<char>((pt & 0x3F) | 0x80);
} else {
*out += static_cast<char>((pt >> 18) | 0xF0);
*out += static_cast<char>(((pt >> 12) & 0x3F) | 0x80);
*out += static_cast<char>(((pt >> 6) & 0x3F) | 0x80);
*out += static_cast<char>((pt & 0x3F) | 0x80);
}
}
/* parse_string()
*
* Parse a string, starting at the current position.
*/
string parse_string() {
string out;
int64_t last_escaped_codepoint = -1;
while (true) {
if (i == str_len) return fail("Unexpected end of input in string", "");
char ch = str[i++];
if (ch == '"') {
encode_utf8(last_escaped_codepoint, &out);
return out;
}
if (in_range<int64_t>(ch, 0, 0x1f))
return fail("Unescaped " + esc(ch) + " in string", "");
// The usual case: non-escaped characters
if (ch != '\\') {
encode_utf8(last_escaped_codepoint, &out);
last_escaped_codepoint = -1;
out += ch;
continue;
}
// Handle escapes
if (i == str_len) return fail("Unexpected end of input in string", "");
ch = str[i++];
if (ch == 'u') {
// Extract 4-byte escape sequence
string esc = string(str + i, 4);
// Explicitly check length of the substring. The following loop
// relies on std::string returning the terminating NUL when
// accessing str[length]. Checking here reduces brittleness.
if (esc.length() < 4) {
return fail("Bad \\u escape: " + esc, "");
}
for (size_t j = 0; j < 4; j++) {
if (!in_range(esc[j], 'a', 'f') && !in_range(esc[j], 'A', 'F') &&
!in_range(esc[j], '0', '9'))
return fail("Bad \\u escape: " + esc, "");
}
int64_t codepoint =
static_cast<int64_t>(strtol(esc.data(), nullptr, 16));
// JSON specifies that characters outside the BMP shall be encoded as a
// pair of 4-hex-digit \u escapes encoding their surrogate pair
// components. Check whether we're in the middle of such a beast: the
// previous codepoint was an escaped lead (high) surrogate, and this is
// a trail (low) surrogate.
if (in_range<int64_t>(last_escaped_codepoint, 0xD800, 0xDBFF) &&
in_range<int64_t>(codepoint, 0xDC00, 0xDFFF)) {
// Reassemble the two surrogate pairs into one astral-plane character,
// per the UTF-16 algorithm.
encode_utf8((((last_escaped_codepoint - 0xD800) << 10) |
(codepoint - 0xDC00)) +
0x10000,
&out);
last_escaped_codepoint = -1;
} else {
encode_utf8(last_escaped_codepoint, &out);
last_escaped_codepoint = codepoint;
}
i += 4;
continue;
}
encode_utf8(last_escaped_codepoint, &out);
last_escaped_codepoint = -1;
if (ch == 'b') {
out += '\b';
} else if (ch == 'f') {
out += '\f';
} else if (ch == 'n') {
out += '\n';
} else if (ch == 'r') {
out += '\r';
} else if (ch == 't') {
out += '\t';
} else if (ch == '"' || ch == '\\' || ch == '/') {
out += ch;
} else {
return fail("Invalid escape character " + esc(ch), "");
}
}
}
/* parse_number()
*
* Parse a double.
*/
Json parse_number() {
size_t start_pos = i;
if (str[i] == '-') i++;
// Integer part
if (str[i] == '0') {
i++;
if (in_range(str[i], '0', '9'))
return fail("Leading 0s not permitted in numbers");
} else if (in_range(str[i], '1', '9')) {
i++;
while (in_range(str[i], '0', '9')) i++;
} else {
return fail("Invalid " + esc(str[i]) + " in number");
}
if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' &&
(i - start_pos) <=
static_cast<size_t>(std::numeric_limits<int>::digits10)) {
return Json(std::atoi(str + start_pos));
}
// Decimal part
if (str[i] == '.') {
i++;
if (!in_range(str[i], '0', '9'))
return fail("At least one digit required in fractional part");
while (in_range(str[i], '0', '9')) i++;
}
// Exponent part
if (str[i] == 'e' || str[i] == 'E') {
i++;
if (str[i] == '+' || str[i] == '-') i++;
if (!in_range(str[i], '0', '9'))
return fail("At least one digit required in exponent");
while (in_range(str[i], '0', '9')) i++;
}
return Json(std::strtod(str + start_pos, nullptr));
}
/* expect(str, res)
*
* Expect that 'str' starts at the character that was just read. If it does,
* advance the input and return res. If not, flag an error.
*/
Json expect(const string &expected, Json res) {
CHECK_NE(i, 0)
i--;
auto substr = string(str + i, expected.length());
if (substr == expected) {
i += expected.length();
return res;
} else {
return fail("Parse error: expected " + expected + ", got " + substr);
}
}
/* parse_json()
*
* Parse a JSON object.
*/
Json parse_json(int depth) {
if (depth > max_depth) {
return fail("Exceeded maximum nesting depth");
}
char ch = get_next_token();
if (failed) return Json();
if (ch == '-' || (ch >= '0' && ch <= '9')) {
i--;
return parse_number();
}
if (ch == 't') return expect("true", Json(true));
if (ch == 'f') return expect("false", Json(false));
if (ch == 'n') return expect("null", Json());
if (ch == '"') return Json(parse_string());
if (ch == '{') {
map<string, Json> data;
ch = get_next_token();
if (ch == '}') return Json(data);
while (1) {
if (ch != '"') return fail("Expected '\"' in object, got " + esc(ch));
string key = parse_string();
if (failed) return Json();
ch = get_next_token();
if (ch != ':') return fail("Expected ':' in object, got " + esc(ch));
data[std::move(key)] = parse_json(depth + 1);
if (failed) return Json();
ch = get_next_token();
if (ch == '}') break;
if (ch != ',') return fail("Expected ',' in object, got " + esc(ch));
ch = get_next_token();
}
return Json(data);
}
if (ch == '[') {
vector<Json> data;
ch = get_next_token();
if (ch == ']') return Json(data);
while (1) {
i--;
data.push_back(parse_json(depth + 1));
if (failed) return Json();
ch = get_next_token();
if (ch == ']') break;
if (ch != ',') return fail("Expected ',' in list, got " + esc(ch));
ch = get_next_token();
(void)ch;
}
return Json(data);
}
return fail("Expected value, got " + esc(ch));
}
};
} // namespace
Json Json::parse(const string &in, string *err, JsonParse strategy) {
JsonParser parser{in.c_str(), in.size(), 0, err, false, strategy};
Json result = parser.parse_json(0);
// Check for any trailing garbage
parser.consume_garbage();
if (parser.failed) return Json();
if (parser.i != in.size())
return parser.fail("Unexpected trailing " + esc(in[parser.i]));
return result;
}
// Documented in json11.hpp
vector<Json> Json::parse_multi(const string &in,
std::string::size_type *parser_stop_pos,
string *err, JsonParse strategy) {
JsonParser parser{in.c_str(), in.size(), 0, err, false, strategy};
*parser_stop_pos = 0;
vector<Json> json_vec;
while (parser.i != in.size() && !parser.failed) {
json_vec.push_back(parser.parse_json(0));
if (parser.failed) break;
// Check for another object
parser.consume_garbage();
if (parser.failed) break;
*parser_stop_pos = parser.i;
}
return json_vec;
}
/* * * * * * * * * * * * * * * * * * * *
* Shape-checking
*/
bool Json::has_shape(const shape &types, string *err) const {
if (!is_object()) {
*err = "Expected JSON object, got " + dump();
return false;
}
for (auto &item : types) {
if ((*this)[item.first].type() != item.second) {
*err = "Bad type for " + item.first + " in " + dump();
return false;
}
}
return true;
}
} // namespace json11_internal_lightgbm
src/io/metadata.cpp
+946
View File
@@ -0,0 +1,946 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/dataset.h>
#include <LightGBM/utils/common.h>
#include <set>
#include <string>
#include <unordered_map>
#include <vector>
#ifndef LGB_R_BUILD
#include "../arrow/array.hpp"
#endif // LGB_R_BUILD
namespace LightGBM {
Metadata::Metadata() {
num_weights_ = 0;
num_init_score_ = 0;
num_data_ = 0;
num_queries_ = 0;
num_positions_ = 0;
weight_load_from_file_ = false;
position_load_from_file_ = false;
query_load_from_file_ = false;
init_score_load_from_file_ = false;
#ifdef USE_CUDA
cuda_metadata_ = nullptr;
#endif // USE_CUDA
}
void Metadata::Init(const char* data_filename) {
data_filename_ = data_filename;
// for lambdarank, it needs query data for partition data in distributed learning
LoadQueryBoundaries();
LoadWeights();
LoadPositions();
CalculateQueryWeights();
LoadInitialScore(data_filename_);
}
Metadata::~Metadata() {
}
void Metadata::Init(data_size_t num_data, int weight_idx, int query_idx) {
num_data_ = num_data;
label_ = std::vector<label_t>(num_data_);
if (weight_idx >= 0) {
if (!weights_.empty()) {
Log::Info("Using weights in data file, ignoring the additional weights file");
weights_.clear();
}
weights_ = std::vector<label_t>(num_data_, 0.0f);
num_weights_ = num_data_;
weight_load_from_file_ = false;
}
if (query_idx >= 0) {
if (!query_boundaries_.empty()) {
Log::Info("Using query id in data file, ignoring the additional query file");
query_boundaries_.clear();
}
if (!query_weights_.empty()) {
query_weights_.clear();
}
queries_ = std::vector<data_size_t>(num_data_, 0);
query_load_from_file_ = false;
}
}
void Metadata::InitByReference(data_size_t num_data, const Metadata* reference) {
int has_weights = reference->num_weights_ > 0;
int has_init_scores = reference->num_init_score_ > 0;
int has_queries = reference->num_queries_ > 0;
int nclasses = reference->num_init_score_classes();
Init(num_data, has_weights, has_init_scores, has_queries, nclasses);
}
void Metadata::Init(data_size_t num_data, int32_t has_weights, int32_t has_init_scores, int32_t has_queries, int32_t nclasses) {
num_data_ = num_data;
label_ = std::vector<label_t>(num_data_);
if (has_weights) {
if (!weights_.empty()) {
Log::Fatal("Calling Init() on Metadata weights that have already been initialized");
}
weights_.resize(num_data_, 0.0f);
num_weights_ = num_data_;
weight_load_from_file_ = false;
}
if (has_init_scores) {
if (!init_score_.empty()) {
Log::Fatal("Calling Init() on Metadata initial scores that have already been initialized");
}
num_init_score_ = static_cast<int64_t>(num_data) * nclasses;
init_score_.resize(num_init_score_, 0);
}
if (has_queries) {
if (!query_weights_.empty()) {
Log::Fatal("Calling Init() on Metadata queries that have already been initialized");
}
queries_.resize(num_data_, 0);
query_load_from_file_ = false;
}
}
void Metadata::Init(const Metadata& fullset, const data_size_t* used_indices, data_size_t num_used_indices) {
num_data_ = num_used_indices;
label_.resize(num_used_indices);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (num_used_indices >= 1024)
for (data_size_t i = 0; i < num_used_indices; ++i) {
label_[i] = fullset.label_[used_indices[i]];
}
if (!fullset.weights_.empty()) {
weights_ = std::vector<label_t>(num_used_indices);
num_weights_ = num_used_indices;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (num_used_indices >= 1024)
for (data_size_t i = 0; i < num_used_indices; ++i) {
weights_[i] = fullset.weights_[used_indices[i]];
}
} else {
num_weights_ = 0;
}
if (!fullset.init_score_.empty()) {
int num_class = static_cast<int>(fullset.num_init_score_ / fullset.num_data_);
init_score_ = std::vector<double>(static_cast<size_t>(num_used_indices) * num_class);
num_init_score_ = static_cast<int64_t>(num_used_indices) * num_class;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int k = 0; k < num_class; ++k) {
const size_t offset_dest = static_cast<size_t>(k) * num_data_;
const size_t offset_src = static_cast<size_t>(k) * fullset.num_data_;
for (data_size_t i = 0; i < num_used_indices; ++i) {
init_score_[offset_dest + i] = fullset.init_score_[offset_src + used_indices[i]];
}
}
} else {
num_init_score_ = 0;
}
if (!fullset.query_boundaries_.empty()) {
std::vector<data_size_t> used_query;
data_size_t data_idx = 0;
for (data_size_t qid = 0; qid < num_queries_ && data_idx < num_used_indices; ++qid) {
data_size_t start = fullset.query_boundaries_[qid];
data_size_t end = fullset.query_boundaries_[qid + 1];
data_size_t len = end - start;
if (used_indices[data_idx] > start) {
continue;
} else if (used_indices[data_idx] == start) {
if (num_used_indices >= data_idx + len && used_indices[data_idx + len - 1] == end - 1) {
used_query.push_back(qid);
data_idx += len;
} else {
Log::Fatal("Data partition error, data didn't match queries");
}
} else {
Log::Fatal("Data partition error, data didn't match queries");
}
}
query_boundaries_ = std::vector<data_size_t>(used_query.size() + 1);
num_queries_ = static_cast<data_size_t>(used_query.size());
query_boundaries_[0] = 0;
for (data_size_t i = 0; i < num_queries_; ++i) {
data_size_t qid = used_query[i];
data_size_t len = fullset.query_boundaries_[qid + 1] - fullset.query_boundaries_[qid];
query_boundaries_[i + 1] = query_boundaries_[i] + len;
}
} else {
num_queries_ = 0;
}
}
void Metadata::PartitionLabel(const std::vector<data_size_t>& used_indices) {
if (used_indices.empty()) {
return;
}
auto old_label = label_;
num_data_ = static_cast<data_size_t>(used_indices.size());
label_ = std::vector<label_t>(num_data_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (num_data_ >= 1024)
for (data_size_t i = 0; i < num_data_; ++i) {
label_[i] = old_label[used_indices[i]];
}
old_label.clear();
}
void Metadata::CalculateQueryBoundaries() {
if (!queries_.empty()) {
// need convert query_id to boundaries
std::vector<data_size_t> tmp_buffer;
data_size_t last_qid = -1;
data_size_t cur_cnt = 0;
for (data_size_t i = 0; i < num_data_; ++i) {
if (last_qid != queries_[i]) {
if (cur_cnt > 0) {
tmp_buffer.push_back(cur_cnt);
}
cur_cnt = 0;
last_qid = queries_[i];
}
++cur_cnt;
}
tmp_buffer.push_back(cur_cnt);
query_boundaries_ = std::vector<data_size_t>(tmp_buffer.size() + 1);
num_queries_ = static_cast<data_size_t>(tmp_buffer.size());
query_boundaries_[0] = 0;
for (size_t i = 0; i < tmp_buffer.size(); ++i) {
query_boundaries_[i + 1] = query_boundaries_[i] + tmp_buffer[i];
}
CalculateQueryWeights();
queries_.clear();
}
}
void Metadata::CheckOrPartition(data_size_t num_all_data, const std::vector<data_size_t>& used_data_indices) {
if (used_data_indices.empty()) {
CalculateQueryBoundaries();
// check weights
if (!weights_.empty() && num_weights_ != num_data_) {
weights_.clear();
num_weights_ = 0;
Log::Fatal("Weights size doesn't match data size");
}
// check positions
if (!positions_.empty() && num_positions_ != num_data_) {
Log::Fatal("Positions size (%i) doesn't match data size (%i)", num_positions_, num_data_);
positions_.clear();
num_positions_ = 0;
}
// check query boundaries
if (!query_boundaries_.empty() && query_boundaries_[num_queries_] != num_data_) {
query_boundaries_.clear();
num_queries_ = 0;
Log::Fatal("Query size doesn't match data size");
}
// contain initial score file
if (!init_score_.empty() && (num_init_score_ % num_data_) != 0) {
init_score_.clear();
num_init_score_ = 0;
Log::Fatal("Initial score size doesn't match data size");
}
} else {
if (!queries_.empty()) {
Log::Fatal("Cannot used query_id for distributed training");
}
data_size_t num_used_data = static_cast<data_size_t>(used_data_indices.size());
// check weights
if (weight_load_from_file_) {
if (weights_.size() > 0 && num_weights_ != num_all_data) {
weights_.clear();
num_weights_ = 0;
Log::Fatal("Weights size doesn't match data size");
}
// get local weights
if (!weights_.empty()) {
auto old_weights = weights_;
num_weights_ = num_data_;
weights_ = std::vector<label_t>(num_data_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512)
for (int i = 0; i < static_cast<int>(used_data_indices.size()); ++i) {
weights_[i] = old_weights[used_data_indices[i]];
}
old_weights.clear();
}
}
// check positions
if (position_load_from_file_) {
if (positions_.size() > 0 && num_positions_ != num_all_data) {
positions_.clear();
num_positions_ = 0;
Log::Fatal("Positions size (%i) doesn't match data size (%i)", num_positions_, num_data_);
}
// get local positions
if (!positions_.empty()) {
auto old_positions = positions_;
num_positions_ = num_data_;
positions_ = std::vector<data_size_t>(num_data_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512)
for (int i = 0; i < static_cast<int>(used_data_indices.size()); ++i) {
positions_[i] = old_positions[used_data_indices[i]];
}
old_positions.clear();
}
}
if (query_load_from_file_) {
// check query boundaries
if (!query_boundaries_.empty() && query_boundaries_[num_queries_] != num_all_data) {
query_boundaries_.clear();
num_queries_ = 0;
Log::Fatal("Query size doesn't match data size");
}
// get local query boundaries
if (!query_boundaries_.empty()) {
std::vector<data_size_t> used_query;
data_size_t data_idx = 0;
for (data_size_t qid = 0; qid < num_queries_ && data_idx < num_used_data; ++qid) {
data_size_t start = query_boundaries_[qid];
data_size_t end = query_boundaries_[qid + 1];
data_size_t len = end - start;
if (used_data_indices[data_idx] > start) {
continue;
} else if (used_data_indices[data_idx] == start) {
if (num_used_data >= data_idx + len && used_data_indices[data_idx + len - 1] == end - 1) {
used_query.push_back(qid);
data_idx += len;
} else {
Log::Fatal("Data partition error, data didn't match queries");
}
} else {
Log::Fatal("Data partition error, data didn't match queries");
}
}
auto old_query_boundaries = query_boundaries_;
query_boundaries_ = std::vector<data_size_t>(used_query.size() + 1);
num_queries_ = static_cast<data_size_t>(used_query.size());
query_boundaries_[0] = 0;
for (data_size_t i = 0; i < num_queries_; ++i) {
data_size_t qid = used_query[i];
data_size_t len = old_query_boundaries[qid + 1] - old_query_boundaries[qid];
query_boundaries_[i + 1] = query_boundaries_[i] + len;
}
old_query_boundaries.clear();
}
}
if (init_score_load_from_file_) {
// contain initial score file
if (!init_score_.empty() && (num_init_score_ % num_all_data) != 0) {
init_score_.clear();
num_init_score_ = 0;
Log::Fatal("Initial score size doesn't match data size");
}
// get local initial scores
if (!init_score_.empty()) {
auto old_scores = init_score_;
int num_class = static_cast<int>(num_init_score_ / num_all_data);
num_init_score_ = static_cast<int64_t>(num_data_) * num_class;
init_score_ = std::vector<double>(num_init_score_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (int k = 0; k < num_class; ++k) {
const size_t offset_dest = static_cast<size_t>(k) * num_data_;
const size_t offset_src = static_cast<size_t>(k) * num_all_data;
for (size_t i = 0; i < used_data_indices.size(); ++i) {
init_score_[offset_dest + i] = old_scores[offset_src + used_data_indices[i]];
}
}
old_scores.clear();
}
}
// re-calculate query weight
CalculateQueryWeights();
}
if (num_queries_ > 0) {
Log::Debug("Number of queries in %s: %i. Average number of rows per query: %f.",
data_filename_.c_str(), static_cast<int>(num_queries_), static_cast<double>(num_data_) / num_queries_);
}
}
template <typename It>
void Metadata::SetInitScoresFromIterator(It first, It last) {
std::lock_guard<std::mutex> lock(mutex_);
// Clear init scores on empty input
if (last - first == 0) {
init_score_.clear();
num_init_score_ = 0;
return;
}
if (((last - first) % num_data_) != 0) {
Log::Fatal("Initial score size doesn't match data size");
}
if (init_score_.empty()) {
init_score_.resize(last - first);
}
num_init_score_ = last - first;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (num_init_score_ >= 1024)
for (int64_t i = 0; i < num_init_score_; ++i) {
init_score_[i] = Common::AvoidInf(first[i]);
}
init_score_load_from_file_ = false;
#ifdef USE_CUDA
if (cuda_metadata_ != nullptr) {
cuda_metadata_->SetInitScore(init_score_.data(), init_score_.size());
}
#endif // USE_CUDA
}
void Metadata::SetInitScore(const double* init_score, data_size_t len) {
SetInitScoresFromIterator(init_score, init_score + len);
}
#ifndef LGB_R_BUILD
ArrowChunkedArray::View InitScoreView(const ArrowChunkedArray& chunked_array) {
auto view = chunked_array.view();
// For multiclass classification, the init scores are provided in multiple columns. In this
// case, we must concatenate all fields of the chunked array.
if (chunked_array.is_struct()) {
std::vector<ArrowChunkedArray::View> concat_views;
concat_views.reserve(chunked_array.get_num_fields());
for (int64_t i = 0; i < chunked_array.get_num_fields(); ++i) {
concat_views.push_back(view.field(i));
}
view = ArrowChunkedArray::View(concat_views);
}
return view;
}
void Metadata::SetInitScore(struct ArrowArrayStream* stream) {
ArrowChunkedArray chunked_array(stream);
auto view = InitScoreView(chunked_array);
view.visit<double>([&](auto&& visitor) {
SetInitScoresFromIterator(visitor.begin(), visitor.end());
});
}
void Metadata::SetInitScore(int64_t n_chunks, struct ArrowArray* chunks,
struct ArrowSchema* schema) {
ArrowChunkedArray chunked_array(n_chunks, chunks, schema);
auto view = InitScoreView(chunked_array);
view.visit<double>([&](auto&& visitor) {
SetInitScoresFromIterator(visitor.begin(), visitor.end());
});
}
#endif // LGB_R_BUILD
void Metadata::InsertInitScores(const double* init_scores, data_size_t start_index, data_size_t len, data_size_t source_size) {
if (num_init_score_ <= 0) {
Log::Fatal("Inserting initial score data into dataset with no initial scores");
}
if (start_index + len > num_data_) {
// Note that len here is row count, not num_init_score, so we compare against num_data
Log::Fatal("Inserted initial score data is too large for dataset");
}
if (init_score_.empty()) {
init_score_.resize(num_init_score_);
}
int nclasses = num_init_score_classes();
for (int32_t col = 0; col < nclasses; ++col) {
int32_t dest_offset = num_data_ * col + start_index;
// We need to use source_size here, because len might not equal size (due to a partially loaded dataset)
int32_t source_offset = source_size * col;
memcpy(init_score_.data() + dest_offset, init_scores + source_offset, sizeof(double) * len);
}
init_score_load_from_file_ = false;
// CUDA is handled after all insertions are complete
}
template <typename It>
void Metadata::SetLabelsFromIterator(It first, It last) {
std::lock_guard<std::mutex> lock(mutex_);
if (num_data_ != last - first) {
Log::Fatal("Length of labels differs from the length of #data");
}
if (label_.empty()) {
label_.resize(num_data_);
}
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (num_data_ >= 1024)
for (data_size_t i = 0; i < num_data_; ++i) {
label_[i] = Common::AvoidInf(first[i]);
}
#ifdef USE_CUDA
if (cuda_metadata_ != nullptr) {
cuda_metadata_->SetLabel(label_.data(), label_.size());
}
#endif // USE_CUDA
}
void Metadata::SetLabel(const label_t* label, data_size_t len) {
if (label == nullptr) {
Log::Fatal("label cannot be nullptr");
}
SetLabelsFromIterator(label, label + len);
}
#ifndef LGB_R_BUILD
void Metadata::SetLabel(struct ArrowArrayStream* stream) {
ArrowChunkedArray chunked_array(stream);
chunked_array.view().visit<label_t>([&](auto&& visitor) {
SetLabelsFromIterator(visitor.begin(), visitor.end());
});
}
void Metadata::SetLabel(int64_t n_chunks, struct ArrowArray* chunks,
struct ArrowSchema* schema) {
ArrowChunkedArray chunked_array(n_chunks, chunks, schema);
chunked_array.view().visit<label_t>([&](auto&& visitor) {
SetLabelsFromIterator(visitor.begin(), visitor.end());
});
}
#endif // LGB_R_BUILD
void Metadata::InsertLabels(const label_t* labels, data_size_t start_index, data_size_t len) {
if (labels == nullptr) {
Log::Fatal("label cannot be nullptr");
}
if (start_index + len > num_data_) {
Log::Fatal("Inserted label data is too large for dataset");
}
if (label_.empty()) {
label_.resize(num_data_);
}
memcpy(label_.data() + start_index, labels, sizeof(label_t) * len);
// CUDA is handled after all insertions are complete
}
template <typename It>
void Metadata::SetWeightsFromIterator(It first, It last) {
std::lock_guard<std::mutex> lock(mutex_);
// Clear weights on empty input
if (last - first == 0) {
weights_.clear();
num_weights_ = 0;
return;
}
if (num_data_ != last - first) {
Log::Fatal("Length of weights differs from the length of #data");
}
if (weights_.empty()) {
weights_.resize(num_data_);
}
num_weights_ = num_data_;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (num_weights_ >= 1024)
for (data_size_t i = 0; i < num_weights_; ++i) {
weights_[i] = Common::AvoidInf(first[i]);
}
CalculateQueryWeights();
weight_load_from_file_ = false;
#ifdef USE_CUDA
if (cuda_metadata_ != nullptr) {
cuda_metadata_->SetWeights(weights_.data(), weights_.size());
}
#endif // USE_CUDA
}
void Metadata::SetWeights(const label_t* weights, data_size_t len) {
SetWeightsFromIterator(weights, weights + len);
}
#ifndef LGB_R_BUILD
void Metadata::SetWeights(struct ArrowArrayStream* stream) {
ArrowChunkedArray chunked_array(stream);
chunked_array.view().visit<label_t>([&](auto&& visitor) {
SetWeightsFromIterator(visitor.begin(), visitor.end());
});
}
void Metadata::SetWeights(int64_t n_chunks, struct ArrowArray* chunks,
struct ArrowSchema* schema) {
ArrowChunkedArray chunked_array(n_chunks, chunks, schema);
chunked_array.view().visit<label_t>([&](auto&& visitor) {
SetWeightsFromIterator(visitor.begin(), visitor.end());
});
}
#endif // LGB_R_BUILD
void Metadata::InsertWeights(const label_t* weights, data_size_t start_index, data_size_t len) {
if (!weights) {
Log::Fatal("Passed null weights");
}
if (num_weights_ <= 0) {
Log::Fatal("Inserting weight data into dataset with no weights");
}
if (start_index + len > num_weights_) {
Log::Fatal("Inserted weight data is too large for dataset");
}
if (weights_.empty()) {
weights_.resize(num_weights_);
}
memcpy(weights_.data() + start_index, weights, sizeof(label_t) * len);
weight_load_from_file_ = false;
// CUDA is handled after all insertions are complete
}
template <typename It>
void Metadata::SetQueriesFromIterator(It first, It last) {
std::lock_guard<std::mutex> lock(mutex_);
// Clear query boundaries on empty input
if (last - first == 0) {
query_boundaries_.clear();
num_queries_ = 0;
return;
}
data_size_t sum = 0;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum)
for (data_size_t i = 0; i < static_cast<data_size_t>(last - first); ++i) {
sum += first[i];
}
if (num_data_ != sum) {
Log::Fatal("Sum of query counts (%i) differs from the length of #data (%i)", num_data_, sum);
}
num_queries_ = last - first;
query_boundaries_.resize(num_queries_ + 1);
query_boundaries_[0] = 0;
for (data_size_t i = 0; i < num_queries_; ++i) {
query_boundaries_[i + 1] = query_boundaries_[i] + first[i];
}
CalculateQueryWeights();
query_load_from_file_ = false;
#ifdef USE_CUDA
if (cuda_metadata_ != nullptr) {
if (query_weights_.size() > 0) {
CHECK_EQ(query_weights_.size(), static_cast<size_t>(num_queries_));
cuda_metadata_->SetQuery(query_boundaries_.data(), query_weights_.data(), num_queries_);
} else {
cuda_metadata_->SetQuery(query_boundaries_.data(), nullptr, num_queries_);
}
}
#endif // USE_CUDA
}
void Metadata::SetQuery(const data_size_t* query, data_size_t len) {
SetQueriesFromIterator(query, query + len);
}
#ifndef LGB_R_BUILD
void Metadata::SetQuery(struct ArrowArrayStream* stream) {
ArrowChunkedArray chunked_array(stream);
chunked_array.view().visit<data_size_t>([&](auto&& visitor) {
SetQueriesFromIterator(visitor.begin(), visitor.end());
});
}
void Metadata::SetQuery(int64_t n_chunks, struct ArrowArray* chunks,
struct ArrowSchema* schema) {
ArrowChunkedArray chunked_array(n_chunks, chunks, schema);
chunked_array.view().visit<data_size_t>([&](auto&& visitor) {
SetQueriesFromIterator(visitor.begin(), visitor.end());
});
}
#endif // LGB_R_BUILD
void Metadata::SetPosition(const data_size_t* positions, data_size_t len) {
std::lock_guard<std::mutex> lock(mutex_);
// save to nullptr
if (positions == nullptr || len == 0) {
positions_.clear();
num_positions_ = 0;
return;
}
#ifdef USE_CUDA
Log::Fatal("Positions in learning to rank is not supported in CUDA version yet.");
#endif // USE_CUDA
if (num_data_ != len) {
Log::Fatal("Positions size (%i) doesn't match data size (%i)", len, num_data_);
}
if (positions_.empty()) {
positions_.resize(num_data_);
} else {
Log::Warning("Overwriting positions in dataset.");
}
num_positions_ = num_data_;
position_load_from_file_ = false;
position_ids_.clear();
std::unordered_map<data_size_t, int> map_id2pos;
for (data_size_t i = 0; i < num_positions_; ++i) {
if (map_id2pos.count(positions[i]) == 0) {
int pos = static_cast<int>(map_id2pos.size());
map_id2pos[positions[i]] = pos;
position_ids_.push_back(std::to_string(positions[i]));
}
}
Log::Debug("number of unique positions found = %ld", position_ids_.size());
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (num_positions_ >= 1024)
for (data_size_t i = 0; i < num_positions_; ++i) {
positions_[i] = map_id2pos.at(positions[i]);
}
}
void Metadata::InsertQueries(const data_size_t* queries, data_size_t start_index, data_size_t len) {
if (!queries) {
Log::Fatal("Passed null queries");
}
if (queries_.size() <= 0) {
Log::Fatal("Inserting query data into dataset with no queries");
}
if (static_cast<size_t>(start_index + len) > queries_.size()) {
Log::Fatal("Inserted query data is too large for dataset");
}
memcpy(queries_.data() + start_index, queries, sizeof(data_size_t) * len);
query_load_from_file_ = false;
// CUDA is handled after all insertions are complete
}
void Metadata::LoadWeights() {
num_weights_ = 0;
std::string weight_filename(data_filename_);
// default weight file name
weight_filename.append(".weight");
TextReader<size_t> reader(weight_filename.c_str(), false);
reader.ReadAllLines();
if (reader.Lines().empty()) {
return;
}
Log::Info("Loading weights...");
num_weights_ = static_cast<data_size_t>(reader.Lines().size());
weights_ = std::vector<label_t>(num_weights_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_weights_; ++i) {
double tmp_weight = 0.0f;
Common::Atof(reader.Lines()[i].c_str(), &tmp_weight);
weights_[i] = Common::AvoidInf(static_cast<label_t>(tmp_weight));
}
weight_load_from_file_ = true;
}
void Metadata::LoadPositions() {
num_positions_ = 0;
std::string position_filename(data_filename_);
// default position file name
position_filename.append(".position");
TextReader<size_t> reader(position_filename.c_str(), false);
reader.ReadAllLines();
if (reader.Lines().empty()) {
return;
}
Log::Info("Loading positions from %s ...", position_filename.c_str());
num_positions_ = static_cast<data_size_t>(reader.Lines().size());
positions_ = std::vector<data_size_t>(num_positions_);
position_ids_ = std::vector<std::string>();
std::unordered_map<std::string, data_size_t> map_id2pos;
for (data_size_t i = 0; i < num_positions_; ++i) {
std::string& line = reader.Lines()[i];
if (map_id2pos.count(line) == 0) {
map_id2pos[line] = static_cast<data_size_t>(position_ids_.size());
position_ids_.push_back(line);
}
positions_[i] = map_id2pos.at(line);
}
position_load_from_file_ = true;
}
void Metadata::LoadInitialScore(const std::string& data_filename) {
num_init_score_ = 0;
std::string init_score_filename(data_filename);
init_score_filename = std::string(data_filename);
// default init_score file name
init_score_filename.append(".init");
TextReader<size_t> reader(init_score_filename.c_str(), false);
reader.ReadAllLines();
if (reader.Lines().empty()) {
return;
}
Log::Info("Loading initial scores...");
// use first line to count number class
int num_class = static_cast<int>(Common::Split(reader.Lines()[0].c_str(), '\t').size());
data_size_t num_line = static_cast<data_size_t>(reader.Lines().size());
num_init_score_ = static_cast<int64_t>(num_line) * num_class;
init_score_ = std::vector<double>(num_init_score_);
if (num_class == 1) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_line; ++i) {
double tmp = 0.0f;
Common::Atof(reader.Lines()[i].c_str(), &tmp);
init_score_[i] = Common::AvoidInf(static_cast<double>(tmp));
}
} else {
std::vector<std::string> oneline_init_score;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_line; ++i) {
double tmp = 0.0f;
oneline_init_score = Common::Split(reader.Lines()[i].c_str(), '\t');
if (static_cast<int>(oneline_init_score.size()) != num_class) {
Log::Fatal("Invalid initial score file. Redundant or insufficient columns");
}
for (int k = 0; k < num_class; ++k) {
Common::Atof(oneline_init_score[k].c_str(), &tmp);
init_score_[static_cast<size_t>(k) * num_line + i] = Common::AvoidInf(static_cast<double>(tmp));
}
}
}
init_score_load_from_file_ = true;
}
void Metadata::LoadQueryBoundaries() {
num_queries_ = 0;
std::string query_filename(data_filename_);
// default query file name
query_filename.append(".query");
TextReader<size_t> reader(query_filename.c_str(), false);
reader.ReadAllLines();
if (reader.Lines().empty()) {
return;
}
Log::Info("Calculating query boundaries...");
query_boundaries_ = std::vector<data_size_t>(reader.Lines().size() + 1);
num_queries_ = static_cast<data_size_t>(reader.Lines().size());
query_boundaries_[0] = 0;
for (size_t i = 0; i < reader.Lines().size(); ++i) {
int tmp_cnt;
Common::Atoi(reader.Lines()[i].c_str(), &tmp_cnt);
query_boundaries_[i + 1] = query_boundaries_[i] + static_cast<data_size_t>(tmp_cnt);
}
query_load_from_file_ = true;
}
void Metadata::CalculateQueryWeights() {
if (weights_.size() == 0 || query_boundaries_.size() == 0) {
return;
}
query_weights_.clear();
Log::Info("Calculating query weights...");
query_weights_ = std::vector<label_t>(num_queries_);
for (data_size_t i = 0; i < num_queries_; ++i) {
query_weights_[i] = 0.0f;
for (data_size_t j = query_boundaries_[i]; j < query_boundaries_[i + 1]; ++j) {
query_weights_[i] += weights_[j];
}
query_weights_[i] /= (query_boundaries_[i + 1] - query_boundaries_[i]);
}
}
void Metadata::InsertAt(data_size_t start_index,
data_size_t count,
const float* labels,
const float* weights,
const double* init_scores,
const int32_t* queries) {
if (num_data_ < count + start_index) {
Log::Fatal("Length of metadata is too long to append #data");
}
InsertLabels(labels, start_index, count);
if (weights) {
InsertWeights(weights, start_index, count);
}
if (init_scores) {
InsertInitScores(init_scores, start_index, count, count);
}
if (queries) {
InsertQueries(queries, start_index, count);
}
}
void Metadata::FinishLoad() {
CalculateQueryBoundaries();
}
#ifdef USE_CUDA
void Metadata::CreateCUDAMetadata(const int gpu_device_id) {
cuda_metadata_.reset(new CUDAMetadata(gpu_device_id));
cuda_metadata_->Init(label_, weights_, query_boundaries_, query_weights_, init_score_);
}
#endif // USE_CUDA
void Metadata::LoadFromMemory(const void* memory) {
const char* mem_ptr = reinterpret_cast<const char*>(memory);
num_data_ = *(reinterpret_cast<const data_size_t*>(mem_ptr));
mem_ptr += VirtualFileWriter::AlignedSize(sizeof(num_data_));
num_weights_ = *(reinterpret_cast<const data_size_t*>(mem_ptr));
mem_ptr += VirtualFileWriter::AlignedSize(sizeof(num_weights_));
num_queries_ = *(reinterpret_cast<const data_size_t*>(mem_ptr));
mem_ptr += VirtualFileWriter::AlignedSize(sizeof(num_queries_));
if (!label_.empty()) {
label_.clear();
}
label_ = std::vector<label_t>(num_data_);
std::memcpy(label_.data(), mem_ptr, sizeof(label_t) * num_data_);
mem_ptr += VirtualFileWriter::AlignedSize(sizeof(label_t) * num_data_);
if (num_weights_ > 0) {
if (!weights_.empty()) {
weights_.clear();
}
weights_ = std::vector<label_t>(num_weights_);
std::memcpy(weights_.data(), mem_ptr, sizeof(label_t) * num_weights_);
mem_ptr += VirtualFileWriter::AlignedSize(sizeof(label_t) * num_weights_);
weight_load_from_file_ = true;
}
if (num_queries_ > 0) {
if (!query_boundaries_.empty()) {
query_boundaries_.clear();
}
query_boundaries_ = std::vector<data_size_t>(num_queries_ + 1);
std::memcpy(query_boundaries_.data(), mem_ptr, sizeof(data_size_t) * (num_queries_ + 1));
mem_ptr += VirtualFileWriter::AlignedSize(sizeof(data_size_t) *
(num_queries_ + 1));
query_load_from_file_ = true;
}
CalculateQueryWeights();
}
void Metadata::SaveBinaryToFile(BinaryWriter* writer) const {
writer->AlignedWrite(&num_data_, sizeof(num_data_));
writer->AlignedWrite(&num_weights_, sizeof(num_weights_));
writer->AlignedWrite(&num_queries_, sizeof(num_queries_));
writer->AlignedWrite(label_.data(), sizeof(label_t) * num_data_);
if (!weights_.empty()) {
writer->AlignedWrite(weights_.data(), sizeof(label_t) * num_weights_);
}
if (!query_boundaries_.empty()) {
writer->AlignedWrite(query_boundaries_.data(),
sizeof(data_size_t) * (num_queries_ + 1));
}
if (num_init_score_ > 0) {
Log::Warning("Please note that `init_score` is not saved in binary file.\n"
"If you need it, please set it again after loading Dataset.");
}
}
size_t Metadata::SizesInByte() const {
size_t size = VirtualFileWriter::AlignedSize(sizeof(num_data_)) +
VirtualFileWriter::AlignedSize(sizeof(num_weights_)) +
VirtualFileWriter::AlignedSize(sizeof(num_queries_));
size += VirtualFileWriter::AlignedSize(sizeof(label_t) * num_data_);
if (!weights_.empty()) {
size += VirtualFileWriter::AlignedSize(sizeof(label_t) * num_weights_);
}
if (!query_boundaries_.empty()) {
size += VirtualFileWriter::AlignedSize(sizeof(data_size_t) *
(num_queries_ + 1));
}
return size;
}
} // namespace LightGBM
src/io/multi_val_dense_bin.hpp
+360
View File
@@ -0,0 +1,360 @@
/*!
* Copyright (c) 2020-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2020-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_IO_MULTI_VAL_DENSE_BIN_HPP_
#define LIGHTGBM_SRC_IO_MULTI_VAL_DENSE_BIN_HPP_
#include <LightGBM/bin.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <LightGBM/utils/threading.h>
#include <algorithm>
#include <cstdint>
#include <cstring>
#include <vector>
namespace LightGBM {
template <typename VAL_T>
class MultiValDenseBin : public MultiValBin {
public:
explicit MultiValDenseBin(data_size_t num_data, int num_bin, int num_feature,
const std::vector<uint32_t>& offsets)
: num_data_(num_data), num_bin_(num_bin), num_feature_(num_feature),
offsets_(offsets) {
data_.resize(static_cast<size_t>(num_data_) * num_feature_, static_cast<VAL_T>(0));
}
~MultiValDenseBin() {
}
data_size_t num_data() const override {
return num_data_;
}
int num_bin() const override {
return num_bin_;
}
double num_element_per_row() const override { return num_feature_; }
const std::vector<uint32_t>& offsets() const override { return offsets_; }
void PushOneRow(int , data_size_t idx, const std::vector<uint32_t>& values) override {
auto start = RowPtr(idx);
for (auto i = 0; i < num_feature_; ++i) {
data_[start + i] = static_cast<VAL_T>(values[i]);
}
}
void FinishLoad() override {
}
bool IsSparse() override {
return false;
}
template<bool USE_INDICES, bool USE_PREFETCH, bool ORDERED>
void ConstructHistogramInner(const data_size_t* data_indices, data_size_t start, data_size_t end,
const score_t* gradients, const score_t* hessians, hist_t* out) const {
data_size_t i = start;
hist_t* grad = out;
hist_t* hess = out + 1;
if (USE_PREFETCH) {
const data_size_t pf_offset = 32 / sizeof(VAL_T);
const data_size_t pf_end = end - pf_offset;
for (; i < pf_end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto pf_idx = USE_INDICES ? data_indices[i + pf_offset] : i + pf_offset;
if (!ORDERED) {
PREFETCH_T0(gradients + pf_idx);
PREFETCH_T0(hessians + pf_idx);
}
PREFETCH_T0(data_.data() + RowPtr(pf_idx));
const auto j_start = RowPtr(idx);
const VAL_T* data_ptr = data_.data() + j_start;
const score_t gradient = ORDERED ? gradients[i] : gradients[idx];
const score_t hessian = ORDERED ? hessians[i] : hessians[idx];
for (int j = 0; j < num_feature_; ++j) {
const uint32_t bin = static_cast<uint32_t>(data_ptr[j]);
const auto ti = (bin + offsets_[j]) << 1;
grad[ti] += gradient;
hess[ti] += hessian;
}
}
}
for (; i < end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto j_start = RowPtr(idx);
const VAL_T* data_ptr = data_.data() + j_start;
const score_t gradient = ORDERED ? gradients[i] : gradients[idx];
const score_t hessian = ORDERED ? hessians[i] : hessians[idx];
for (int j = 0; j < num_feature_; ++j) {
const uint32_t bin = static_cast<uint32_t>(data_ptr[j]);
const auto ti = (bin + offsets_[j]) << 1;
grad[ti] += gradient;
hess[ti] += hessian;
}
}
}
void ConstructHistogram(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* gradients,
const score_t* hessians, hist_t* out) const override {
ConstructHistogramInner<true, true, false>(data_indices, start, end,
gradients, hessians, out);
}
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* gradients, const score_t* hessians,
hist_t* out) const override {
ConstructHistogramInner<false, false, false>(
nullptr, start, end, gradients, hessians, out);
}
void ConstructHistogramOrdered(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients,
const score_t* hessians,
hist_t* out) const override {
ConstructHistogramInner<true, true, true>(data_indices, start, end,
gradients, hessians, out);
}
template<bool USE_INDICES, bool USE_PREFETCH, bool ORDERED, typename PACKED_HIST_T, int HIST_BITS>
void ConstructHistogramIntInner(const data_size_t* data_indices, data_size_t start, data_size_t end,
const score_t* gradients_and_hessians, hist_t* out) const {
data_size_t i = start;
const VAL_T* data_ptr_base = data_.data();
const int16_t* gradients_and_hessians_ptr = reinterpret_cast<const int16_t*>(gradients_and_hessians);
PACKED_HIST_T* out_ptr = reinterpret_cast<PACKED_HIST_T*>(out);
if (USE_PREFETCH) {
const data_size_t pf_offset = 32 / sizeof(VAL_T);
const data_size_t pf_end = end - pf_offset;
for (; i < pf_end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto pf_idx = USE_INDICES ? data_indices[i + pf_offset] : i + pf_offset;
if (!ORDERED) {
PREFETCH_T0(gradients_and_hessians_ptr + pf_idx);
}
PREFETCH_T0(data_ptr_base + RowPtr(pf_idx));
const auto j_start = RowPtr(idx);
const VAL_T* data_ptr = data_ptr_base + j_start;
const int16_t gradient_16 = gradients_and_hessians_ptr[idx];
const PACKED_HIST_T gradient_packed = (HIST_BITS == 8) ? gradient_16 :
((static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) |
static_cast<PACKED_HIST_T>(gradient_16 & 0xff));
for (int j = 0; j < num_feature_; ++j) {
const uint32_t bin = static_cast<uint32_t>(data_ptr[j]);
const auto ti = (bin + offsets_[j]);
out_ptr[ti] += gradient_packed;
}
}
}
for (; i < end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto j_start = RowPtr(idx);
const VAL_T* data_ptr = data_ptr_base + j_start;
const int16_t gradient_16 = gradients_and_hessians_ptr[idx];
const PACKED_HIST_T gradient_packed = (HIST_BITS == 8) ? gradient_16 :
((static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) |
static_cast<PACKED_HIST_T>(gradient_16 & 0xff));
for (int j = 0; j < num_feature_; ++j) {
const uint32_t bin = static_cast<uint32_t>(data_ptr[j]);
const auto ti = (bin + offsets_[j]);
out_ptr[ti] += gradient_packed;
}
}
}
void ConstructHistogramInt32(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* gradients,
const score_t* /*hessians*/, hist_t* out) const override {
ConstructHistogramIntInner<true, true, false, int64_t, 32>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt32(data_size_t start, data_size_t end,
const score_t* gradients, const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, false, int64_t, 32>(
nullptr, start, end, gradients, out);
}
void ConstructHistogramOrderedInt32(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients,
const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, true, int64_t, 32>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt16(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* gradients,
const score_t* /*hessians*/, hist_t* out) const override {
ConstructHistogramIntInner<true, true, false, int32_t, 16>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt16(data_size_t start, data_size_t end,
const score_t* gradients, const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, false, int32_t, 16>(
nullptr, start, end, gradients, out);
}
void ConstructHistogramOrderedInt16(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients,
const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, true, int32_t, 16>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt8(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* gradients,
const score_t* /*hessians*/, hist_t* out) const override {
ConstructHistogramIntInner<true, true, false, int16_t, 8>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt8(data_size_t start, data_size_t end,
const score_t* gradients, const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, false, int16_t, 8>(
nullptr, start, end, gradients, out);
}
void ConstructHistogramOrderedInt8(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients,
const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, true, int16_t, 8>(data_indices, start, end,
gradients, out);
}
MultiValBin* CreateLike(data_size_t num_data, int num_bin, int num_feature, double,
const std::vector<uint32_t>& offsets) const override {
return new MultiValDenseBin<VAL_T>(num_data, num_bin, num_feature, offsets);
}
void ReSize(data_size_t num_data, int num_bin, int num_feature,
double, const std::vector<uint32_t>& offsets) override {
num_data_ = num_data;
num_bin_ = num_bin;
num_feature_ = num_feature;
offsets_ = offsets;
size_t new_size = static_cast<size_t>(num_feature_) * num_data_;
if (data_.size() < new_size) {
data_.resize(new_size, 0);
}
}
template <bool SUBROW, bool SUBCOL>
void CopyInner(const MultiValBin* full_bin, const data_size_t* used_indices,
data_size_t num_used_indices,
const std::vector<int>& used_feature_index) {
const auto other_bin =
reinterpret_cast<const MultiValDenseBin<VAL_T>*>(full_bin);
if (SUBROW) {
CHECK_EQ(num_data_, num_used_indices);
}
int n_block = 1;
data_size_t block_size = num_data_;
Threading::BlockInfo<data_size_t>(num_data_, 1024, &n_block,
&block_size);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 1)
for (int tid = 0; tid < n_block; ++tid) {
data_size_t start = tid * block_size;
data_size_t end = std::min(num_data_, start + block_size);
for (data_size_t i = start; i < end; ++i) {
const auto j_start = RowPtr(i);
const auto other_j_start =
SUBROW ? other_bin->RowPtr(used_indices[i]) : other_bin->RowPtr(i);
for (int j = 0; j < num_feature_; ++j) {
if (SUBCOL) {
if (other_bin->data_[other_j_start + used_feature_index[j]] > 0) {
data_[j_start + j] = static_cast<VAL_T>(
other_bin->data_[other_j_start + used_feature_index[j]]);
} else {
data_[j_start + j] = 0;
}
} else {
data_[j_start + j] =
static_cast<VAL_T>(other_bin->data_[other_j_start + j]);
}
}
}
}
}
void CopySubrow(const MultiValBin* full_bin, const data_size_t* used_indices,
data_size_t num_used_indices) override {
CopyInner<true, false>(full_bin, used_indices, num_used_indices,
std::vector<int>());
}
void CopySubcol(const MultiValBin* full_bin,
const std::vector<int>& used_feature_index,
const std::vector<uint32_t>&,
const std::vector<uint32_t>&,
const std::vector<uint32_t>&) override {
CopyInner<false, true>(full_bin, nullptr, num_data_, used_feature_index);
}
void CopySubrowAndSubcol(const MultiValBin* full_bin,
const data_size_t* used_indices,
data_size_t num_used_indices,
const std::vector<int>& used_feature_index,
const std::vector<uint32_t>&,
const std::vector<uint32_t>&,
const std::vector<uint32_t>&) override {
CopyInner<true, true>(full_bin, used_indices, num_used_indices,
used_feature_index);
}
inline size_t RowPtr(data_size_t idx) const {
return static_cast<size_t>(idx) * num_feature_;
}
MultiValDenseBin<VAL_T>* Clone() override;
#ifdef USE_CUDA
const void* GetRowWiseData(uint8_t* bit_type,
size_t* total_size,
bool* is_sparse,
const void** out_data_ptr,
uint8_t* data_ptr_bit_type) const override;
#endif // USE_CUDA
private:
data_size_t num_data_;
int num_bin_;
int num_feature_;
std::vector<uint32_t> offsets_;
std::vector<VAL_T, Common::AlignmentAllocator<VAL_T, 32>> data_;
MultiValDenseBin(const MultiValDenseBin<VAL_T>& other)
: num_data_(other.num_data_), num_bin_(other.num_bin_), num_feature_(other.num_feature_),
offsets_(other.offsets_), data_(other.data_) {
}
};
template<typename VAL_T>
MultiValDenseBin<VAL_T>* MultiValDenseBin<VAL_T>::Clone() {
return new MultiValDenseBin<VAL_T>(*this);
}
} // namespace LightGBM
#endif // LIGHTGBM_SRC_IO_MULTI_VAL_DENSE_BIN_HPP_
src/io/multi_val_sparse_bin.hpp
+449
View File
@@ -0,0 +1,449 @@
/*!
* Copyright (c) 2020-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2020-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_IO_MULTI_VAL_SPARSE_BIN_HPP_
#define LIGHTGBM_SRC_IO_MULTI_VAL_SPARSE_BIN_HPP_
#include <LightGBM/bin.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <LightGBM/utils/threading.h>
#include <algorithm>
#include <cstdint>
#include <cstring>
#include <vector>
namespace LightGBM {
template <typename INDEX_T, typename VAL_T>
class MultiValSparseBin : public MultiValBin {
public:
explicit MultiValSparseBin(data_size_t num_data, int num_bin,
double estimate_element_per_row)
: num_data_(num_data),
num_bin_(num_bin),
estimate_element_per_row_(estimate_element_per_row) {
row_ptr_.resize(num_data_ + 1, 0);
INDEX_T estimate_num_data = static_cast<INDEX_T>(estimate_element_per_row_ * 1.1 * num_data_);
int num_threads = OMP_NUM_THREADS();
if (num_threads > 1) {
t_data_.resize(num_threads - 1);
for (size_t i = 0; i < t_data_.size(); ++i) {
t_data_[i].resize(estimate_num_data / num_threads);
}
}
t_size_.resize(num_threads, 0);
data_.resize(estimate_num_data / num_threads);
}
~MultiValSparseBin() {}
data_size_t num_data() const override { return num_data_; }
int num_bin() const override { return num_bin_; }
double num_element_per_row() const override {
return estimate_element_per_row_;
}
const std::vector<uint32_t>& offsets() const override { return offsets_; }
void PushOneRow(int tid, data_size_t idx,
const std::vector<uint32_t>& values) override {
const int pre_alloc_size = 50;
row_ptr_[idx + 1] = static_cast<INDEX_T>(values.size());
if (tid == 0) {
if (t_size_[tid] + row_ptr_[idx + 1] >
static_cast<INDEX_T>(data_.size())) {
data_.resize(t_size_[tid] + row_ptr_[idx + 1] * pre_alloc_size);
}
for (auto val : values) {
data_[t_size_[tid]++] = static_cast<VAL_T>(val);
}
} else {
if (t_size_[tid] + row_ptr_[idx + 1] >
static_cast<INDEX_T>(t_data_[tid - 1].size())) {
t_data_[tid - 1].resize(t_size_[tid] +
row_ptr_[idx + 1] * pre_alloc_size);
}
for (auto val : values) {
t_data_[tid - 1][t_size_[tid]++] = static_cast<VAL_T>(val);
}
}
}
void MergeData(const INDEX_T* sizes) {
Common::FunctionTimer fun_time("MultiValSparseBin::MergeData", global_timer);
for (data_size_t i = 0; i < num_data_; ++i) {
row_ptr_[i + 1] += row_ptr_[i];
}
if (t_data_.size() > 0) {
std::vector<INDEX_T> offsets(1 + t_data_.size());
offsets[0] = sizes[0];
for (size_t tid = 0; tid < t_data_.size() - 1; ++tid) {
offsets[tid + 1] = offsets[tid] + sizes[tid + 1];
}
data_.resize(row_ptr_[num_data_]);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 1)
for (int tid = 0; tid < static_cast<int>(t_data_.size()); ++tid) {
std::copy_n(t_data_[tid].data(), sizes[tid + 1],
data_.data() + offsets[tid]);
}
} else {
data_.resize(row_ptr_[num_data_]);
}
}
void FinishLoad() override {
MergeData(t_size_.data());
t_size_.clear();
row_ptr_.shrink_to_fit();
data_.shrink_to_fit();
t_data_.clear();
t_data_.shrink_to_fit();
// update estimate_element_per_row_ by all data
estimate_element_per_row_ =
static_cast<double>(row_ptr_[num_data_]) / num_data_;
}
bool IsSparse() override { return true; }
template <bool USE_INDICES, bool USE_PREFETCH, bool ORDERED>
void ConstructHistogramInner(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients,
const score_t* hessians, hist_t* out) const {
data_size_t i = start;
hist_t* grad = out;
hist_t* hess = out + 1;
const VAL_T* data_ptr = data_.data();
if (USE_PREFETCH) {
const data_size_t pf_offset = 32 / sizeof(VAL_T);
const data_size_t pf_end = end - pf_offset;
for (; i < pf_end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto pf_idx =
USE_INDICES ? data_indices[i + pf_offset] : i + pf_offset;
if (!ORDERED) {
PREFETCH_T0(gradients + pf_idx);
PREFETCH_T0(hessians + pf_idx);
}
PREFETCH_T0(row_ptr_.data() + pf_idx);
PREFETCH_T0(data_ptr + row_ptr_[pf_idx]);
const auto j_start = RowPtr(idx);
const auto j_end = RowPtr(idx + 1);
const score_t gradient = ORDERED ? gradients[i] : gradients[idx];
const score_t hessian = ORDERED ? hessians[i] : hessians[idx];
for (auto j = j_start; j < j_end; ++j) {
const auto ti = static_cast<uint32_t>(data_ptr[j]) << 1;
grad[ti] += gradient;
hess[ti] += hessian;
}
}
}
for (; i < end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto j_start = RowPtr(idx);
const auto j_end = RowPtr(idx + 1);
const score_t gradient = ORDERED ? gradients[i] : gradients[idx];
const score_t hessian = ORDERED ? hessians[i] : hessians[idx];
for (auto j = j_start; j < j_end; ++j) {
const auto ti = static_cast<uint32_t>(data_ptr[j]) << 1;
grad[ti] += gradient;
hess[ti] += hessian;
}
}
}
void ConstructHistogram(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* gradients,
const score_t* hessians, hist_t* out) const override {
ConstructHistogramInner<true, true, false>(data_indices, start, end,
gradients, hessians, out);
}
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* gradients, const score_t* hessians,
hist_t* out) const override {
ConstructHistogramInner<false, false, false>(
nullptr, start, end, gradients, hessians, out);
}
void ConstructHistogramOrdered(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients,
const score_t* hessians,
hist_t* out) const override {
ConstructHistogramInner<true, true, true>(data_indices, start, end,
gradients, hessians, out);
}
template <bool USE_INDICES, bool USE_PREFETCH, bool ORDERED, typename PACKED_HIST_T, int HIST_BITS>
void ConstructHistogramIntInner(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients_and_hessians, hist_t* out) const {
data_size_t i = start;
PACKED_HIST_T* out_ptr = reinterpret_cast<PACKED_HIST_T*>(out);
const int16_t* gradients_and_hessians_ptr = reinterpret_cast<const int16_t*>(gradients_and_hessians);
const VAL_T* data_ptr = data_.data();
const INDEX_T* row_ptr_base = row_ptr_.data();
if (USE_PREFETCH) {
const data_size_t pf_offset = 32 / sizeof(VAL_T);
const data_size_t pf_end = end - pf_offset;
for (; i < pf_end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto pf_idx =
USE_INDICES ? data_indices[i + pf_offset] : i + pf_offset;
if (!ORDERED) {
PREFETCH_T0(gradients_and_hessians_ptr + pf_idx);
}
PREFETCH_T0(row_ptr_base + pf_idx);
PREFETCH_T0(data_ptr + row_ptr_[pf_idx]);
const auto j_start = RowPtr(idx);
const auto j_end = RowPtr(idx + 1);
const int16_t gradient_16 = ORDERED ? gradients_and_hessians_ptr[i] : gradients_and_hessians_ptr[idx];
const PACKED_HIST_T gradient_packed = (HIST_BITS == 8) ? gradient_16 :
((static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) |
static_cast<PACKED_HIST_T>(gradient_16 & 0xff));
for (auto j = j_start; j < j_end; ++j) {
const auto ti = static_cast<uint32_t>(data_ptr[j]);
out_ptr[ti] += gradient_packed;
}
}
}
for (; i < end; ++i) {
const auto idx = USE_INDICES ? data_indices[i] : i;
const auto j_start = RowPtr(idx);
const auto j_end = RowPtr(idx + 1);
const int16_t gradient_16 = ORDERED ? gradients_and_hessians_ptr[i] : gradients_and_hessians_ptr[idx];
const PACKED_HIST_T gradient_packed = (HIST_BITS == 8) ? gradient_16 :
((static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) |
static_cast<PACKED_HIST_T>(gradient_16 & 0xff));
for (auto j = j_start; j < j_end; ++j) {
const auto ti = static_cast<uint32_t>(data_ptr[j]);
out_ptr[ti] += gradient_packed;
}
}
}
void ConstructHistogramInt32(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* gradients,
const score_t* /*hessians*/, hist_t* out) const override {
ConstructHistogramIntInner<true, true, false, int64_t, 32>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt32(data_size_t start, data_size_t end,
const score_t* gradients, const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, false, int64_t, 32>(
nullptr, start, end, gradients, out);
}
void ConstructHistogramOrderedInt32(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients,
const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, true, int64_t, 32>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt16(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* gradients,
const score_t* /*hessians*/, hist_t* out) const override {
ConstructHistogramIntInner<true, true, false, int32_t, 16>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt16(data_size_t start, data_size_t end,
const score_t* gradients, const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, false, int32_t, 16>(
nullptr, start, end, gradients, out);
}
void ConstructHistogramOrderedInt16(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients,
const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, true, int32_t, 16>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt8(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* gradients,
const score_t* /*hessians*/, hist_t* out) const override {
ConstructHistogramIntInner<true, true, false, int16_t, 8>(data_indices, start, end,
gradients, out);
}
void ConstructHistogramInt8(data_size_t start, data_size_t end,
const score_t* gradients, const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<false, false, false, int16_t, 8>(
nullptr, start, end, gradients, out);
}
void ConstructHistogramOrderedInt8(const data_size_t* data_indices,
data_size_t start, data_size_t end,
const score_t* gradients,
const score_t* /*hessians*/,
hist_t* out) const override {
ConstructHistogramIntInner<true, true, true, int16_t, 8>(data_indices, start, end,
gradients, out);
}
MultiValBin* CreateLike(data_size_t num_data, int num_bin, int,
double estimate_element_per_row,
const std::vector<uint32_t>& /*offsets*/) const override {
return new MultiValSparseBin<INDEX_T, VAL_T>(num_data, num_bin,
estimate_element_per_row);
}
void ReSize(data_size_t num_data, int num_bin, int,
double estimate_element_per_row, const std::vector<uint32_t>& /*offsets*/) override {
num_data_ = num_data;
num_bin_ = num_bin;
estimate_element_per_row_ = estimate_element_per_row;
INDEX_T estimate_num_data =
static_cast<INDEX_T>(estimate_element_per_row_ * 1.1 * num_data_);
size_t npart = 1 + t_data_.size();
INDEX_T avg_num_data = static_cast<INDEX_T>(estimate_num_data / npart);
if (static_cast<INDEX_T>(data_.size()) < avg_num_data) {
data_.resize(avg_num_data, 0);
}
for (size_t i = 0; i < t_data_.size(); ++i) {
if (static_cast<INDEX_T>(t_data_[i].size()) < avg_num_data) {
t_data_[i].resize(avg_num_data, 0);
}
}
if (num_data_ + 1 > static_cast<data_size_t>(row_ptr_.size())) {
row_ptr_.resize(num_data_ + 1);
}
}
template <bool SUBROW, bool SUBCOL>
void CopyInner(const MultiValBin* full_bin, const data_size_t* used_indices,
data_size_t num_used_indices,
const std::vector<uint32_t>& lower,
const std::vector<uint32_t>& upper,
const std::vector<uint32_t>& delta) {
const auto other =
reinterpret_cast<const MultiValSparseBin<INDEX_T, VAL_T>*>(full_bin);
if (SUBROW) {
CHECK_EQ(num_data_, num_used_indices);
}
int n_block = 1;
data_size_t block_size = num_data_;
Threading::BlockInfo<data_size_t>(static_cast<int>(t_data_.size() + 1),
num_data_, 1024, &n_block, &block_size);
std::vector<INDEX_T> sizes(t_data_.size() + 1, 0);
const int pre_alloc_size = 50;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 1)
for (int tid = 0; tid < n_block; ++tid) {
data_size_t start = tid * block_size;
data_size_t end = std::min(num_data_, start + block_size);
auto& buf = (tid == 0) ? data_ : t_data_[tid - 1];
INDEX_T size = 0;
for (data_size_t i = start; i < end; ++i) {
const auto j_start =
SUBROW ? other->RowPtr(used_indices[i]) : other->RowPtr(i);
const auto j_end =
SUBROW ? other->RowPtr(used_indices[i] + 1) : other->RowPtr(i + 1);
if (size + (j_end - j_start) > static_cast<INDEX_T>(buf.size())) {
buf.resize(size + (j_end - j_start) * pre_alloc_size);
}
int k = 0;
const auto pre_size = size;
for (auto j = j_start; j < j_end; ++j) {
const auto val = other->data_[j];
if (SUBCOL) {
while (val >= upper[k]) {
++k;
}
if (val >= lower[k]) {
buf[size++] = static_cast<VAL_T>(val - delta[k]);
}
} else {
buf[size++] = val;
}
}
row_ptr_[i + 1] = size - pre_size;
}
sizes[tid] = size;
}
MergeData(sizes.data());
}
void CopySubrow(const MultiValBin* full_bin, const data_size_t* used_indices,
data_size_t num_used_indices) override {
CopyInner<true, false>(full_bin, used_indices, num_used_indices,
std::vector<uint32_t>(), std::vector<uint32_t>(),
std::vector<uint32_t>());
}
void CopySubcol(const MultiValBin* full_bin, const std::vector<int>&,
const std::vector<uint32_t>& lower,
const std::vector<uint32_t>& upper,
const std::vector<uint32_t>& delta) override {
CopyInner<false, true>(full_bin, nullptr, num_data_, lower, upper, delta);
}
void CopySubrowAndSubcol(const MultiValBin* full_bin,
const data_size_t* used_indices,
data_size_t num_used_indices,
const std::vector<int>&,
const std::vector<uint32_t>& lower,
const std::vector<uint32_t>& upper,
const std::vector<uint32_t>& delta) override {
CopyInner<true, true>(full_bin, used_indices, num_used_indices, lower,
upper, delta);
}
inline INDEX_T RowPtr(data_size_t idx) const { return row_ptr_[idx]; }
MultiValSparseBin<INDEX_T, VAL_T>* Clone() override;
#ifdef USE_CUDA
const void* GetRowWiseData(uint8_t* bit_type,
size_t* total_size,
bool* is_sparse,
const void** out_data_ptr,
uint8_t* data_ptr_bit_type) const override;
#endif // USE_CUDA
private:
data_size_t num_data_;
int num_bin_;
double estimate_element_per_row_;
std::vector<VAL_T, Common::AlignmentAllocator<VAL_T, 32>> data_;
std::vector<INDEX_T, Common::AlignmentAllocator<INDEX_T, 32>>
row_ptr_;
std::vector<std::vector<VAL_T, Common::AlignmentAllocator<VAL_T, 32>>>
t_data_;
std::vector<INDEX_T> t_size_;
std::vector<uint32_t> offsets_;
MultiValSparseBin(const MultiValSparseBin<INDEX_T, VAL_T>& other)
: num_data_(other.num_data_),
num_bin_(other.num_bin_),
estimate_element_per_row_(other.estimate_element_per_row_),
data_(other.data_),
row_ptr_(other.row_ptr_) {}
};
template <typename INDEX_T, typename VAL_T>
MultiValSparseBin<INDEX_T, VAL_T>* MultiValSparseBin<INDEX_T, VAL_T>::Clone() {
return new MultiValSparseBin<INDEX_T, VAL_T>(*this);
}
} // namespace LightGBM
#endif // LIGHTGBM_SRC_IO_MULTI_VAL_SPARSE_BIN_HPP_
src/io/parser.cpp
+319
View File
@@ -0,0 +1,319 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include "parser.hpp"
#include <algorithm>
#include <functional>
#include <map>
#include <memory>
#include <string>
#include <vector>
namespace LightGBM {
void GetStatistic(const char* str, int* comma_cnt, int* tab_cnt, int* colon_cnt) {
*comma_cnt = 0;
*tab_cnt = 0;
*colon_cnt = 0;
for (int i = 0; str[i] != '\0'; ++i) {
if (str[i] == ',') {
++(*comma_cnt);
} else if (str[i] == '\t') {
++(*tab_cnt);
} else if (str[i] == ':') {
++(*colon_cnt);
}
}
}
int GetLabelIdxForLibsvm(const std::string& str, int num_features, int label_idx) {
if (num_features <= 0) {
return label_idx;
}
auto str2 = Common::Trim(str);
auto pos_space = str2.find_first_of(" \f\n\r\t\v");
auto pos_colon = str2.find_first_of(":");
if (pos_space == std::string::npos || pos_space < pos_colon) {
return label_idx;
} else {
return -1;
}
}
int GetLabelIdxForTSV(const std::string& str, int num_features, int label_idx) {
if (num_features <= 0) {
return label_idx;
}
auto str2 = Common::Trim(str);
auto tokens = Common::Split(str2.c_str(), '\t');
if (static_cast<int>(tokens.size()) == num_features) {
return -1;
} else {
return label_idx;
}
}
int GetLabelIdxForCSV(const std::string& str, int num_features, int label_idx) {
if (num_features <= 0) {
return label_idx;
}
auto str2 = Common::Trim(str);
auto tokens = Common::Split(str2.c_str(), ',');
if (static_cast<int>(tokens.size()) == num_features) {
return -1;
} else {
return label_idx;
}
}
enum DataType {
INVALID,
CSV,
TSV,
LIBSVM
};
void GetLine(std::stringstream* ss, std::string* line, const VirtualFileReader* reader, std::vector<char>* buffer, size_t buffer_size) {
std::getline(*ss, *line);
while (ss->eof()) {
size_t read_len = reader->Read(buffer->data(), buffer_size);
if (read_len <= 0) {
break;
}
ss->clear();
ss->str(std::string(buffer->data(), read_len));
std::string tmp;
std::getline(*ss, tmp);
*line += tmp;
}
}
std::vector<std::string> ReadKLineFromFile(const char* filename, bool header, int k) {
auto reader = VirtualFileReader::Make(filename);
if (!reader->Init()) {
Log::Fatal("Data file %s doesn't exist.", filename);
}
std::vector<std::string> ret;
std::string cur_line;
const size_t buffer_size = 1024 * 1024;
auto buffer = std::vector<char>(buffer_size);
size_t read_len = reader->Read(buffer.data(), buffer_size);
if (read_len <= 0) {
Log::Fatal("Data file %s couldn't be read.", filename);
}
std::string read_str = std::string(buffer.data(), read_len);
std::stringstream tmp_file(read_str);
if (header) {
if (!tmp_file.eof()) {
GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size);
}
}
for (int i = 0; i < k; ++i) {
if (!tmp_file.eof()) {
GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size);
cur_line = Common::Trim(cur_line);
if (!cur_line.empty()) {
ret.push_back(cur_line);
}
} else {
break;
}
}
if (ret.empty()) {
Log::Fatal("Data file %s should have at least one line.", filename);
} else if (ret.size() == 1) {
Log::Warning("Data file %s only has one line.", filename);
}
return ret;
}
int GetNumColFromLIBSVMFile(const char* filename, bool header) {
auto reader = VirtualFileReader::Make(filename);
if (!reader->Init()) {
Log::Fatal("Data file %s doesn't exist.", filename);
}
std::vector<std::string> ret;
std::string cur_line;
const size_t buffer_size = 1024 * 1024;
auto buffer = std::vector<char>(buffer_size);
size_t read_len = reader->Read(buffer.data(), buffer_size);
if (read_len <= 0) {
Log::Fatal("Data file %s couldn't be read.", filename);
}
std::string read_str = std::string(buffer.data(), read_len);
std::stringstream tmp_file(read_str);
if (header) {
if (!tmp_file.eof()) {
GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size);
}
}
int max_col_idx = 0;
int max_line_idx = 0;
const int stop_round = 1 << 7;
const int max_line = 1 << 13;
for (int i = 0; i < max_line; ++i) {
if (!tmp_file.eof()) {
GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size);
cur_line = Common::Trim(cur_line);
auto colon_pos = cur_line.find_last_of(":");
auto space_pos = cur_line.find_last_of(" \f\t\v");
auto sub_str = cur_line.substr(space_pos + 1, space_pos - colon_pos - 1);
int cur_idx = 0;
Common::Atoi(sub_str.c_str(), &cur_idx);
if (cur_idx > max_col_idx) {
max_col_idx = cur_idx;
max_line_idx = i;
}
if (i - max_line_idx >= stop_round) {
break;
}
} else {
break;
}
}
CHECK_GT(max_col_idx, 0);
return max_col_idx;
}
DataType GetDataType(const char* filename, bool header,
const std::vector<std::string>& lines, int* num_col) {
DataType type = DataType::INVALID;
if (lines.empty()) {
return type;
}
int comma_cnt = 0;
int tab_cnt = 0;
int colon_cnt = 0;
GetStatistic(lines[0].c_str(), &comma_cnt, &tab_cnt, &colon_cnt);
size_t num_lines = lines.size();
if (num_lines == 1) {
if (colon_cnt > 0) {
type = DataType::LIBSVM;
} else if (tab_cnt > 0) {
type = DataType::TSV;
} else if (comma_cnt > 0) {
type = DataType::CSV;
}
} else {
int comma_cnt2 = 0;
int tab_cnt2 = 0;
int colon_cnt2 = 0;
GetStatistic(lines[1].c_str(), &comma_cnt2, &tab_cnt2, &colon_cnt2);
if (colon_cnt > 0 || colon_cnt2 > 0) {
type = DataType::LIBSVM;
} else if (tab_cnt == tab_cnt2 && tab_cnt > 0) {
type = DataType::TSV;
} else if (comma_cnt == comma_cnt2 && comma_cnt > 0) {
type = DataType::CSV;
}
if (type == DataType::TSV || type == DataType::CSV) {
// valid the type
for (size_t i = 2; i < num_lines; ++i) {
GetStatistic(lines[i].c_str(), &comma_cnt2, &tab_cnt2, &colon_cnt2);
if (type == DataType::TSV && tab_cnt2 != tab_cnt) {
type = DataType::INVALID;
break;
} else if (type == DataType::CSV && comma_cnt != comma_cnt2) {
type = DataType::INVALID;
break;
}
}
}
}
if (type == DataType::LIBSVM) {
int max_col_idx = GetNumColFromLIBSVMFile(filename, header);
*num_col = max_col_idx + 1;
} else if (type == DataType::CSV) {
*num_col = comma_cnt + 1;
} else if (type == DataType::TSV) {
*num_col = tab_cnt + 1;
}
return type;
}
// parser factory implementation.
ParserFactory& ParserFactory::getInstance() {
static ParserFactory factory;
return factory;
}
void ParserFactory::Register(std::string class_name, std::function<Parser*(std::string)> m_objc) {
if (m_objc) {
object_map_.insert(
std::map<std::string, std::function<Parser*(std::string)>>::value_type(class_name, m_objc));
}
}
Parser* ParserFactory::getObject(std::string class_name, std::string config_str) {
std::map<std::string, std::function<Parser*(std::string)>>::const_iterator iter =
object_map_.find(class_name);
if (iter != object_map_.end()) {
return iter->second(config_str);
} else {
Log::Fatal("Cannot find parser class '%s', please register first or check config format.", class_name.c_str());
return nullptr;
}
}
Parser* Parser::CreateParser(const char* filename, bool header, int num_features, int label_idx, bool precise_float_parser) {
const int n_read_line = 32;
auto lines = ReadKLineFromFile(filename, header, n_read_line);
int num_col = 0;
DataType type = GetDataType(filename, header, lines, &num_col);
if (type == DataType::INVALID) {
Log::Fatal("Unknown format of training data. Only CSV, TSV, and LibSVM (zero-based) formatted text files are supported.");
}
std::unique_ptr<Parser> ret;
int output_label_index = -1;
AtofFunc atof = precise_float_parser ? Common::AtofPrecise : Common::Atof;
if (type == DataType::LIBSVM) {
output_label_index = GetLabelIdxForLibsvm(lines[0], num_features, label_idx);
ret.reset(new LibSVMParser(output_label_index, num_col, atof));
} else if (type == DataType::TSV) {
output_label_index = GetLabelIdxForTSV(lines[0], num_features, label_idx);
ret.reset(new TSVParser(output_label_index, num_col, atof));
} else if (type == DataType::CSV) {
output_label_index = GetLabelIdxForCSV(lines[0], num_features, label_idx);
ret.reset(new CSVParser(output_label_index, num_col, atof));
}
if (output_label_index < 0 && label_idx >= 0) {
Log::Info("Data file %s doesn't contain a label column.", filename);
}
return ret.release();
}
Parser* Parser::CreateParser(const char* filename, bool header, int num_features, int label_idx, bool precise_float_parser, std::string parser_config_str) {
// customized parser add-on.
if (!parser_config_str.empty()) {
std::unique_ptr<Parser> ret;
std::string class_name = Common::GetFromParserConfig(parser_config_str, "className");
Log::Info("Custom parser class name: %s", class_name.c_str());
Parser* p = ParserFactory::getInstance().getObject(class_name, parser_config_str);
ret.reset(p);
return ret.release();
}
return CreateParser(filename, header, num_features, label_idx, precise_float_parser);
}
std::string Parser::GenerateParserConfigStr(const char* filename, const char* parser_config_filename, bool header, int label_idx) {
TextReader<data_size_t> parser_config_reader(parser_config_filename, false);
parser_config_reader.ReadAllLines();
std::string parser_config_str = parser_config_reader.JoinedLines();
if (!parser_config_str.empty()) {
// save header to parser config in case needed.
if (header && Common::GetFromParserConfig(parser_config_str, "header").empty()) {
TextReader<data_size_t> text_reader(filename, header);
parser_config_str = Common::SaveToParserConfig(parser_config_str, "header", text_reader.first_line());
}
// save label id to parser config in case needed.
if (Common::GetFromParserConfig(parser_config_str, "labelId").empty()) {
parser_config_str = Common::SaveToParserConfig(parser_config_str, "labelId", std::to_string(label_idx));
}
}
return parser_config_str;
}
} // namespace LightGBM
src/io/parser.hpp
+136
View File
@@ -0,0 +1,136 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_IO_PARSER_HPP_
#define LIGHTGBM_SRC_IO_PARSER_HPP_
#include <LightGBM/dataset.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/log.h>
#include <unordered_map>
#include <utility>
#include <vector>
namespace LightGBM {
class CSVParser: public Parser {
public:
explicit CSVParser(int label_idx, int total_columns, AtofFunc atof)
:label_idx_(label_idx), total_columns_(total_columns), atof_(atof) {
}
inline void ParseOneLine(const char* str,
std::vector<std::pair<int, double>>* out_features, double* out_label) const override {
int idx = 0;
double val = 0.0f;
int offset = 0;
*out_label = 0.0f;
while (*str != '\0') {
str = atof_(str, &val);
if (idx == label_idx_) {
*out_label = val;
offset = -1;
} else if (std::fabs(val) > kZeroThreshold || std::isnan(val)) {
out_features->emplace_back(idx + offset, val);
}
++idx;
if (*str == ',') {
++str;
} else if (*str != '\0') {
Log::Fatal("Input format error when parsing as CSV");
}
}
}
inline int NumFeatures() const override {
return total_columns_ - (label_idx_ >= 0);
}
private:
int label_idx_ = 0;
int total_columns_ = -1;
AtofFunc atof_;
};
class TSVParser: public Parser {
public:
explicit TSVParser(int label_idx, int total_columns, AtofFunc atof)
:label_idx_(label_idx), total_columns_(total_columns), atof_(atof) {
}
inline void ParseOneLine(const char* str,
std::vector<std::pair<int, double>>* out_features, double* out_label) const override {
int idx = 0;
double val = 0.0f;
int offset = 0;
while (*str != '\0') {
str = atof_(str, &val);
if (idx == label_idx_) {
*out_label = val;
offset = -1;
} else if (std::fabs(val) > kZeroThreshold || std::isnan(val)) {
out_features->emplace_back(idx + offset, val);
}
++idx;
if (*str == '\t') {
++str;
} else if (*str != '\0') {
Log::Fatal("Input format error when parsing as TSV");
}
}
}
inline int NumFeatures() const override {
return total_columns_ - (label_idx_ >= 0);
}
private:
int label_idx_ = 0;
int total_columns_ = -1;
AtofFunc atof_;
};
class LibSVMParser: public Parser {
public:
explicit LibSVMParser(int label_idx, int total_columns, AtofFunc atof)
:label_idx_(label_idx), total_columns_(total_columns), atof_(atof) {
if (label_idx > 0) {
Log::Fatal("Label should be the first column in a LibSVM file");
}
}
inline void ParseOneLine(const char* str,
std::vector<std::pair<int, double>>* out_features, double* out_label) const override {
int idx = 0;
double val = 0.0f;
if (label_idx_ == 0) {
str = atof_(str, &val);
*out_label = val;
str = Common::SkipSpaceAndTab(str);
}
while (*str != '\0') {
str = Common::Atoi(str, &idx);
str = Common::SkipSpaceAndTab(str);
if (*str == ':') {
++str;
str = Common::Atof(str, &val);
out_features->emplace_back(idx, val);
} else {
Log::Fatal("Input format error when parsing as LibSVM");
}
str = Common::SkipSpaceAndTab(str);
}
}
inline int NumFeatures() const override {
return total_columns_;
}
private:
int label_idx_ = 0;
int total_columns_ = -1;
AtofFunc atof_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_IO_PARSER_HPP_
src/io/sparse_bin.hpp
+858
View File
@@ -0,0 +1,858 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_IO_SPARSE_BIN_HPP_
#define LIGHTGBM_SRC_IO_SPARSE_BIN_HPP_
#include <LightGBM/bin.h>
#include <LightGBM/utils/log.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <algorithm>
#include <cstdint>
#include <cstring>
#include <limits>
#include <utility>
#include <vector>
namespace LightGBM {
template <typename VAL_T>
class SparseBin;
const size_t kNumFastIndex = 64;
template <typename VAL_T>
class SparseBinIterator : public BinIterator {
public:
SparseBinIterator(const SparseBin<VAL_T>* bin_data, uint32_t min_bin,
uint32_t max_bin, uint32_t most_freq_bin)
: bin_data_(bin_data),
min_bin_(static_cast<VAL_T>(min_bin)),
max_bin_(static_cast<VAL_T>(max_bin)),
most_freq_bin_(static_cast<VAL_T>(most_freq_bin)) {
if (most_freq_bin_ == 0) {
offset_ = 1;
} else {
offset_ = 0;
}
Reset(0);
}
SparseBinIterator(const SparseBin<VAL_T>* bin_data, data_size_t start_idx)
: bin_data_(bin_data) {
Reset(start_idx);
}
inline uint32_t RawGet(data_size_t idx) override;
inline VAL_T InnerRawGet(data_size_t idx);
inline uint32_t Get(data_size_t idx) override {
VAL_T ret = InnerRawGet(idx);
if (ret >= min_bin_ && ret <= max_bin_) {
return ret - min_bin_ + offset_;
} else {
return most_freq_bin_;
}
}
inline void Reset(data_size_t idx) override;
private:
const SparseBin<VAL_T>* bin_data_;
data_size_t cur_pos_;
data_size_t i_delta_;
VAL_T min_bin_;
VAL_T max_bin_;
VAL_T most_freq_bin_;
uint8_t offset_;
};
template <typename VAL_T>
class SparseBin : public Bin {
public:
friend class SparseBinIterator<VAL_T>;
explicit SparseBin(data_size_t num_data) : num_data_(num_data) {
int num_threads = OMP_NUM_THREADS();
push_buffers_.resize(num_threads);
}
~SparseBin() {}
void InitStreaming(uint32_t num_thread, int32_t omp_max_threads) override {
// Each external thread needs its own set of OpenMP push buffers,
// so allocate num_thread times the maximum number of OMP threads per external thread
push_buffers_.resize(omp_max_threads * num_thread);
};
void ReSize(data_size_t num_data) override { num_data_ = num_data; }
void Push(int tid, data_size_t idx, uint32_t value) override {
auto cur_bin = static_cast<VAL_T>(value);
if (cur_bin != 0) {
push_buffers_[tid].emplace_back(idx, cur_bin);
}
}
BinIterator* GetIterator(uint32_t min_bin, uint32_t max_bin,
uint32_t most_freq_bin) const override;
#define ACC_GH(hist, i, g, h) \
const auto ti = static_cast<int>(i) << 1; \
hist[ti] += g; \
hist[ti + 1] += h;
void ConstructHistogram(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
const score_t* ordered_hessians,
hist_t* out) const override {
data_size_t i_delta, cur_pos;
InitIndex(data_indices[start], &i_delta, &cur_pos);
data_size_t i = start;
for (;;) {
if (cur_pos < data_indices[i]) {
cur_pos += deltas_[++i_delta];
if (i_delta >= num_vals_) {
break;
}
} else if (cur_pos > data_indices[i]) {
if (++i >= end) {
break;
}
} else {
const VAL_T bin = vals_[i_delta];
ACC_GH(out, bin, ordered_gradients[i], ordered_hessians[i]);
if (++i >= end) {
break;
}
cur_pos += deltas_[++i_delta];
if (i_delta >= num_vals_) {
break;
}
}
}
}
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
const score_t* ordered_hessians,
hist_t* out) const override {
data_size_t i_delta, cur_pos;
InitIndex(start, &i_delta, &cur_pos);
while (cur_pos < start && i_delta < num_vals_) {
cur_pos += deltas_[++i_delta];
}
while (cur_pos < end && i_delta < num_vals_) {
const VAL_T bin = vals_[i_delta];
ACC_GH(out, bin, ordered_gradients[cur_pos], ordered_hessians[cur_pos]);
cur_pos += deltas_[++i_delta];
}
}
void ConstructHistogram(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
hist_t* out) const override {
data_size_t i_delta, cur_pos;
InitIndex(data_indices[start], &i_delta, &cur_pos);
data_size_t i = start;
hist_t* grad = out;
hist_cnt_t* cnt = reinterpret_cast<hist_cnt_t*>(out + 1);
for (;;) {
if (cur_pos < data_indices[i]) {
cur_pos += deltas_[++i_delta];
if (i_delta >= num_vals_) {
break;
}
} else if (cur_pos > data_indices[i]) {
if (++i >= end) {
break;
}
} else {
const uint32_t ti = static_cast<uint32_t>(vals_[i_delta]) << 1;
grad[ti] += ordered_gradients[i];
++cnt[ti];
if (++i >= end) {
break;
}
cur_pos += deltas_[++i_delta];
if (i_delta >= num_vals_) {
break;
}
}
}
}
void ConstructHistogram(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
hist_t* out) const override {
data_size_t i_delta, cur_pos;
InitIndex(start, &i_delta, &cur_pos);
hist_t* grad = out;
hist_cnt_t* cnt = reinterpret_cast<hist_cnt_t*>(out + 1);
while (cur_pos < start && i_delta < num_vals_) {
cur_pos += deltas_[++i_delta];
}
while (cur_pos < end && i_delta < num_vals_) {
const uint32_t ti = static_cast<uint32_t>(vals_[i_delta]) << 1;
grad[ti] += ordered_gradients[cur_pos];
++cnt[ti];
cur_pos += deltas_[++i_delta];
}
}
#undef ACC_GH
template <bool USE_HESSIAN, typename PACKED_HIST_T, typename GRAD_HIST_T, typename HESS_HIST_T, int HIST_BITS>
void ConstructIntHistogramInner(data_size_t start, data_size_t end,
const score_t* ordered_gradients_and_hessians,
hist_t* out) const {
data_size_t i_delta, cur_pos;
InitIndex(start, &i_delta, &cur_pos);
if (USE_HESSIAN) {
PACKED_HIST_T* out_ptr = reinterpret_cast<PACKED_HIST_T*>(out);
const int16_t* gradients_and_hessians_ptr = reinterpret_cast<const int16_t*>(ordered_gradients_and_hessians);
while (cur_pos < start && i_delta < num_vals_) {
cur_pos += deltas_[++i_delta];
}
while (cur_pos < end && i_delta < num_vals_) {
const VAL_T bin = vals_[i_delta];
const int16_t gradient_16 = gradients_and_hessians_ptr[cur_pos];
const PACKED_HIST_T gradient_64 = (static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) | (gradient_16 & 0xff);
out_ptr[bin] += gradient_64;
cur_pos += deltas_[++i_delta];
}
} else {
GRAD_HIST_T* grad = reinterpret_cast<GRAD_HIST_T*>(out);
HESS_HIST_T* cnt = reinterpret_cast<HESS_HIST_T*>(out) + 1;
const int8_t* gradients_and_hessians_ptr = reinterpret_cast<const int8_t*>(ordered_gradients_and_hessians);
while (cur_pos < start && i_delta < num_vals_) {
cur_pos += deltas_[++i_delta];
}
while (cur_pos < end && i_delta < num_vals_) {
const uint32_t ti = static_cast<uint32_t>(vals_[i_delta]) << 1;
grad[ti] += gradients_and_hessians_ptr[cur_pos];
++cnt[ti];
cur_pos += deltas_[++i_delta];
}
}
}
template <bool USE_HESSIAN, typename PACKED_HIST_T, typename GRAD_HIST_T, typename HESS_HIST_T, int HIST_BITS>
void ConstructIntHistogramInner(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients_and_hessians,
hist_t* out) const {
data_size_t i_delta, cur_pos;
InitIndex(data_indices[start], &i_delta, &cur_pos);
data_size_t i = start;
if (USE_HESSIAN) {
PACKED_HIST_T* out_ptr = reinterpret_cast<PACKED_HIST_T*>(out);
const int16_t* gradients_and_hessians_ptr = reinterpret_cast<const int16_t*>(ordered_gradients_and_hessians);
for (;;) {
if (cur_pos < data_indices[i]) {
cur_pos += deltas_[++i_delta];
if (i_delta >= num_vals_) {
break;
}
} else if (cur_pos > data_indices[i]) {
if (++i >= end) {
break;
}
} else {
const VAL_T bin = vals_[i_delta];
const int16_t gradient_16 = gradients_and_hessians_ptr[i];
const PACKED_HIST_T gradient_packed = (HIST_BITS == 8) ? gradient_16 :
(static_cast<PACKED_HIST_T>(static_cast<int8_t>(gradient_16 >> 8)) << HIST_BITS) | (gradient_16 & 0xff);
out_ptr[bin] += gradient_packed;
if (++i >= end) {
break;
}
cur_pos += deltas_[++i_delta];
if (i_delta >= num_vals_) {
break;
}
}
}
} else {
GRAD_HIST_T* grad = reinterpret_cast<GRAD_HIST_T*>(out);
HESS_HIST_T* cnt = reinterpret_cast<HESS_HIST_T*>(out) + 1;
const int8_t* gradients_and_hessians_ptr = reinterpret_cast<const int8_t*>(ordered_gradients_and_hessians);
for (;;) {
if (cur_pos < data_indices[i]) {
cur_pos += deltas_[++i_delta];
if (i_delta >= num_vals_) {
break;
}
} else if (cur_pos > data_indices[i]) {
if (++i >= end) {
break;
}
} else {
const uint32_t ti = static_cast<uint32_t>(vals_[i_delta]) << 1;
grad[ti] += gradients_and_hessians_ptr[i << 1];
++cnt[ti];
if (++i >= end) {
break;
}
cur_pos += deltas_[++i_delta];
if (i_delta >= num_vals_) {
break;
}
}
}
}
}
void ConstructHistogramInt32(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructIntHistogramInner<true, int64_t, int32_t, uint32_t, 32>(data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt32(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructIntHistogramInner<true, int64_t, int32_t, uint32_t, 32>(start, end, ordered_gradients, out);
}
void ConstructHistogramInt32(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
hist_t* out) const override {
ConstructIntHistogramInner<false, int64_t, int32_t, uint32_t, 32>(data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt32(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
hist_t* out) const override {
ConstructIntHistogramInner<false, int64_t, int32_t, uint32_t, 32>(start, end, ordered_gradients, out);
}
void ConstructHistogramInt16(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructIntHistogramInner<true, int32_t, int16_t, uint16_t, 16>(data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt16(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructIntHistogramInner<true, int32_t, int16_t, uint16_t, 16>(start, end, ordered_gradients, out);
}
void ConstructHistogramInt16(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
hist_t* out) const override {
ConstructIntHistogramInner<false, int32_t, int16_t, uint16_t, 16>(data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt16(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
hist_t* out) const override {
ConstructIntHistogramInner<false, int32_t, int16_t, uint16_t, 16>(start, end, ordered_gradients, out);
}
void ConstructHistogramInt8(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructIntHistogramInner<true, int16_t, uint8_t, uint8_t, 8>(data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt8(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
const score_t* /*ordered_hessians*/,
hist_t* out) const override {
ConstructIntHistogramInner<true, int16_t, uint8_t, uint8_t, 8>(start, end, ordered_gradients, out);
}
void ConstructHistogramInt8(const data_size_t* data_indices, data_size_t start,
data_size_t end, const score_t* ordered_gradients,
hist_t* out) const override {
ConstructIntHistogramInner<false, int16_t, uint8_t, uint8_t, 8>(data_indices, start, end, ordered_gradients, out);
}
void ConstructHistogramInt8(data_size_t start, data_size_t end,
const score_t* ordered_gradients,
hist_t* out) const override {
ConstructIntHistogramInner<false, int16_t, uint8_t, uint8_t, 8>(start, end, ordered_gradients, out);
}
inline void NextNonzeroFast(data_size_t* i_delta,
data_size_t* cur_pos) const {
*cur_pos += deltas_[++(*i_delta)];
if (*i_delta >= num_vals_) {
*cur_pos = num_data_;
}
}
inline bool NextNonzero(data_size_t* i_delta, data_size_t* cur_pos) const {
*cur_pos += deltas_[++(*i_delta)];
if (*i_delta < num_vals_) {
return true;
} else {
*cur_pos = num_data_;
return false;
}
}
template <bool MISS_IS_ZERO, bool MISS_IS_NA, bool MFB_IS_ZERO,
bool MFB_IS_NA, bool USE_MIN_BIN>
data_size_t SplitInner(uint32_t min_bin, uint32_t max_bin,
uint32_t default_bin, uint32_t most_freq_bin,
bool default_left, uint32_t threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const {
auto th = static_cast<VAL_T>(threshold + min_bin);
auto t_zero_bin = static_cast<VAL_T>(min_bin + default_bin);
if (most_freq_bin == 0) {
--th;
--t_zero_bin;
}
const auto minb = static_cast<VAL_T>(min_bin);
const auto maxb = static_cast<VAL_T>(max_bin);
data_size_t lte_count = 0;
data_size_t gt_count = 0;
data_size_t* default_indices = gt_indices;
data_size_t* default_count = &gt_count;
data_size_t* missing_default_indices = gt_indices;
data_size_t* missing_default_count = &gt_count;
if (most_freq_bin <= threshold) {
default_indices = lte_indices;
default_count = &lte_count;
}
if (MISS_IS_ZERO || MISS_IS_NA) {
if (default_left) {
missing_default_indices = lte_indices;
missing_default_count = &lte_count;
}
}
SparseBinIterator<VAL_T> iterator(this, data_indices[0]);
if (min_bin < max_bin) {
for (data_size_t i = 0; i < cnt; ++i) {
const data_size_t idx = data_indices[i];
const auto bin = iterator.InnerRawGet(idx);
if ((MISS_IS_ZERO && !MFB_IS_ZERO && bin == t_zero_bin) ||
(MISS_IS_NA && !MFB_IS_NA && bin == maxb)) {
missing_default_indices[(*missing_default_count)++] = idx;
} else if ((USE_MIN_BIN && (bin < minb || bin > maxb)) ||
(!USE_MIN_BIN && bin == 0)) {
if ((MISS_IS_NA && MFB_IS_NA) || (MISS_IS_ZERO && MFB_IS_ZERO)) {
missing_default_indices[(*missing_default_count)++] = idx;
} else {
default_indices[(*default_count)++] = idx;
}
} else if (bin > th) {
gt_indices[gt_count++] = idx;
} else {
lte_indices[lte_count++] = idx;
}
}
} else {
data_size_t* max_bin_indices = gt_indices;
data_size_t* max_bin_count = &gt_count;
if (maxb <= th) {
max_bin_indices = lte_indices;
max_bin_count = &lte_count;
}
for (data_size_t i = 0; i < cnt; ++i) {
const data_size_t idx = data_indices[i];
const auto bin = iterator.InnerRawGet(idx);
if (MISS_IS_ZERO && !MFB_IS_ZERO && bin == t_zero_bin) {
missing_default_indices[(*missing_default_count)++] = idx;
} else if (bin != maxb) {
if ((MISS_IS_NA && MFB_IS_NA) || (MISS_IS_ZERO && MFB_IS_ZERO)) {
missing_default_indices[(*missing_default_count)++] = idx;
} else {
default_indices[(*default_count)++] = idx;
}
} else {
if (MISS_IS_NA && !MFB_IS_NA) {
missing_default_indices[(*missing_default_count)++] = idx;
} else {
max_bin_indices[(*max_bin_count)++] = idx;
}
}
}
}
return lte_count;
}
data_size_t Split(uint32_t min_bin, uint32_t max_bin, uint32_t default_bin,
uint32_t most_freq_bin, MissingType missing_type,
bool default_left, uint32_t threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const override {
#define ARGUMENTS \
min_bin, max_bin, default_bin, most_freq_bin, default_left, threshold, \
data_indices, cnt, lte_indices, gt_indices
if (missing_type == MissingType::None) {
return SplitInner<false, false, false, false, true>(ARGUMENTS);
} else if (missing_type == MissingType::Zero) {
if (default_bin == most_freq_bin) {
return SplitInner<true, false, true, false, true>(ARGUMENTS);
} else {
return SplitInner<true, false, false, false, true>(ARGUMENTS);
}
} else {
if (max_bin == most_freq_bin + min_bin && most_freq_bin > 0) {
return SplitInner<false, true, false, true, true>(ARGUMENTS);
} else {
return SplitInner<false, true, false, false, true>(ARGUMENTS);
}
}
#undef ARGUMENTS
}
data_size_t Split(uint32_t max_bin, uint32_t default_bin,
uint32_t most_freq_bin, MissingType missing_type,
bool default_left, uint32_t threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const override {
#define ARGUMENTS \
1, max_bin, default_bin, most_freq_bin, default_left, threshold, \
data_indices, cnt, lte_indices, gt_indices
if (missing_type == MissingType::None) {
return SplitInner<false, false, false, false, false>(ARGUMENTS);
} else if (missing_type == MissingType::Zero) {
if (default_bin == most_freq_bin) {
return SplitInner<true, false, true, false, false>(ARGUMENTS);
} else {
return SplitInner<true, false, false, false, false>(ARGUMENTS);
}
} else {
if (max_bin == most_freq_bin + 1 && most_freq_bin > 0) {
return SplitInner<false, true, false, true, false>(ARGUMENTS);
} else {
return SplitInner<false, true, false, false, false>(ARGUMENTS);
}
}
#undef ARGUMENTS
}
template <bool USE_MIN_BIN>
data_size_t SplitCategoricalInner(uint32_t min_bin, uint32_t max_bin,
uint32_t most_freq_bin,
const uint32_t* threshold,
int num_threshold,
const data_size_t* data_indices,
data_size_t cnt, data_size_t* lte_indices,
data_size_t* gt_indices) const {
data_size_t lte_count = 0;
data_size_t gt_count = 0;
data_size_t* default_indices = gt_indices;
data_size_t* default_count = &gt_count;
SparseBinIterator<VAL_T> iterator(this, data_indices[0]);
int8_t offset = most_freq_bin == 0 ? 1 : 0;
if (most_freq_bin > 0 && Common::FindInBitset(threshold, num_threshold, most_freq_bin)) {
default_indices = lte_indices;
default_count = &lte_count;
}
for (data_size_t i = 0; i < cnt; ++i) {
const data_size_t idx = data_indices[i];
const uint32_t bin = iterator.RawGet(idx);
if (USE_MIN_BIN && (bin < min_bin || bin > max_bin)) {
default_indices[(*default_count)++] = idx;
} else if (!USE_MIN_BIN && bin == 0) {
default_indices[(*default_count)++] = idx;
} else if (Common::FindInBitset(threshold, num_threshold,
bin - min_bin + offset)) {
lte_indices[lte_count++] = idx;
} else {
gt_indices[gt_count++] = idx;
}
}
return lte_count;
}
data_size_t SplitCategorical(uint32_t min_bin, uint32_t max_bin,
uint32_t most_freq_bin,
const uint32_t* threshold, int num_threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const override {
return SplitCategoricalInner<true>(min_bin, max_bin, most_freq_bin,
threshold, num_threshold, data_indices,
cnt, lte_indices, gt_indices);
}
data_size_t SplitCategorical(uint32_t max_bin, uint32_t most_freq_bin,
const uint32_t* threshold, int num_threshold,
const data_size_t* data_indices, data_size_t cnt,
data_size_t* lte_indices,
data_size_t* gt_indices) const override {
return SplitCategoricalInner<false>(1, max_bin, most_freq_bin, threshold,
num_threshold, data_indices, cnt,
lte_indices, gt_indices);
}
data_size_t num_data() const override { return num_data_; }
void* get_data() override { return nullptr; }
void FinishLoad() override {
// get total non zero size
size_t pair_cnt = 0;
for (size_t i = 0; i < push_buffers_.size(); ++i) {
pair_cnt += push_buffers_[i].size();
}
std::vector<std::pair<data_size_t, VAL_T>>& idx_val_pairs =
push_buffers_[0];
idx_val_pairs.reserve(pair_cnt);
for (size_t i = 1; i < push_buffers_.size(); ++i) {
idx_val_pairs.insert(idx_val_pairs.end(), push_buffers_[i].begin(),
push_buffers_[i].end());
push_buffers_[i].clear();
push_buffers_[i].shrink_to_fit();
}
// sort by data index
std::sort(idx_val_pairs.begin(), idx_val_pairs.end(),
[](const std::pair<data_size_t, VAL_T>& a,
const std::pair<data_size_t, VAL_T>& b) {
return a.first < b.first;
});
// load delta array
LoadFromPair(idx_val_pairs);
}
void LoadFromPair(
const std::vector<std::pair<data_size_t, VAL_T>>& idx_val_pairs) {
deltas_.clear();
vals_.clear();
deltas_.reserve(idx_val_pairs.size());
vals_.reserve(idx_val_pairs.size());
// transform to delta array
data_size_t last_idx = 0;
for (size_t i = 0; i < idx_val_pairs.size(); ++i) {
const data_size_t cur_idx = idx_val_pairs[i].first;
const VAL_T bin = idx_val_pairs[i].second;
data_size_t cur_delta = cur_idx - last_idx;
// disallow the multi-val in one row
if (i > 0 && cur_delta == 0) {
continue;
}
while (cur_delta >= 256) {
deltas_.push_back(255);
vals_.push_back(0);
cur_delta -= 255;
}
deltas_.push_back(static_cast<uint8_t>(cur_delta));
vals_.push_back(bin);
last_idx = cur_idx;
}
// avoid out of range
deltas_.push_back(0);
num_vals_ = static_cast<data_size_t>(vals_.size());
// reduce memory cost
deltas_.shrink_to_fit();
vals_.shrink_to_fit();
// generate fast index
GetFastIndex();
}
void GetFastIndex() {
fast_index_.clear();
// get shift cnt
data_size_t mod_size = (num_data_ + kNumFastIndex - 1) / kNumFastIndex;
data_size_t pow2_mod_size = 1;
fast_index_shift_ = 0;
while (pow2_mod_size < mod_size) {
pow2_mod_size <<= 1;
++fast_index_shift_;
}
// build fast index
data_size_t i_delta = -1;
data_size_t cur_pos = 0;
data_size_t next_threshold = 0;
while (NextNonzero(&i_delta, &cur_pos)) {
while (next_threshold <= cur_pos) {
fast_index_.emplace_back(i_delta, cur_pos);
next_threshold += pow2_mod_size;
}
}
// avoid out of range
while (next_threshold < num_data_) {
fast_index_.emplace_back(num_vals_ - 1, cur_pos);
next_threshold += pow2_mod_size;
}
fast_index_.shrink_to_fit();
}
void SaveBinaryToFile(BinaryWriter* writer) const override {
writer->AlignedWrite(&num_vals_, sizeof(num_vals_));
writer->AlignedWrite(deltas_.data(), sizeof(uint8_t) * (num_vals_ + 1));
writer->AlignedWrite(vals_.data(), sizeof(VAL_T) * num_vals_);
}
size_t SizesInByte() const override {
return VirtualFileWriter::AlignedSize(sizeof(num_vals_)) +
VirtualFileWriter::AlignedSize(sizeof(uint8_t) * (num_vals_ + 1)) +
VirtualFileWriter::AlignedSize(sizeof(VAL_T) * num_vals_);
}
void LoadFromMemory(
const void* memory,
const std::vector<data_size_t>& local_used_indices) override {
const char* mem_ptr = reinterpret_cast<const char*>(memory);
data_size_t tmp_num_vals = *(reinterpret_cast<const data_size_t*>(mem_ptr));
mem_ptr += VirtualFileWriter::AlignedSize(sizeof(tmp_num_vals));
const uint8_t* tmp_delta = reinterpret_cast<const uint8_t*>(mem_ptr);
mem_ptr += VirtualFileWriter::AlignedSize(sizeof(uint8_t) * (tmp_num_vals + 1));
const VAL_T* tmp_vals = reinterpret_cast<const VAL_T*>(mem_ptr);
deltas_.clear();
vals_.clear();
num_vals_ = tmp_num_vals;
for (data_size_t i = 0; i < num_vals_; ++i) {
deltas_.push_back(tmp_delta[i]);
vals_.push_back(tmp_vals[i]);
}
deltas_.push_back(0);
// reduce memory cost
deltas_.shrink_to_fit();
vals_.shrink_to_fit();
if (local_used_indices.empty()) {
// generate fast index
GetFastIndex();
} else {
std::vector<std::pair<data_size_t, VAL_T>> tmp_pair;
data_size_t cur_pos = 0;
data_size_t j = -1;
for (data_size_t i = 0;
i < static_cast<data_size_t>(local_used_indices.size()); ++i) {
const data_size_t idx = local_used_indices[i];
while (cur_pos < idx && j < num_vals_) {
NextNonzero(&j, &cur_pos);
}
if (cur_pos == idx && j < num_vals_ && vals_[j] > 0) {
// new row index is i
tmp_pair.emplace_back(i, vals_[j]);
}
}
LoadFromPair(tmp_pair);
}
}
void CopySubrow(const Bin* full_bin, const data_size_t* used_indices,
data_size_t num_used_indices) override {
auto other_bin = dynamic_cast<const SparseBin<VAL_T>*>(full_bin);
deltas_.clear();
vals_.clear();
data_size_t start = 0;
if (num_used_indices > 0) {
start = used_indices[0];
}
SparseBinIterator<VAL_T> iterator(other_bin, start);
// transform to delta array
data_size_t last_idx = 0;
for (data_size_t i = 0; i < num_used_indices; ++i) {
auto bin = iterator.InnerRawGet(used_indices[i]);
if (bin > 0) {
data_size_t cur_delta = i - last_idx;
while (cur_delta >= 256) {
deltas_.push_back(255);
vals_.push_back(0);
cur_delta -= 255;
}
deltas_.push_back(static_cast<uint8_t>(cur_delta));
vals_.push_back(bin);
last_idx = i;
}
}
// avoid out of range
deltas_.push_back(0);
num_vals_ = static_cast<data_size_t>(vals_.size());
// reduce memory cost
deltas_.shrink_to_fit();
vals_.shrink_to_fit();
// generate fast index
GetFastIndex();
}
SparseBin<VAL_T>* Clone() override;
SparseBin(const SparseBin<VAL_T>& other)
: num_data_(other.num_data_),
deltas_(other.deltas_),
vals_(other.vals_),
num_vals_(other.num_vals_),
push_buffers_(other.push_buffers_),
fast_index_(other.fast_index_),
fast_index_shift_(other.fast_index_shift_) {}
void InitIndex(data_size_t start_idx, data_size_t* i_delta,
data_size_t* cur_pos) const {
auto idx = start_idx >> fast_index_shift_;
if (static_cast<size_t>(idx) < fast_index_.size()) {
const auto fast_pair = fast_index_[start_idx >> fast_index_shift_];
*i_delta = fast_pair.first;
*cur_pos = fast_pair.second;
} else {
*i_delta = -1;
*cur_pos = 0;
}
}
const void* GetColWiseData(uint8_t* bit_type, bool* is_sparse, std::vector<BinIterator*>* bin_iterator, const int num_threads) const override;
const void* GetColWiseData(uint8_t* bit_type, bool* is_sparse, BinIterator** bin_iterator) const override;
private:
data_size_t num_data_;
std::vector<uint8_t, Common::AlignmentAllocator<uint8_t, kAlignedSize>>
deltas_;
std::vector<VAL_T, Common::AlignmentAllocator<VAL_T, kAlignedSize>> vals_;
data_size_t num_vals_;
std::vector<std::vector<std::pair<data_size_t, VAL_T>>> push_buffers_;
std::vector<std::pair<data_size_t, data_size_t>> fast_index_;
data_size_t fast_index_shift_;
};
template <typename VAL_T>
SparseBin<VAL_T>* SparseBin<VAL_T>::Clone() {
return new SparseBin(*this);
}
template <typename VAL_T>
inline uint32_t SparseBinIterator<VAL_T>::RawGet(data_size_t idx) {
return InnerRawGet(idx);
}
template <typename VAL_T>
inline VAL_T SparseBinIterator<VAL_T>::InnerRawGet(data_size_t idx) {
while (cur_pos_ < idx) {
bin_data_->NextNonzeroFast(&i_delta_, &cur_pos_);
}
if (cur_pos_ == idx) {
return bin_data_->vals_[i_delta_];
} else {
return 0;
}
}
template <typename VAL_T>
inline void SparseBinIterator<VAL_T>::Reset(data_size_t start_idx) {
bin_data_->InitIndex(start_idx, &i_delta_, &cur_pos_);
}
template <typename VAL_T>
BinIterator* SparseBin<VAL_T>::GetIterator(uint32_t min_bin, uint32_t max_bin,
uint32_t most_freq_bin) const {
return new SparseBinIterator<VAL_T>(this, min_bin, max_bin, most_freq_bin);
}
} // namespace LightGBM
#endif // LIGHTGBM_SRC_IO_SPARSE_BIN_HPP_
src/io/train_share_states.cpp
+540
View File
@@ -0,0 +1,540 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#include <LightGBM/train_share_states.h>
#include <algorithm>
#include <memory>
#include <vector>
namespace LightGBM {
MultiValBinWrapper::MultiValBinWrapper(MultiValBin* bin, data_size_t num_data,
const std::vector<int>& feature_groups_contained, const int num_grad_quant_bins):
feature_groups_contained_(feature_groups_contained) {
num_threads_ = OMP_NUM_THREADS();
num_data_ = num_data;
multi_val_bin_.reset(bin);
if (bin == nullptr) {
return;
}
num_bin_ = bin->num_bin();
num_bin_aligned_ = (num_bin_ + kAlignedSize - 1) / kAlignedSize * kAlignedSize;
num_grad_quant_bins_ = num_grad_quant_bins;
}
void MultiValBinWrapper::InitTrain(const std::vector<int>& group_feature_start,
const std::vector<std::unique_ptr<FeatureGroup>>& feature_groups,
const std::vector<int8_t>& is_feature_used,
const data_size_t* bagging_use_indices,
data_size_t bagging_indices_cnt) {
is_use_subcol_ = false;
if (multi_val_bin_ == nullptr) {
return;
}
CopyMultiValBinSubset(group_feature_start, feature_groups,
is_feature_used, bagging_use_indices, bagging_indices_cnt);
const auto cur_multi_val_bin = (is_use_subcol_ || is_use_subrow_)
? multi_val_bin_subset_.get()
: multi_val_bin_.get();
if (cur_multi_val_bin != nullptr) {
num_bin_ = cur_multi_val_bin->num_bin();
num_bin_aligned_ = (num_bin_ + kAlignedSize - 1) / kAlignedSize * kAlignedSize;
auto num_element_per_row = cur_multi_val_bin->num_element_per_row();
min_block_size_ = std::min<int>(static_cast<int>(0.3f * num_bin_ /
(num_element_per_row + kZeroThreshold)) + 1, 1024);
min_block_size_ = std::max<int>(min_block_size_, 32);
}
}
template <bool USE_QUANT_GRAD, int HIST_BITS, int INNER_HIST_BITS>
void MultiValBinWrapper::HistMove(const std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>& hist_buf) {
if (!is_use_subcol_ && INNER_HIST_BITS != 8) {
return;
}
if (USE_QUANT_GRAD) {
if (HIST_BITS == 32) {
const int64_t* src = reinterpret_cast<const int64_t*>(hist_buf.data()) + hist_buf.size() / 2 -
static_cast<size_t>(num_bin_aligned_);
#pragma omp parallel for schedule(static) num_threads(num_threads_)
for (int i = 0; i < static_cast<int>(hist_move_src_.size()); ++i) {
std::copy_n(src + hist_move_src_[i] / 2, hist_move_size_[i] / 2,
reinterpret_cast<int64_t*>(origin_hist_data_) + hist_move_dest_[i] / 2);
}
} else if (HIST_BITS == 16) {
if (is_use_subcol_) {
const int32_t* src = reinterpret_cast<const int32_t*>(hist_buf.data()) + hist_buf.size() / 2 -
static_cast<size_t>(num_bin_aligned_);
#pragma omp parallel for schedule(static) num_threads(num_threads_)
for (int i = 0; i < static_cast<int>(hist_move_src_.size()); ++i) {
std::copy_n(src + hist_move_src_[i] / 2, hist_move_size_[i] / 2,
reinterpret_cast<int32_t*>(origin_hist_data_) + hist_move_dest_[i] / 2);
}
} else {
CHECK_EQ(INNER_HIST_BITS, 8);
const int32_t* src = reinterpret_cast<const int32_t*>(hist_buf.data()) + hist_buf.size() / 2;
int32_t* orig_ptr = reinterpret_cast<int32_t*>(origin_hist_data_);
#pragma omp parallel for schedule(static) num_threads(num_threads_)
for (int i = 0; i < num_bin_; ++i) {
orig_ptr[i] = src[i];
}
}
}
} else {
const hist_t* src = hist_buf.data() + hist_buf.size() -
2 * static_cast<size_t>(num_bin_aligned_);
#pragma omp parallel for schedule(static) num_threads(num_threads_)
for (int i = 0; i < static_cast<int>(hist_move_src_.size()); ++i) {
std::copy_n(src + hist_move_src_[i], hist_move_size_[i],
origin_hist_data_ + hist_move_dest_[i]);
}
}
}
template void MultiValBinWrapper::HistMove<false, 0, 0>(const std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>& hist_buf);
template void MultiValBinWrapper::HistMove<false, 0, 8>(const std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>& hist_buf);
template void MultiValBinWrapper::HistMove<true, 16, 8>(const std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>& hist_buf);
template void MultiValBinWrapper::HistMove<true, 16, 16>(const std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>& hist_buf);
template void MultiValBinWrapper::HistMove<true, 32, 8>(const std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>& hist_buf);
template void MultiValBinWrapper::HistMove<true, 32, 32>(const std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>& hist_buf);
template <bool USE_QUANT_GRAD, int HIST_BITS, int INNER_HIST_BITS>
void MultiValBinWrapper::HistMerge(std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>* hist_buf) {
int n_bin_block = 1;
int bin_block_size = num_bin_;
Threading::BlockInfo<data_size_t>(num_threads_, num_bin_, 512, &n_bin_block,
&bin_block_size);
if (USE_QUANT_GRAD) {
if (HIST_BITS == 32) {
int64_t* dst = reinterpret_cast<int64_t*>(origin_hist_data_);
if (is_use_subcol_) {
dst = reinterpret_cast<int64_t*>(hist_buf->data()) + hist_buf->size() / 2 - static_cast<size_t>(num_bin_aligned_);
}
#pragma omp parallel for schedule(static, 1) num_threads(num_threads_)
for (int t = 0; t < n_bin_block; ++t) {
const int start = t * bin_block_size;
const int end = std::min(start + bin_block_size, num_bin_);
for (int tid = 1; tid < n_data_block_; ++tid) {
auto src_ptr = reinterpret_cast<const int64_t*>(hist_buf->data()) + static_cast<size_t>(num_bin_aligned_) * (tid - 1);
for (int i = start; i < end; ++i) {
dst[i] += src_ptr[i];
}
}
}
} else if (HIST_BITS == 16 && INNER_HIST_BITS == 16) {
int32_t* dst = reinterpret_cast<int32_t*>(origin_hist_data_);
if (is_use_subcol_) {
dst = reinterpret_cast<int32_t*>(hist_buf->data()) + hist_buf->size() / 2 - static_cast<size_t>(num_bin_aligned_);
}
#pragma omp parallel for schedule(static, 1) num_threads(num_threads_)
for (int t = 0; t < n_bin_block; ++t) {
const int start = t * bin_block_size;
const int end = std::min(start + bin_block_size, num_bin_);
for (int tid = 1; tid < n_data_block_; ++tid) {
auto src_ptr = reinterpret_cast<const int32_t*>(hist_buf->data()) + static_cast<size_t>(num_bin_aligned_) * (tid - 1);
for (int i = start; i < end; ++i) {
dst[i] += src_ptr[i];
}
}
}
} else if (HIST_BITS == 16 && INNER_HIST_BITS == 8) {
int32_t* dst = reinterpret_cast<int32_t*>(hist_buf->data()) + hist_buf->size() / 2;
std::memset(reinterpret_cast<void*>(dst), 0, num_bin_ * kInt16HistBufferEntrySize);
#pragma omp parallel for schedule(static, 1) num_threads(num_threads_)
for (int t = 0; t < n_bin_block; ++t) {
const int start = t * bin_block_size;
const int end = std::min(start + bin_block_size, num_bin_);
for (int tid = 0; tid < n_data_block_; ++tid) {
auto src_ptr = reinterpret_cast<const int16_t*>(hist_buf->data()) + static_cast<size_t>(num_bin_aligned_) * tid;
for (int i = start; i < end; ++i) {
const int16_t packed_hist = src_ptr[i];
const int32_t packed_hist_int32 = (static_cast<int32_t>(static_cast<int8_t>(packed_hist >> 8)) << 16) | static_cast<int32_t>(packed_hist & 0x00ff);
dst[i] += packed_hist_int32;
}
}
}
}
} else {
hist_t* dst = origin_hist_data_;
if (is_use_subcol_) {
dst = hist_buf->data() + hist_buf->size() - 2 * static_cast<size_t>(num_bin_aligned_);
}
#pragma omp parallel for schedule(static, 1) num_threads(num_threads_)
for (int t = 0; t < n_bin_block; ++t) {
const int start = t * bin_block_size;
const int end = std::min(start + bin_block_size, num_bin_);
for (int tid = 1; tid < n_data_block_; ++tid) {
auto src_ptr = hist_buf->data() + static_cast<size_t>(num_bin_aligned_) * 2 * (tid - 1);
for (int i = start * 2; i < end * 2; ++i) {
dst[i] += src_ptr[i];
}
}
}
}
}
template void MultiValBinWrapper::HistMerge<false, 0, 0>(std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>* hist_buf);
template void MultiValBinWrapper::HistMerge<false, 0, 8>(std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>* hist_buf);
template void MultiValBinWrapper::HistMerge<true, 16, 8>(std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>* hist_buf);
template void MultiValBinWrapper::HistMerge<true, 16, 16>(std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>* hist_buf);
template void MultiValBinWrapper::HistMerge<true, 32, 8>(std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>* hist_buf);
template void MultiValBinWrapper::HistMerge<true, 32, 32>(std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>* hist_buf);
void MultiValBinWrapper::ResizeHistBuf(std::vector<hist_t,
Common::AlignmentAllocator<hist_t, kAlignedSize>>* hist_buf,
MultiValBin* sub_multi_val_bin,
hist_t* origin_hist_data) {
num_bin_ = sub_multi_val_bin->num_bin();
num_bin_aligned_ = (num_bin_ + kAlignedSize - 1) / kAlignedSize * kAlignedSize;
origin_hist_data_ = origin_hist_data;
size_t new_buf_size = static_cast<size_t>(n_data_block_) * static_cast<size_t>(num_bin_aligned_) * 2;
if (hist_buf->size() < new_buf_size) {
hist_buf->resize(new_buf_size);
}
}
void MultiValBinWrapper::CopyMultiValBinSubset(
const std::vector<int>& group_feature_start,
const std::vector<std::unique_ptr<FeatureGroup>>& feature_groups,
const std::vector<int8_t>& is_feature_used,
const data_size_t* bagging_use_indices,
data_size_t bagging_indices_cnt) {
double sum_used_dense_ratio = 0.0;
double sum_dense_ratio = 0.0;
int num_used = 0;
int total = 0;
std::vector<int> used_feature_index;
for (int i : feature_groups_contained_) {
int f_start = group_feature_start[i];
if (feature_groups[i]->is_multi_val_) {
for (int j = 0; j < feature_groups[i]->num_feature_; ++j) {
const auto dense_rate =
1.0 - feature_groups[i]->bin_mappers_[j]->sparse_rate();
if (is_feature_used[f_start + j]) {
++num_used;
used_feature_index.push_back(total);
sum_used_dense_ratio += dense_rate;
}
sum_dense_ratio += dense_rate;
++total;
}
} else {
bool is_group_used = false;
double dense_rate = 0;
for (int j = 0; j < feature_groups[i]->num_feature_; ++j) {
if (is_feature_used[f_start + j]) {
is_group_used = true;
}
dense_rate += 1.0 - feature_groups[i]->bin_mappers_[j]->sparse_rate();
}
if (is_group_used) {
++num_used;
used_feature_index.push_back(total);
sum_used_dense_ratio += dense_rate;
}
sum_dense_ratio += dense_rate;
++total;
}
}
const double k_subfeature_threshold = 0.6;
if (sum_used_dense_ratio >= sum_dense_ratio * k_subfeature_threshold) {
// only need to copy subset
if (is_use_subrow_ && !is_subrow_copied_) {
if (multi_val_bin_subset_ == nullptr) {
multi_val_bin_subset_.reset(multi_val_bin_->CreateLike(
bagging_indices_cnt, multi_val_bin_->num_bin(), total,
multi_val_bin_->num_element_per_row(), multi_val_bin_->offsets()));
} else {
multi_val_bin_subset_->ReSize(
bagging_indices_cnt, multi_val_bin_->num_bin(), total,
multi_val_bin_->num_element_per_row(), multi_val_bin_->offsets());
}
multi_val_bin_subset_->CopySubrow(
multi_val_bin_.get(), bagging_use_indices,
bagging_indices_cnt);
// avoid to copy subset many times
is_subrow_copied_ = true;
}
} else {
is_use_subcol_ = true;
std::vector<uint32_t> upper_bound;
std::vector<uint32_t> lower_bound;
std::vector<uint32_t> delta;
std::vector<uint32_t> offsets;
hist_move_src_.clear();
hist_move_dest_.clear();
hist_move_size_.clear();
const int offset = multi_val_bin_->IsSparse() ? 1 : 0;
int num_total_bin = offset;
int new_num_total_bin = offset;
offsets.push_back(static_cast<uint32_t>(new_num_total_bin));
for (int i : feature_groups_contained_) {
int f_start = group_feature_start[i];
if (feature_groups[i]->is_multi_val_) {
for (int j = 0; j < feature_groups[i]->num_feature_; ++j) {
const auto& bin_mapper = feature_groups[i]->bin_mappers_[j];
if (i == 0 && j == 0 && bin_mapper->GetMostFreqBin() > 0) {
num_total_bin = 1;
}
int cur_num_bin = bin_mapper->num_bin();
if (bin_mapper->GetMostFreqBin() == 0) {
cur_num_bin -= offset;
}
num_total_bin += cur_num_bin;
if (is_feature_used[f_start + j]) {
new_num_total_bin += cur_num_bin;
offsets.push_back(static_cast<uint32_t>(new_num_total_bin));
lower_bound.push_back(num_total_bin - cur_num_bin);
upper_bound.push_back(num_total_bin);
hist_move_src_.push_back(
(new_num_total_bin - cur_num_bin) * 2);
hist_move_dest_.push_back((num_total_bin - cur_num_bin) *
2);
hist_move_size_.push_back(cur_num_bin * 2);
delta.push_back(num_total_bin - new_num_total_bin);
}
}
} else {
bool is_group_used = false;
for (int j = 0; j < feature_groups[i]->num_feature_; ++j) {
if (is_feature_used[f_start + j]) {
is_group_used = true;
break;
}
}
int cur_num_bin = feature_groups[i]->bin_offsets_.back() - offset;
num_total_bin += cur_num_bin;
if (is_group_used) {
new_num_total_bin += cur_num_bin;
offsets.push_back(static_cast<uint32_t>(new_num_total_bin));
lower_bound.push_back(num_total_bin - cur_num_bin);
upper_bound.push_back(num_total_bin);
hist_move_src_.push_back(
(new_num_total_bin - cur_num_bin) * 2);
hist_move_dest_.push_back((num_total_bin - cur_num_bin) *
2);
hist_move_size_.push_back(cur_num_bin * 2);
delta.push_back(num_total_bin - new_num_total_bin);
}
}
}
// avoid out of range
lower_bound.push_back(num_total_bin);
upper_bound.push_back(num_total_bin);
data_size_t num_data = is_use_subrow_ ? bagging_indices_cnt : num_data_;
if (multi_val_bin_subset_ == nullptr) {
multi_val_bin_subset_.reset(multi_val_bin_->CreateLike(
num_data, new_num_total_bin, num_used, sum_used_dense_ratio, offsets));
} else {
multi_val_bin_subset_->ReSize(num_data, new_num_total_bin,
num_used, sum_used_dense_ratio, offsets);
}
if (is_use_subrow_) {
multi_val_bin_subset_->CopySubrowAndSubcol(
multi_val_bin_.get(), bagging_use_indices,
bagging_indices_cnt, used_feature_index, lower_bound,
upper_bound, delta);
// may need to recopy subset
is_subrow_copied_ = false;
} else {
multi_val_bin_subset_->CopySubcol(
multi_val_bin_.get(), used_feature_index, lower_bound, upper_bound, delta);
}
}
}
void TrainingShareStates::CalcBinOffsets(const std::vector<std::unique_ptr<FeatureGroup>>& feature_groups,
std::vector<uint32_t>* offsets, bool in_is_col_wise) {
offsets->clear();
feature_hist_offsets_.clear();
if (in_is_col_wise) {
uint32_t cur_num_bin = 0;
uint32_t hist_cur_num_bin = 0;
for (int group = 0; group < static_cast<int>(feature_groups.size()); ++group) {
const std::unique_ptr<FeatureGroup>& feature_group = feature_groups[group];
if (feature_group->is_multi_val_) {
if (feature_group->is_dense_multi_val_) {
for (int i = 0; i < feature_group->num_feature_; ++i) {
const std::unique_ptr<BinMapper>& bin_mapper = feature_group->bin_mappers_[i];
if (group == 0 && i == 0 && bin_mapper->GetMostFreqBin() > 0) {
cur_num_bin += 1;
hist_cur_num_bin += 1;
}
offsets->push_back(cur_num_bin);
feature_hist_offsets_.push_back(hist_cur_num_bin);
int num_bin = bin_mapper->num_bin();
hist_cur_num_bin += num_bin;
if (bin_mapper->GetMostFreqBin() == 0) {
feature_hist_offsets_.back() += 1;
}
cur_num_bin += num_bin;
}
offsets->push_back(cur_num_bin);
CHECK(cur_num_bin == feature_group->bin_offsets_.back());
} else {
cur_num_bin += 1;
hist_cur_num_bin += 1;
for (int i = 0; i < feature_group->num_feature_; ++i) {
offsets->push_back(cur_num_bin);
feature_hist_offsets_.push_back(hist_cur_num_bin);
const std::unique_ptr<BinMapper>& bin_mapper = feature_group->bin_mappers_[i];
int num_bin = bin_mapper->num_bin();
if (bin_mapper->GetMostFreqBin() == 0) {
num_bin -= 1;
}
hist_cur_num_bin += num_bin;
cur_num_bin += num_bin;
}
offsets->push_back(cur_num_bin);
CHECK(cur_num_bin == feature_group->bin_offsets_.back());
}
} else {
for (int i = 0; i < feature_group->num_feature_; ++i) {
feature_hist_offsets_.push_back(hist_cur_num_bin + feature_group->bin_offsets_[i]);
}
hist_cur_num_bin += feature_group->bin_offsets_.back();
}
}
feature_hist_offsets_.push_back(hist_cur_num_bin);
num_hist_total_bin_ = static_cast<int>(feature_hist_offsets_.back());
} else {
double sum_dense_ratio = 0.0f;
int ncol = 0;
for (int gid = 0; gid < static_cast<int>(feature_groups.size()); ++gid) {
if (feature_groups[gid]->is_multi_val_) {
ncol += feature_groups[gid]->num_feature_;
} else {
++ncol;
}
for (int fid = 0; fid < feature_groups[gid]->num_feature_; ++fid) {
const auto& bin_mapper = feature_groups[gid]->bin_mappers_[fid];
sum_dense_ratio += 1.0f - bin_mapper->sparse_rate();
}
}
sum_dense_ratio /= ncol;
const bool is_sparse_row_wise = (1.0f - sum_dense_ratio) >=
MultiValBin::multi_val_bin_sparse_threshold ? 1 : 0;
if (is_sparse_row_wise) {
int cur_num_bin = 1;
uint32_t hist_cur_num_bin = 1;
for (int group = 0; group < static_cast<int>(feature_groups.size()); ++group) {
const std::unique_ptr<FeatureGroup>& feature_group = feature_groups[group];
if (feature_group->is_multi_val_) {
for (int i = 0; i < feature_group->num_feature_; ++i) {
offsets->push_back(cur_num_bin);
feature_hist_offsets_.push_back(hist_cur_num_bin);
const std::unique_ptr<BinMapper>& bin_mapper = feature_group->bin_mappers_[i];
int num_bin = bin_mapper->num_bin();
if (bin_mapper->GetMostFreqBin() == 0) {
num_bin -= 1;
}
cur_num_bin += num_bin;
hist_cur_num_bin += num_bin;
}
} else {
offsets->push_back(cur_num_bin);
cur_num_bin += feature_group->bin_offsets_.back() - 1;
for (int i = 0; i < feature_group->num_feature_; ++i) {
feature_hist_offsets_.push_back(hist_cur_num_bin + feature_group->bin_offsets_[i] - 1);
}
hist_cur_num_bin += feature_group->bin_offsets_.back() - 1;
}
}
offsets->push_back(cur_num_bin);
feature_hist_offsets_.push_back(hist_cur_num_bin);
} else {
int cur_num_bin = 0;
uint32_t hist_cur_num_bin = 0;
for (int group = 0; group < static_cast<int>(feature_groups.size()); ++group) {
const std::unique_ptr<FeatureGroup>& feature_group = feature_groups[group];
if (feature_group->is_multi_val_) {
for (int i = 0; i < feature_group->num_feature_; ++i) {
const std::unique_ptr<BinMapper>& bin_mapper = feature_group->bin_mappers_[i];
if (group == 0 && i == 0 && bin_mapper->GetMostFreqBin() > 0) {
cur_num_bin += 1;
hist_cur_num_bin += 1;
}
offsets->push_back(cur_num_bin);
feature_hist_offsets_.push_back(hist_cur_num_bin);
int num_bin = bin_mapper->num_bin();
cur_num_bin += num_bin;
hist_cur_num_bin += num_bin;
if (bin_mapper->GetMostFreqBin() == 0) {
feature_hist_offsets_.back() += 1;
}
}
} else {
offsets->push_back(cur_num_bin);
cur_num_bin += feature_group->bin_offsets_.back();
for (int i = 0; i < feature_group->num_feature_; ++i) {
feature_hist_offsets_.push_back(hist_cur_num_bin + feature_group->bin_offsets_[i]);
}
hist_cur_num_bin += feature_group->bin_offsets_.back();
}
}
offsets->push_back(cur_num_bin);
feature_hist_offsets_.push_back(hist_cur_num_bin);
}
num_hist_total_bin_ = static_cast<int>(feature_hist_offsets_.back());
}
#ifdef USE_CUDA
column_hist_offsets_ = *offsets;
#endif // USE_CUDA
}
void TrainingShareStates::SetMultiValBin(MultiValBin* bin, data_size_t num_data,
const std::vector<std::unique_ptr<FeatureGroup>>& feature_groups,
bool dense_only, bool sparse_only, const int num_grad_quant_bins) {
num_threads = OMP_NUM_THREADS();
if (bin == nullptr) {
return;
}
std::vector<int> feature_groups_contained;
for (int group = 0; group < static_cast<int>(feature_groups.size()); ++group) {
const auto& feature_group = feature_groups[group];
if (feature_group->is_multi_val_) {
if (!dense_only) {
feature_groups_contained.push_back(group);
}
} else if (!sparse_only) {
feature_groups_contained.push_back(group);
}
}
num_total_bin_ += bin->num_bin();
num_elements_per_row_ += bin->num_element_per_row();
multi_val_bin_wrapper_.reset(new MultiValBinWrapper(
bin, num_data, feature_groups_contained, num_grad_quant_bins));
}
} // namespace LightGBM
src/io/tree.cpp
+1060
View File
File diff suppressed because it is too large Load Diff
src/main.cpp
+46
View File
@@ -0,0 +1,46 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/application.h>
#include <iostream>
#include <string>
#ifdef USE_MPI
#include "network/linkers.h"
#endif
int main(int argc, char** argv) {
bool success = false;
try {
LightGBM::Application app(argc, argv);
app.Run();
#ifdef USE_MPI
LightGBM::Linkers::MpiFinalizeIfIsParallel();
#endif
success = true;
}
catch (const std::exception& ex) {
std::cerr << "Met Exceptions:" << std::endl;
std::cerr << ex.what() << std::endl;
}
catch (const std::string& ex) {
std::cerr << "Met Exceptions:" << std::endl;
std::cerr << ex << std::endl;
}
catch (...) {
std::cerr << "Unknown Exceptions" << std::endl;
}
if (!success) {
#ifdef USE_MPI
LightGBM::Linkers::MpiAbortIfIsParallel();
#endif
exit(-1);
}
}
src/metric/binary_metric.hpp
+389
View File
@@ -0,0 +1,389 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_METRIC_BINARY_METRIC_HPP_
#define LIGHTGBM_SRC_METRIC_BINARY_METRIC_HPP_
#include <LightGBM/metric.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/log.h>
#include <string>
#include <algorithm>
#include <sstream>
#include <vector>
namespace LightGBM {
/*!
* \brief Metric for binary classification task.
* Use static class "PointWiseLossCalculator" to calculate loss point-wise
*/
template<typename PointWiseLossCalculator>
class BinaryMetric: public Metric {
public:
explicit BinaryMetric(const Config&) {
}
virtual ~BinaryMetric() {
}
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back(PointWiseLossCalculator::Name());
num_data_ = num_data;
// get label
label_ = metadata.label();
// get weights
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data; ++i) {
sum_weights_ += weights_[i];
}
}
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return -1.0f;
}
std::vector<double> Eval(const double* score, const ObjectiveFunction* objective) const override {
double sum_loss = 0.0f;
if (objective == nullptr) {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], score[i]);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], score[i]) * weights_[i];
}
}
} else {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double prob = 0;
objective->ConvertOutput(&score[i], &prob);
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], prob);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double prob = 0;
objective->ConvertOutput(&score[i], &prob);
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], prob) * weights_[i];
}
}
}
double loss = sum_loss / sum_weights_;
return std::vector<double>(1, loss);
}
protected:
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Pointer of weighs */
const label_t* weights_;
/*! \brief Sum weights */
double sum_weights_;
/*! \brief Name of test set */
std::vector<std::string> name_;
};
/*!
* \brief Log loss metric for binary classification task.
*/
class BinaryLoglossMetric: public BinaryMetric<BinaryLoglossMetric> {
public:
explicit BinaryLoglossMetric(const Config& config) :BinaryMetric<BinaryLoglossMetric>(config) {}
inline static double LossOnPoint(label_t label, double prob) {
if (label <= 0) {
if (1.0f - prob > kEpsilon) {
return -std::log(1.0f - prob);
}
} else {
if (prob > kEpsilon) {
return -std::log(prob);
}
}
return -std::log(kEpsilon);
}
inline static const char* Name() {
return "binary_logloss";
}
};
/*!
* \brief Error rate metric for binary classification task.
*/
class BinaryErrorMetric: public BinaryMetric<BinaryErrorMetric> {
public:
explicit BinaryErrorMetric(const Config& config) :BinaryMetric<BinaryErrorMetric>(config) {}
inline static double LossOnPoint(label_t label, double prob) {
if (prob <= 0.5f) {
return label > 0;
} else {
return label <= 0;
}
}
inline static const char* Name() {
return "binary_error";
}
};
/*!
* \brief Auc Metric for binary classification task.
*/
class AUCMetric: public Metric {
public:
explicit AUCMetric(const Config&) {
}
virtual ~AUCMetric() {
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return 1.0f;
}
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back("auc");
num_data_ = num_data;
// get label
label_ = metadata.label();
// get weights
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data; ++i) {
sum_weights_ += weights_[i];
}
}
}
std::vector<double> Eval(const double* score, const ObjectiveFunction*) const override {
// get indices sorted by score, descent order
std::vector<data_size_t> sorted_idx;
for (data_size_t i = 0; i < num_data_; ++i) {
sorted_idx.emplace_back(i);
}
Common::ParallelSort(sorted_idx.begin(), sorted_idx.end(), [score](data_size_t a, data_size_t b) {return score[a] > score[b]; });
// temp sum of positive label
double cur_pos = 0.0f;
// total sum of positive label
double sum_pos = 0.0f;
// accumulate of AUC
double accum = 0.0f;
// temp sum of negative label
double cur_neg = 0.0f;
double threshold = score[sorted_idx[0]];
if (weights_ == nullptr) { // no weights
for (data_size_t i = 0; i < num_data_; ++i) {
const label_t cur_label = label_[sorted_idx[i]];
const double cur_score = score[sorted_idx[i]];
// new threshold
if (cur_score != threshold) {
threshold = cur_score;
// accumulate
accum += cur_neg*(cur_pos * 0.5f + sum_pos);
sum_pos += cur_pos;
// reset
cur_neg = cur_pos = 0.0f;
}
cur_neg += (cur_label <= 0);
cur_pos += (cur_label > 0);
}
} else { // has weights
for (data_size_t i = 0; i < num_data_; ++i) {
const label_t cur_label = label_[sorted_idx[i]];
const double cur_score = score[sorted_idx[i]];
const label_t cur_weight = weights_[sorted_idx[i]];
// new threshold
if (cur_score != threshold) {
threshold = cur_score;
// accumulate
accum += cur_neg*(cur_pos * 0.5f + sum_pos);
sum_pos += cur_pos;
// reset
cur_neg = cur_pos = 0.0f;
}
cur_neg += (cur_label <= 0)*cur_weight;
cur_pos += (cur_label > 0)*cur_weight;
}
}
accum += cur_neg*(cur_pos * 0.5f + sum_pos);
sum_pos += cur_pos;
double auc = 1.0f;
if (sum_pos > 0.0f && sum_pos != sum_weights_) {
auc = accum / (sum_pos *(sum_weights_ - sum_pos));
}
return std::vector<double>(1, auc);
}
private:
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Pointer of weighs */
const label_t* weights_;
/*! \brief Sum weights */
double sum_weights_;
/*! \brief Name of test set */
std::vector<std::string> name_;
};
/*!
* \brief Average Precision Metric for binary classification task.
*/
class AveragePrecisionMetric: public Metric {
public:
explicit AveragePrecisionMetric(const Config&) {
}
virtual ~AveragePrecisionMetric() {
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return 1.0f;
}
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back("average_precision");
num_data_ = num_data;
// get label
label_ = metadata.label();
// get weights
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data; ++i) {
sum_weights_ += weights_[i];
}
}
}
std::vector<double> Eval(const double* score, const ObjectiveFunction*) const override {
// get indices sorted by score, descending order
std::vector<data_size_t> sorted_idx;
for (data_size_t i = 0; i < num_data_; ++i) {
sorted_idx.emplace_back(i);
}
Common::ParallelSort(sorted_idx.begin(), sorted_idx.end(), [score](data_size_t a, data_size_t b) {return score[a] > score[b]; });
// temp sum of positive label
double cur_actual_pos = 0.0f;
// total sum of positive label
double sum_actual_pos = 0.0f;
// total sum of predicted positive
double sum_pred_pos = 0.0f;
// accumulated precision
double accum_prec = 1.0f;
// accumulated pr-auc
double accum = 0.0f;
// temp sum of negative label
double cur_neg = 0.0f;
double threshold = score[sorted_idx[0]];
if (weights_ == nullptr) { // no weights
for (data_size_t i = 0; i < num_data_; ++i) {
const label_t cur_label = label_[sorted_idx[i]];
const double cur_score = score[sorted_idx[i]];
// new threshold
if (cur_score != threshold) {
threshold = cur_score;
// accumulate
sum_actual_pos += cur_actual_pos;
sum_pred_pos += cur_actual_pos + cur_neg;
accum_prec = sum_actual_pos / sum_pred_pos;
accum += cur_actual_pos * accum_prec;
// reset
cur_neg = cur_actual_pos = 0.0f;
}
cur_neg += (cur_label <= 0);
cur_actual_pos += (cur_label > 0);
}
} else { // has weights
for (data_size_t i = 0; i < num_data_; ++i) {
const label_t cur_label = label_[sorted_idx[i]];
const double cur_score = score[sorted_idx[i]];
const label_t cur_weight = weights_[sorted_idx[i]];
// new threshold
if (cur_score != threshold) {
threshold = cur_score;
// accumulate
sum_actual_pos += cur_actual_pos;
sum_pred_pos += cur_actual_pos + cur_neg;
accum_prec = sum_actual_pos / sum_pred_pos;
accum += cur_actual_pos * accum_prec;
// reset
cur_neg = cur_actual_pos = 0.0f;
}
cur_neg += (cur_label <= 0) * cur_weight;
cur_actual_pos += (cur_label > 0) * cur_weight;
}
}
sum_actual_pos += cur_actual_pos;
sum_pred_pos += cur_actual_pos + cur_neg;
accum_prec = sum_actual_pos / sum_pred_pos;
accum += cur_actual_pos * accum_prec;
double ap = 1.0f;
if (sum_actual_pos > 0.0f && sum_actual_pos != sum_weights_) {
ap = accum / sum_actual_pos;
}
return std::vector<double>(1, ap);
}
private:
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Pointer of weighs */
const label_t* weights_;
/*! \brief Sum weights */
double sum_weights_;
/*! \brief Name of test set */
std::vector<std::string> name_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_METRIC_BINARY_METRIC_HPP_
src/metric/cuda/cuda_binary_metric.cpp
+35
View File
@@ -0,0 +1,35 @@
/*!
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include "cuda_binary_metric.hpp"
#include <vector>
namespace LightGBM {
template <typename HOST_METRIC, typename CUDA_METRIC>
std::vector<double> CUDABinaryMetricInterface<HOST_METRIC, CUDA_METRIC>::Eval(const double* score, const ObjectiveFunction* objective) const {
const double* score_convert = score;
if (objective != nullptr && objective->NeedConvertOutputCUDA()) {
this->score_convert_buffer_.Resize(static_cast<size_t>(this->num_data_) * static_cast<size_t>(this->num_class_));
score_convert = objective->ConvertOutputCUDA(this->num_data_, score, this->score_convert_buffer_.RawData());
}
double sum_loss = 0.0, sum_weight = 0.0;
this->LaunchEvalKernel(score_convert, &sum_loss, &sum_weight);
const double eval_score = sum_loss / sum_weight;
return std::vector<double>{eval_score};
}
CUDABinaryLoglossMetric::CUDABinaryLoglossMetric(const Config& config):CUDABinaryMetricInterface<BinaryLoglossMetric, CUDABinaryLoglossMetric>(config) {}
CUDABinaryErrorMetric::CUDABinaryErrorMetric(const Config& config):CUDABinaryMetricInterface<BinaryErrorMetric, CUDABinaryErrorMetric>(config) {}
} // namespace LightGBM
#endif // USE_CUDA
src/metric/cuda/cuda_binary_metric.hpp
+73
View File
@@ -0,0 +1,73 @@
/*!
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_METRIC_CUDA_CUDA_BINARY_METRIC_HPP_
#define LIGHTGBM_SRC_METRIC_CUDA_CUDA_BINARY_METRIC_HPP_
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_metric.hpp>
#include <LightGBM/cuda/cuda_utils.hu>
#include <vector>
#include "cuda_regression_metric.hpp"
#include "../binary_metric.hpp"
namespace LightGBM {
template <typename HOST_METRIC, typename CUDA_METRIC>
class CUDABinaryMetricInterface: public CUDAPointwiseMetricInterface<HOST_METRIC, CUDA_METRIC> {
public:
explicit CUDABinaryMetricInterface(const Config& config): CUDAPointwiseMetricInterface<HOST_METRIC, CUDA_METRIC>(config) {}
virtual ~CUDABinaryMetricInterface() {}
std::vector<double> Eval(const double* score, const ObjectiveFunction* objective) const override;
};
class CUDABinaryLoglossMetric: public CUDABinaryMetricInterface<BinaryLoglossMetric, CUDABinaryLoglossMetric> {
public:
explicit CUDABinaryLoglossMetric(const Config& config);
virtual ~CUDABinaryLoglossMetric() {}
__device__ static double MetricOnPointCUDA(label_t label, double score, const double /*param*/) {
// score should have been converted to probability
if (label <= 0) {
if (1.0f - score > kEpsilon) {
return -log(1.0f - score);
}
} else {
if (score > kEpsilon) {
return -log(score);
}
}
return -log(kEpsilon);
}
};
class CUDABinaryErrorMetric: public CUDABinaryMetricInterface<BinaryErrorMetric, CUDABinaryErrorMetric> {
public:
explicit CUDABinaryErrorMetric(const Config& config);
virtual ~CUDABinaryErrorMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, const double /*param*/) {
if (score <= 0.5f) {
return label > 0;
} else {
return label <= 0;
}
}
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_METRIC_CUDA_CUDA_BINARY_METRIC_HPP_
src/metric/cuda/cuda_pointwise_metric.cpp
+52
View File
@@ -0,0 +1,52 @@
/*!
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include "cuda_binary_metric.hpp"
#include "cuda_pointwise_metric.hpp"
#include "cuda_regression_metric.hpp"
namespace LightGBM {
template <typename HOST_METRIC, typename CUDA_METRIC>
void CUDAPointwiseMetricInterface<HOST_METRIC, CUDA_METRIC>::Init(const Metadata& metadata, data_size_t num_data) {
CUDAMetricInterface<HOST_METRIC>::Init(metadata, num_data);
const int max_num_reduce_blocks = (this->num_data_ + NUM_DATA_PER_EVAL_THREAD - 1) / NUM_DATA_PER_EVAL_THREAD;
if (this->cuda_weights_ == nullptr) {
reduce_block_buffer_.Resize(max_num_reduce_blocks);
} else {
reduce_block_buffer_.Resize(max_num_reduce_blocks * 2);
}
const int max_num_reduce_blocks_inner = (max_num_reduce_blocks + NUM_DATA_PER_EVAL_THREAD - 1) / NUM_DATA_PER_EVAL_THREAD;
if (this->cuda_weights_ == nullptr) {
reduce_block_buffer_inner_.Resize(max_num_reduce_blocks_inner);
} else {
reduce_block_buffer_inner_.Resize(max_num_reduce_blocks_inner * 2);
}
}
// Regression metrics
template void CUDAPointwiseMetricInterface<RMSEMetric, CUDARMSEMetric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<L2Metric, CUDAL2Metric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<QuantileMetric, CUDAQuantileMetric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<L1Metric, CUDAL1Metric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<HuberLossMetric, CUDAHuberLossMetric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<FairLossMetric, CUDAFairLossMetric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<PoissonMetric, CUDAPoissonMetric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<MAPEMetric, CUDAMAPEMetric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<GammaMetric, CUDAGammaMetric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<GammaDevianceMetric, CUDAGammaDevianceMetric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<TweedieMetric, CUDATweedieMetric>::Init(const Metadata& metadata, data_size_t num_data);
// Binary metrics
template void CUDAPointwiseMetricInterface<BinaryLoglossMetric, CUDABinaryLoglossMetric>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDAPointwiseMetricInterface<BinaryErrorMetric, CUDABinaryErrorMetric>::Init(const Metadata& metadata, data_size_t num_data);
} // namespace LightGBM
#endif // USE_CUDA
src/metric/cuda/cuda_pointwise_metric.cu
+85
View File
@@ -0,0 +1,85 @@
/*!
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
* Modifications Copyright(C) 2023 Advanced Micro Devices, Inc. All rights reserved.
*/
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_algorithms.hpp>
#include <LightGBM/cuda/cuda_rocm_interop.h>
#include "cuda_binary_metric.hpp"
#include "cuda_pointwise_metric.hpp"
#include "cuda_regression_metric.hpp"
namespace LightGBM {
template <typename CUDA_METRIC, bool USE_WEIGHTS>
__global__ void EvalKernel(const data_size_t num_data, const label_t* labels, const label_t* weights,
const double* scores, double* reduce_block_buffer, const double param) {
__shared__ double shared_mem_buffer[WARPSIZE];
const data_size_t index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
double point_metric = 0.0;
if (index < num_data) {
point_metric = USE_WEIGHTS ?
CUDA_METRIC::MetricOnPointCUDA(labels[index], scores[index], param) * weights[index] :
CUDA_METRIC::MetricOnPointCUDA(labels[index], scores[index], param);
}
const double block_sum_point_metric = ShuffleReduceSum<double>(point_metric, shared_mem_buffer, NUM_DATA_PER_EVAL_THREAD);
if (threadIdx.x == 0) {
reduce_block_buffer[blockIdx.x] = block_sum_point_metric;
}
if (USE_WEIGHTS) {
double weight = 0.0;
if (index < num_data) {
weight = static_cast<double>(weights[index]);
const double block_sum_weight = ShuffleReduceSum<double>(weight, shared_mem_buffer, NUM_DATA_PER_EVAL_THREAD);
if (threadIdx.x == 0) {
reduce_block_buffer[blockIdx.x + gridDim.x] = block_sum_weight;
}
}
}
}
template <typename HOST_METRIC, typename CUDA_METRIC>
void CUDAPointwiseMetricInterface<HOST_METRIC, CUDA_METRIC>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const {
const int num_blocks = (this->num_data_ + NUM_DATA_PER_EVAL_THREAD - 1) / NUM_DATA_PER_EVAL_THREAD;
if (this->cuda_weights_ != nullptr) {
EvalKernel<CUDA_METRIC, true><<<num_blocks, NUM_DATA_PER_EVAL_THREAD>>>(
this->num_data_, this->cuda_labels_, this->cuda_weights_, score, reduce_block_buffer_.RawData(), GetParamFromConfig());
} else {
EvalKernel<CUDA_METRIC, false><<<num_blocks, NUM_DATA_PER_EVAL_THREAD>>>(
this->num_data_, this->cuda_labels_, this->cuda_weights_, score, reduce_block_buffer_.RawData(), GetParamFromConfig());
}
ShuffleReduceSumGlobal<double, double>(reduce_block_buffer_.RawData(), num_blocks, reduce_block_buffer_inner_.RawData());
CopyFromCUDADeviceToHost<double>(sum_loss, reduce_block_buffer_inner_.RawData(), 1, __FILE__, __LINE__);
*sum_weight = static_cast<double>(this->num_data_);
if (this->cuda_weights_ != nullptr) {
ShuffleReduceSumGlobal<double, double>(reduce_block_buffer_.RawData() + num_blocks, num_blocks, reduce_block_buffer_inner_.RawData());
CopyFromCUDADeviceToHost<double>(sum_weight, reduce_block_buffer_inner_.RawData(), 1, __FILE__, __LINE__);
}
}
// Regression metrics
template void CUDAPointwiseMetricInterface<RMSEMetric, CUDARMSEMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<L2Metric, CUDAL2Metric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<QuantileMetric, CUDAQuantileMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<L1Metric, CUDAL1Metric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<HuberLossMetric, CUDAHuberLossMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<FairLossMetric, CUDAFairLossMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<PoissonMetric, CUDAPoissonMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<MAPEMetric, CUDAMAPEMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<GammaMetric, CUDAGammaMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<GammaDevianceMetric, CUDAGammaDevianceMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<TweedieMetric, CUDATweedieMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
// Binary metrics
template void CUDAPointwiseMetricInterface<BinaryLoglossMetric, CUDABinaryLoglossMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
template void CUDAPointwiseMetricInterface<BinaryErrorMetric, CUDABinaryErrorMetric>::LaunchEvalKernel(const double* score, double* sum_loss, double* sum_weight) const;
} // namespace LightGBM
#endif // USE_CUDA
src/metric/cuda/cuda_pointwise_metric.hpp
+46
View File
@@ -0,0 +1,46 @@
/*!
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_METRIC_CUDA_CUDA_POINTWISE_METRIC_HPP_
#define LIGHTGBM_SRC_METRIC_CUDA_CUDA_POINTWISE_METRIC_HPP_
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_metric.hpp>
#include <LightGBM/cuda/cuda_utils.hu>
#include <vector>
#define NUM_DATA_PER_EVAL_THREAD (1024)
namespace LightGBM {
template <typename HOST_METRIC, typename CUDA_METRIC>
class CUDAPointwiseMetricInterface: public CUDAMetricInterface<HOST_METRIC> {
public:
explicit CUDAPointwiseMetricInterface(const Config& config): CUDAMetricInterface<HOST_METRIC>(config), num_class_(config.num_class) {}
virtual ~CUDAPointwiseMetricInterface() {}
void Init(const Metadata& metadata, data_size_t num_data) override;
protected:
void LaunchEvalKernel(const double* score_convert, double* sum_loss, double* sum_weight) const;
virtual double GetParamFromConfig() const { return 0.0; }
mutable CUDAVector<double> score_convert_buffer_;
CUDAVector<double> reduce_block_buffer_;
CUDAVector<double> reduce_block_buffer_inner_;
const int num_class_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_METRIC_CUDA_CUDA_POINTWISE_METRIC_HPP_
src/metric/cuda/cuda_regression_metric.cpp
+53
View File
@@ -0,0 +1,53 @@
/*!
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include <vector>
#include "cuda_regression_metric.hpp"
namespace LightGBM {
template <typename HOST_METRIC, typename CUDA_METRIC>
std::vector<double> CUDARegressionMetricInterface<HOST_METRIC, CUDA_METRIC>::Eval(const double* score, const ObjectiveFunction* objective) const {
const double* score_convert = score;
if (objective != nullptr && objective->NeedConvertOutputCUDA()) {
this->score_convert_buffer_.Resize(static_cast<size_t>(this->num_data_) * static_cast<size_t>(this->num_class_));
score_convert = objective->ConvertOutputCUDA(this->num_data_, score, this->score_convert_buffer_.RawData());
}
double sum_loss = 0.0, sum_weight = 0.0;
this->LaunchEvalKernel(score_convert, &sum_loss, &sum_weight);
const double eval_score = this->AverageLoss(sum_loss, sum_weight);
return std::vector<double>{eval_score};
}
CUDARMSEMetric::CUDARMSEMetric(const Config& config): CUDARegressionMetricInterface<RMSEMetric, CUDARMSEMetric>(config) {}
CUDAL2Metric::CUDAL2Metric(const Config& config): CUDARegressionMetricInterface<L2Metric, CUDAL2Metric>(config) {}
CUDAQuantileMetric::CUDAQuantileMetric(const Config& config): CUDARegressionMetricInterface<QuantileMetric, CUDAQuantileMetric>(config), alpha_(config.alpha) {}
CUDAL1Metric::CUDAL1Metric(const Config& config): CUDARegressionMetricInterface<L1Metric, CUDAL1Metric>(config) {}
CUDAHuberLossMetric::CUDAHuberLossMetric(const Config& config): CUDARegressionMetricInterface<HuberLossMetric, CUDAHuberLossMetric>(config), alpha_(config.alpha) {}
CUDAFairLossMetric::CUDAFairLossMetric(const Config& config): CUDARegressionMetricInterface<FairLossMetric, CUDAFairLossMetric>(config) , fair_c_(config.fair_c) {}
CUDAPoissonMetric::CUDAPoissonMetric(const Config& config): CUDARegressionMetricInterface<PoissonMetric, CUDAPoissonMetric>(config) {}
CUDAMAPEMetric::CUDAMAPEMetric(const Config& config): CUDARegressionMetricInterface<MAPEMetric, CUDAMAPEMetric>(config) {}
CUDAGammaMetric::CUDAGammaMetric(const Config& config): CUDARegressionMetricInterface<GammaMetric, CUDAGammaMetric>(config) {}
CUDAGammaDevianceMetric::CUDAGammaDevianceMetric(const Config& config): CUDARegressionMetricInterface<GammaDevianceMetric, CUDAGammaDevianceMetric>(config) {}
CUDATweedieMetric::CUDATweedieMetric(const Config& config): CUDARegressionMetricInterface<TweedieMetric, CUDATweedieMetric>(config) , tweedie_variance_power_(config.tweedie_variance_power) {}
} // namespace LightGBM
#endif // USE_CUDA
src/metric/cuda/cuda_regression_metric.hpp
+216
View File
@@ -0,0 +1,216 @@
/*!
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_METRIC_CUDA_CUDA_REGRESSION_METRIC_HPP_
#define LIGHTGBM_SRC_METRIC_CUDA_CUDA_REGRESSION_METRIC_HPP_
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_metric.hpp>
#include <LightGBM/cuda/cuda_utils.hu>
#include <vector>
#include "cuda_pointwise_metric.hpp"
#include "../regression_metric.hpp"
namespace LightGBM {
template <typename HOST_METRIC, typename CUDA_METRIC>
class CUDARegressionMetricInterface: public CUDAPointwiseMetricInterface<HOST_METRIC, CUDA_METRIC> {
public:
explicit CUDARegressionMetricInterface(const Config& config):
CUDAPointwiseMetricInterface<HOST_METRIC, CUDA_METRIC>(config) {}
virtual ~CUDARegressionMetricInterface() {}
std::vector<double> Eval(const double* score, const ObjectiveFunction* objective) const override;
};
class CUDARMSEMetric: public CUDARegressionMetricInterface<RMSEMetric, CUDARMSEMetric> {
public:
explicit CUDARMSEMetric(const Config& config);
virtual ~CUDARMSEMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double /*alpha*/) {
return (score - label) * (score - label);
}
};
class CUDAL2Metric : public CUDARegressionMetricInterface<L2Metric, CUDAL2Metric> {
public:
explicit CUDAL2Metric(const Config& config);
virtual ~CUDAL2Metric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double /*alpha*/) {
return (score - label) * (score - label);
}
};
class CUDAQuantileMetric : public CUDARegressionMetricInterface<QuantileMetric, CUDAQuantileMetric> {
public:
explicit CUDAQuantileMetric(const Config& config);
virtual ~CUDAQuantileMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double alpha) {
double delta = label - score;
if (delta < 0) {
return (alpha - 1.0f) * delta;
} else {
return alpha * delta;
}
}
double GetParamFromConfig() const override {
return alpha_;
}
private:
const double alpha_;
};
class CUDAL1Metric : public CUDARegressionMetricInterface<L1Metric, CUDAL1Metric> {
public:
explicit CUDAL1Metric(const Config& config);
virtual ~CUDAL1Metric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double /*alpha*/) {
return std::fabs(score - label);
}
};
class CUDAHuberLossMetric : public CUDARegressionMetricInterface<HuberLossMetric, CUDAHuberLossMetric> {
public:
explicit CUDAHuberLossMetric(const Config& config);
virtual ~CUDAHuberLossMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double alpha) {
const double diff = score - label;
if (std::abs(diff) <= alpha) {
return 0.5f * diff * diff;
} else {
return alpha * (std::abs(diff) - 0.5f * alpha);
}
}
double GetParamFromConfig() const override {
return alpha_;
}
private:
const double alpha_;
};
class CUDAFairLossMetric : public CUDARegressionMetricInterface<FairLossMetric, CUDAFairLossMetric> {
public:
explicit CUDAFairLossMetric(const Config& config);
virtual ~CUDAFairLossMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double fair_c) {
const double x = std::fabs(score - label);
const double c = fair_c;
return c * x - c * c * std::log1p(x / c);
}
double GetParamFromConfig() const override {
return fair_c_;
}
private:
const double fair_c_;
};
class CUDAPoissonMetric : public CUDARegressionMetricInterface<PoissonMetric, CUDAPoissonMetric> {
public:
explicit CUDAPoissonMetric(const Config& config);
virtual ~CUDAPoissonMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double /*alpha*/) {
const double eps = 1e-10f;
if (score < eps) {
score = eps;
}
return score - label * std::log(score);
}
};
class CUDAMAPEMetric : public CUDARegressionMetricInterface<MAPEMetric, CUDAMAPEMetric> {
public:
explicit CUDAMAPEMetric(const Config& config);
virtual ~CUDAMAPEMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double /*alpha*/) {
return std::fabs((label - score)) / fmax(1.0f, std::fabs(label));
}
};
class CUDAGammaMetric : public CUDARegressionMetricInterface<GammaMetric, CUDAGammaMetric> {
public:
explicit CUDAGammaMetric(const Config& config);
virtual ~CUDAGammaMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double /*alpha*/) {
const double psi = 1.0;
const double theta = -1.0 / score;
const double a = psi;
const double b = -SafeLog(-theta);
const double c = 1. / psi * SafeLog(label / psi) - SafeLog(label) - 0; // 0 = std::lgamma(1.0 / psi) = std::lgamma(1.0);
return -((label * theta - b) / a + c);
}
};
class CUDAGammaDevianceMetric : public CUDARegressionMetricInterface<GammaDevianceMetric, CUDAGammaDevianceMetric> {
public:
explicit CUDAGammaDevianceMetric(const Config& config);
virtual ~CUDAGammaDevianceMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double /*alpha*/) {
const double epsilon = 1.0e-9;
const double tmp = label / (score + epsilon);
return tmp - SafeLog(tmp) - 1;
}
};
class CUDATweedieMetric : public CUDARegressionMetricInterface<TweedieMetric, CUDATweedieMetric> {
public:
explicit CUDATweedieMetric(const Config& config);
virtual ~CUDATweedieMetric() {}
__device__ inline static double MetricOnPointCUDA(label_t label, double score, double tweedie_variance_power) {
const double rho = tweedie_variance_power;
const double eps = 1e-10f;
if (score < eps) {
score = eps;
}
const double a = label * std::exp((1 - rho) * std::log(score)) / (1 - rho);
const double b = std::exp((2 - rho) * std::log(score)) / (2 - rho);
return -a + b;
}
double GetParamFromConfig() const override {
return tweedie_variance_power_;
}
private:
const double tweedie_variance_power_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_METRIC_CUDA_CUDA_REGRESSION_METRIC_HPP_
src/metric/dcg_calculator.cpp
+174
View File
@@ -0,0 +1,174 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/metric.h>
#include <LightGBM/utils/log.h>
#include <algorithm>
#include <cmath>
#include <vector>
namespace LightGBM {
/*! \brief Declaration for some static members */
std::vector<double> DCGCalculator::label_gain_;
std::vector<double> DCGCalculator::discount_;
const data_size_t DCGCalculator::kMaxPosition = 10000;
void DCGCalculator::DefaultEvalAt(std::vector<int>* eval_at) {
auto& ref_eval_at = *eval_at;
if (ref_eval_at.empty()) {
for (int i = 1; i <= 5; ++i) {
ref_eval_at.push_back(i);
}
} else {
for (size_t i = 0; i < eval_at->size(); ++i) {
CHECK_GT(ref_eval_at[i], 0);
}
}
}
void DCGCalculator::DefaultLabelGain(std::vector<double>* label_gain) {
if (!label_gain->empty()) {
return;
}
// label_gain = 2^i - 1, may overflow, so we use 31 here
const int max_label = 31;
label_gain->push_back(0.0f);
for (int i = 1; i < max_label; ++i) {
label_gain->push_back(static_cast<double>((1 << i) - 1));
}
}
void DCGCalculator::Init(const std::vector<double>& input_label_gain) {
label_gain_.resize(input_label_gain.size());
for (size_t i = 0; i < input_label_gain.size(); ++i) {
label_gain_[i] = static_cast<double>(input_label_gain[i]);
}
discount_.resize(kMaxPosition);
for (data_size_t i = 0; i < kMaxPosition; ++i) {
discount_[i] = 1.0 / std::log2(2.0 + i);
}
}
double DCGCalculator::CalMaxDCGAtK(data_size_t k, const label_t* label, data_size_t num_data) {
double ret = 0.0f;
// counts for all labels
std::vector<data_size_t> label_cnt(label_gain_.size(), 0);
for (data_size_t i = 0; i < num_data; ++i) {
++label_cnt[static_cast<int>(label[i])];
}
int top_label = static_cast<int>(label_gain_.size()) - 1;
if (k > num_data) {
k = num_data;
}
// start from top label, and accumulate DCG
for (data_size_t j = 0; j < k; ++j) {
while (top_label > 0 && label_cnt[top_label] <= 0) {
top_label -= 1;
}
if (top_label < 0) {
break;
}
ret += discount_[j] * label_gain_[top_label];
label_cnt[top_label] -= 1;
}
return ret;
}
void DCGCalculator::CalMaxDCG(const std::vector<data_size_t>& ks,
const label_t* label,
data_size_t num_data,
std::vector<double>* out) {
std::vector<data_size_t> label_cnt(label_gain_.size(), 0);
// counts for all labels
for (data_size_t i = 0; i < num_data; ++i) {
++label_cnt[static_cast<int>(label[i])];
}
double cur_result = 0.0f;
data_size_t cur_left = 0;
int top_label = static_cast<int>(label_gain_.size()) - 1;
// calculate k Max DCG by one pass
for (size_t i = 0; i < ks.size(); ++i) {
data_size_t cur_k = ks[i];
if (cur_k > num_data) {
cur_k = num_data;
}
for (data_size_t j = cur_left; j < cur_k; ++j) {
while (top_label > 0 && label_cnt[top_label] <= 0) {
top_label -= 1;
}
if (top_label < 0) {
break;
}
cur_result += discount_[j] * label_gain_[top_label];
label_cnt[top_label] -= 1;
}
(*out)[i] = cur_result;
cur_left = cur_k;
}
}
void DCGCalculator::CalDCG(const std::vector<data_size_t>& ks, const label_t* label,
const double * score, data_size_t num_data, std::vector<double>* out) {
// get sorted indices by score
std::vector<data_size_t> sorted_idx(num_data);
for (data_size_t i = 0; i < num_data; ++i) {
sorted_idx[i] = i;
}
std::stable_sort(sorted_idx.begin(), sorted_idx.end(),
[score](data_size_t a, data_size_t b) {return score[a] > score[b]; });
double cur_result = 0.0f;
data_size_t cur_left = 0;
// calculate multi dcg by one pass
for (size_t i = 0; i < ks.size(); ++i) {
data_size_t cur_k = ks[i];
if (cur_k > num_data) {
cur_k = num_data;
}
for (data_size_t j = cur_left; j < cur_k; ++j) {
data_size_t idx = sorted_idx[j];
cur_result += label_gain_[static_cast<int>(label[idx])] * discount_[j];
}
(*out)[i] = cur_result;
cur_left = cur_k;
}
}
void DCGCalculator::CheckMetadata(const Metadata& metadata, data_size_t num_queries) {
const data_size_t* query_boundaries = metadata.query_boundaries();
if (num_queries > 0 && query_boundaries != nullptr) {
for (data_size_t i = 0; i < num_queries; i++) {
data_size_t num_rows = query_boundaries[i + 1] - query_boundaries[i];
if (num_rows > kMaxPosition) {
Log::Fatal("Number of rows %i exceeds upper limit of %i for a query", static_cast<int>(num_rows), static_cast<int>(kMaxPosition));
}
}
}
}
void DCGCalculator::CheckLabel(const label_t* label, data_size_t num_data) {
for (data_size_t i = 0; i < num_data; ++i) {
label_t delta = std::fabs(label[i] - static_cast<int>(label[i]));
if (delta > kEpsilon) {
Log::Fatal("label should be int type (met %f) for ranking task,\n"
"for the gain of label, please set the label_gain parameter", label[i]);
}
if (label[i] < 0) {
Log::Fatal("Label should be non-negative (met %f) for ranking task", label[i]);
}
if (static_cast<size_t>(label[i]) >= label_gain_.size()) {
Log::Fatal("Label %zu is not less than the number of label mappings (%zu)", static_cast<size_t>(label[i]), label_gain_.size());
}
}
}
} // namespace LightGBM
src/metric/map_metric.hpp
+169
View File
@@ -0,0 +1,169 @@
/*!
* Copyright (c) 2017-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2017-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_METRIC_MAP_METRIC_HPP_
#define LIGHTGBM_SRC_METRIC_MAP_METRIC_HPP_
#include <LightGBM/metric.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/log.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <string>
#include <algorithm>
#include <sstream>
#include <vector>
namespace LightGBM {
class MapMetric:public Metric {
public:
explicit MapMetric(const Config& config) {
// get eval position
eval_at_ = config.eval_at;
DCGCalculator::DefaultEvalAt(&eval_at_);
}
~MapMetric() {
}
void Init(const Metadata& metadata, data_size_t num_data) override {
for (auto k : eval_at_) {
name_.emplace_back(std::string("map@") + std::to_string(k));
}
num_data_ = num_data;
// get label
label_ = metadata.label();
// get query boundaries
query_boundaries_ = metadata.query_boundaries();
if (query_boundaries_ == nullptr) {
Log::Fatal("For MAP metric, there should be query information");
}
num_queries_ = metadata.num_queries();
Log::Info("Total groups: %d, total data: %d", num_queries_, num_data_);
// get query weights
query_weights_ = metadata.query_weights();
if (query_weights_ == nullptr) {
sum_query_weights_ = static_cast<double>(num_queries_);
} else {
sum_query_weights_ = 0.0f;
for (data_size_t i = 0; i < num_queries_; ++i) {
sum_query_weights_ += query_weights_[i];
}
}
npos_per_query_.resize(num_queries_, 0);
for (data_size_t i = 0; i < num_queries_; ++i) {
for (data_size_t j = query_boundaries_[i]; j < query_boundaries_[i + 1]; ++j) {
if (label_[j] > 0.5f) {
++npos_per_query_[i];
}
}
}
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return 1.0f;
}
void CalMapAtK(std::vector<int> ks, data_size_t npos, const label_t* label,
const double* score, data_size_t num_data, std::vector<double>* out) const {
// get sorted indices by score
std::vector<data_size_t> sorted_idx;
for (data_size_t i = 0; i < num_data; ++i) {
sorted_idx.emplace_back(i);
}
std::stable_sort(sorted_idx.begin(), sorted_idx.end(),
[score](data_size_t a, data_size_t b) {return score[a] > score[b]; });
int num_hit = 0;
double sum_ap = 0.0f;
data_size_t cur_left = 0;
for (size_t i = 0; i < ks.size(); ++i) {
data_size_t cur_k = static_cast<data_size_t>(ks[i]);
if (cur_k > num_data) {
cur_k = num_data;
}
for (data_size_t j = cur_left; j < cur_k; ++j) {
data_size_t idx = sorted_idx[j];
if (label[idx] > 0.5f) {
++num_hit;
sum_ap += static_cast<double>(num_hit) / (j + 1.0f);
}
}
if (npos > 0) {
(*out)[i] = sum_ap / std::min(npos, cur_k);
} else {
(*out)[i] = 1.0f;
}
cur_left = cur_k;
}
}
std::vector<double> Eval(const double* score, const ObjectiveFunction*) const override {
// some buffers for multi-threading sum up
int num_threads = OMP_NUM_THREADS();
std::vector<std::vector<double>> result_buffer_;
for (int i = 0; i < num_threads; ++i) {
result_buffer_.emplace_back(eval_at_.size(), 0.0f);
}
std::vector<double> tmp_map(eval_at_.size(), 0.0f);
if (query_weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(guided) firstprivate(tmp_map)
for (data_size_t i = 0; i < num_queries_; ++i) {
const int tid = omp_get_thread_num();
CalMapAtK(eval_at_, npos_per_query_[i], label_ + query_boundaries_[i],
score + query_boundaries_[i], query_boundaries_[i + 1] - query_boundaries_[i], &tmp_map);
for (size_t j = 0; j < eval_at_.size(); ++j) {
result_buffer_[tid][j] += tmp_map[j];
}
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(guided) firstprivate(tmp_map)
for (data_size_t i = 0; i < num_queries_; ++i) {
const int tid = omp_get_thread_num();
CalMapAtK(eval_at_, npos_per_query_[i], label_ + query_boundaries_[i],
score + query_boundaries_[i], query_boundaries_[i + 1] - query_boundaries_[i], &tmp_map);
for (size_t j = 0; j < eval_at_.size(); ++j) {
result_buffer_[tid][j] += tmp_map[j] * query_weights_[i];
}
}
}
// Get final average MAP
std::vector<double> result(eval_at_.size(), 0.0f);
for (size_t j = 0; j < result.size(); ++j) {
for (int i = 0; i < num_threads; ++i) {
result[j] += result_buffer_[i][j];
}
result[j] /= sum_query_weights_;
}
return result;
}
private:
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Query boundaries information */
const data_size_t* query_boundaries_;
/*! \brief Number of queries */
data_size_t num_queries_;
/*! \brief Weights of queries */
const label_t* query_weights_;
/*! \brief Sum weights of queries */
double sum_query_weights_;
/*! \brief Evaluate position of Nmap */
std::vector<data_size_t> eval_at_;
std::vector<std::string> name_;
std::vector<data_size_t> npos_per_query_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_METRIC_MAP_METRIC_HPP_
src/metric/metric.cpp
+142
View File
@@ -0,0 +1,142 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/metric.h>
#include <string>
#include "binary_metric.hpp"
#include "map_metric.hpp"
#include "multiclass_metric.hpp"
#include "rank_metric.hpp"
#include "regression_metric.hpp"
#include "xentropy_metric.hpp"
#include "cuda/cuda_binary_metric.hpp"
#include "cuda/cuda_regression_metric.hpp"
namespace LightGBM {
Metric* Metric::CreateMetric(const std::string& type, const Config& config) {
#ifdef USE_CUDA
if (config.device_type == std::string("cuda") && config.boosting == std::string("gbdt")) {
if (type == std::string("l2")) {
return new CUDAL2Metric(config);
} else if (type == std::string("rmse")) {
return new CUDARMSEMetric(config);
} else if (type == std::string("l1")) {
return new CUDAL1Metric(config);
} else if (type == std::string("quantile")) {
return new CUDAQuantileMetric(config);
} else if (type == std::string("huber")) {
return new CUDAHuberLossMetric(config);
} else if (type == std::string("fair")) {
return new CUDAFairLossMetric(config);
} else if (type == std::string("poisson")) {
return new CUDAPoissonMetric(config);
} else if (type == std::string("binary_logloss")) {
return new CUDABinaryLoglossMetric(config);
} else if (type == std::string("binary_error")) {
return new CUDABinaryErrorMetric(config);
} else if (type == std::string("auc")) {
Log::Warning("Metric auc is not implemented in cuda version. Fall back to evaluation on CPU.");
return new AUCMetric(config);
} else if (type == std::string("average_precision")) {
Log::Warning("Metric average_precision is not implemented in cuda version. Fall back to evaluation on CPU.");
return new AveragePrecisionMetric(config);
} else if (type == std::string("auc_mu")) {
Log::Warning("Metric auc_mu is not implemented in cuda version. Fall back to evaluation on CPU.");
return new AucMuMetric(config);
} else if (type == std::string("ndcg")) {
Log::Warning("Metric ndcg is not implemented in cuda version. Fall back to evaluation on CPU.");
return new NDCGMetric(config);
} else if (type == std::string("map")) {
Log::Warning("Metric map is not implemented in cuda version. Fall back to evaluation on CPU.");
return new MapMetric(config);
} else if (type == std::string("multi_logloss")) {
Log::Warning("Metric multi_logloss is not implemented in cuda version. Fall back to evaluation on CPU.");
return new MultiSoftmaxLoglossMetric(config);
} else if (type == std::string("multi_error")) {
Log::Warning("Metric multi_error is not implemented in cuda version. Fall back to evaluation on CPU.");
return new MultiErrorMetric(config);
} else if (type == std::string("cross_entropy")) {
Log::Warning("Metric cross_entropy is not implemented in cuda version. Fall back to evaluation on CPU.");
return new CrossEntropyMetric(config);
} else if (type == std::string("cross_entropy_lambda")) {
Log::Warning("Metric cross_entropy_lambda is not implemented in cuda version. Fall back to evaluation on CPU.");
return new CrossEntropyLambdaMetric(config);
} else if (type == std::string("kullback_leibler")) {
Log::Warning("Metric kullback_leibler is not implemented in cuda version. Fall back to evaluation on CPU.");
return new KullbackLeiblerDivergence(config);
} else if (type == std::string("mape")) {
return new CUDAMAPEMetric(config);
} else if (type == std::string("gamma")) {
return new CUDAGammaMetric(config);
} else if (type == std::string("gamma_deviance")) {
return new CUDAGammaDevianceMetric(config);
} else if (type == std::string("tweedie")) {
return new CUDATweedieMetric(config);
} else if (type == std::string("r2")) {
Log::Warning("Metric r2 is not implemented in cuda version. Fall back to evaluation on CPU.");
return new R2Metric(config);
}
} else {
#endif // USE_CUDA
if (type == std::string("l2")) {
return new L2Metric(config);
} else if (type == std::string("rmse")) {
return new RMSEMetric(config);
} else if (type == std::string("l1")) {
return new L1Metric(config);
} else if (type == std::string("quantile")) {
return new QuantileMetric(config);
} else if (type == std::string("huber")) {
return new HuberLossMetric(config);
} else if (type == std::string("fair")) {
return new FairLossMetric(config);
} else if (type == std::string("poisson")) {
return new PoissonMetric(config);
} else if (type == std::string("binary_logloss")) {
return new BinaryLoglossMetric(config);
} else if (type == std::string("binary_error")) {
return new BinaryErrorMetric(config);
} else if (type == std::string("auc")) {
return new AUCMetric(config);
} else if (type == std::string("average_precision")) {
return new AveragePrecisionMetric(config);
} else if (type == std::string("auc_mu")) {
return new AucMuMetric(config);
} else if (type == std::string("ndcg")) {
return new NDCGMetric(config);
} else if (type == std::string("map")) {
return new MapMetric(config);
} else if (type == std::string("multi_logloss")) {
return new MultiSoftmaxLoglossMetric(config);
} else if (type == std::string("multi_error")) {
return new MultiErrorMetric(config);
} else if (type == std::string("cross_entropy")) {
return new CrossEntropyMetric(config);
} else if (type == std::string("cross_entropy_lambda")) {
return new CrossEntropyLambdaMetric(config);
} else if (type == std::string("kullback_leibler")) {
return new KullbackLeiblerDivergence(config);
} else if (type == std::string("mape")) {
return new MAPEMetric(config);
} else if (type == std::string("gamma")) {
return new GammaMetric(config);
} else if (type == std::string("gamma_deviance")) {
return new GammaDevianceMetric(config);
} else if (type == std::string("tweedie")) {
return new TweedieMetric(config);
} else if (type == std::string("r2")) {
return new R2Metric(config);
}
#ifdef USE_CUDA
}
#endif // USE_CUDA
return nullptr;
}
} // namespace LightGBM
src/metric/multiclass_metric.hpp
+369
View File
@@ -0,0 +1,369 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_METRIC_MULTICLASS_METRIC_HPP_
#define LIGHTGBM_SRC_METRIC_MULTICLASS_METRIC_HPP_
#include <LightGBM/metric.h>
#include <LightGBM/utils/log.h>
#include <string>
#include <cmath>
#include <utility>
#include <vector>
namespace LightGBM {
/*!
* \brief Metric for multiclass task.
* Use static class "PointWiseLossCalculator" to calculate loss point-wise
*/
template<typename PointWiseLossCalculator>
class MulticlassMetric: public Metric {
public:
explicit MulticlassMetric(const Config& config) :config_(config) {
num_class_ = config.num_class;
}
virtual ~MulticlassMetric() {
}
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back(PointWiseLossCalculator::Name(config_));
num_data_ = num_data;
// get label
label_ = metadata.label();
// get weights
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data_; ++i) {
sum_weights_ += weights_[i];
}
}
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return -1.0f;
}
std::vector<double> Eval(const double* score, const ObjectiveFunction* objective) const override {
double sum_loss = 0.0;
int num_tree_per_iteration = num_class_;
int num_pred_per_row = num_class_;
if (objective != nullptr) {
num_tree_per_iteration = objective->NumModelPerIteration();
num_pred_per_row = objective->NumPredictOneRow();
}
if (objective != nullptr) {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
std::vector<double> raw_score(num_tree_per_iteration);
for (int k = 0; k < num_tree_per_iteration; ++k) {
size_t idx = static_cast<size_t>(num_data_) * k + i;
raw_score[k] = static_cast<double>(score[idx]);
}
std::vector<double> rec(num_pred_per_row);
objective->ConvertOutput(raw_score.data(), rec.data());
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], &rec, config_);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
std::vector<double> raw_score(num_tree_per_iteration);
for (int k = 0; k < num_tree_per_iteration; ++k) {
size_t idx = static_cast<size_t>(num_data_) * k + i;
raw_score[k] = static_cast<double>(score[idx]);
}
std::vector<double> rec(num_pred_per_row);
objective->ConvertOutput(raw_score.data(), rec.data());
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], &rec, config_) * weights_[i];
}
}
} else {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
std::vector<double> rec(num_tree_per_iteration);
for (int k = 0; k < num_tree_per_iteration; ++k) {
size_t idx = static_cast<size_t>(num_data_) * k + i;
rec[k] = static_cast<double>(score[idx]);
}
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], &rec, config_);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
std::vector<double> rec(num_tree_per_iteration);
for (int k = 0; k < num_tree_per_iteration; ++k) {
size_t idx = static_cast<size_t>(num_data_) * k + i;
rec[k] = static_cast<double>(score[idx]);
}
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], &rec, config_) * weights_[i];
}
}
}
double loss = sum_loss / sum_weights_;
return std::vector<double>(1, loss);
}
private:
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Pointer of weighs */
const label_t* weights_;
/*! \brief Sum weights */
double sum_weights_;
/*! \brief Name of this test set */
std::vector<std::string> name_;
int num_class_;
/*! \brief config parameters*/
Config config_;
};
/*! \brief top-k error for multiclass task; if k=1 (default) this is the usual multi-error */
class MultiErrorMetric: public MulticlassMetric<MultiErrorMetric> {
public:
explicit MultiErrorMetric(const Config& config) :MulticlassMetric<MultiErrorMetric>(config) {}
inline static double LossOnPoint(label_t label, std::vector<double>* score, const Config& config) {
size_t k = static_cast<size_t>(label);
auto& ref_score = *score;
int num_larger = 0;
for (size_t i = 0; i < score->size(); ++i) {
if (ref_score[i] >= ref_score[k]) ++num_larger;
if (num_larger > config.multi_error_top_k) return 1.0f;
}
return 0.0f;
}
inline static const std::string Name(const Config& config) {
if (config.multi_error_top_k == 1) {
return "multi_error";
} else {
return "multi_error@" + std::to_string(config.multi_error_top_k);
}
}
};
/*! \brief Logloss for multiclass task */
class MultiSoftmaxLoglossMetric: public MulticlassMetric<MultiSoftmaxLoglossMetric> {
public:
explicit MultiSoftmaxLoglossMetric(const Config& config) :MulticlassMetric<MultiSoftmaxLoglossMetric>(config) {}
inline static double LossOnPoint(label_t label, std::vector<double>* score, const Config&) {
size_t k = static_cast<size_t>(label);
auto& ref_score = *score;
if (ref_score[k] > kEpsilon) {
return static_cast<double>(-std::log(ref_score[k]));
} else {
return -std::log(kEpsilon);
}
}
inline static const std::string Name(const Config&) {
return "multi_logloss";
}
};
/*! \brief AUC mu for multiclass task*/
class AucMuMetric : public Metric {
public:
explicit AucMuMetric(const Config& config) : config_(config) {
num_class_ = config.num_class;
class_weights_ = config.auc_mu_weights_matrix;
}
virtual ~AucMuMetric() {}
const std::vector<std::string>& GetName() const override { return name_; }
double factor_to_bigger_better() const override { return 1.0f; }
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back("auc_mu");
num_data_ = num_data;
label_ = metadata.label();
// get weights
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data_; ++i) {
sum_weights_ += weights_[i];
}
}
// sort the data indices by true class
sorted_data_idx_ = std::vector<data_size_t>(num_data_, 0);
for (data_size_t i = 0; i < num_data_; ++i) {
sorted_data_idx_[i] = i;
}
Common::ParallelSort(sorted_data_idx_.begin(), sorted_data_idx_.end(),
[this](data_size_t a, data_size_t b) { return label_[a] < label_[b]; });
// get size of each class
class_sizes_ = std::vector<data_size_t>(num_class_, 0);
for (data_size_t i = 0; i < num_data_; ++i) {
data_size_t curr_label = static_cast<data_size_t>(label_[i]);
++class_sizes_[curr_label];
}
// get total weight of data in each class
class_data_weights_ = std::vector<double>(num_class_, 0);
if (weights_ != nullptr) {
for (data_size_t i = 0; i < num_data_; ++i) {
data_size_t curr_label = static_cast<data_size_t>(label_[i]);
class_data_weights_[curr_label] += weights_[i];
}
}
}
std::vector<double> Eval(const double* score, const ObjectiveFunction*) const override {
// the notation follows that used in the paper introducing the auc-mu metric:
// https://proceedings.mlr.press/v97/kleiman19a.html
auto S = std::vector<std::vector<double>>(num_class_, std::vector<double>(num_class_, 0));
int i_start = 0;
for (int i = 0; i < num_class_; ++i) {
int j_start = i_start + class_sizes_[i];
for (int j = i + 1; j < num_class_; ++j) {
std::vector<double> curr_v;
for (int k = 0; k < num_class_; ++k) {
curr_v.emplace_back(class_weights_[i][k] - class_weights_[j][k]);
}
double t1 = curr_v[i] - curr_v[j];
// extract the data indices belonging to class i or j
std::vector<data_size_t> class_i_j_indices;
class_i_j_indices.assign(sorted_data_idx_.begin() + i_start, sorted_data_idx_.begin() + i_start + class_sizes_[i]);
class_i_j_indices.insert(class_i_j_indices.end(),
sorted_data_idx_.begin() + j_start, sorted_data_idx_.begin() + j_start + class_sizes_[j]);
// sort according to distance from separating hyperplane
std::vector<std::pair<data_size_t, double>> dist;
for (data_size_t k = 0; static_cast<size_t>(k) < class_i_j_indices.size(); ++k) {
data_size_t a = class_i_j_indices[k];
double v_a = 0;
for (int m = 0; m < num_class_; ++m) {
v_a += curr_v[m] * score[num_data_ * m + a];
}
dist.push_back(std::pair<data_size_t, double>(a, t1 * v_a));
}
Common::ParallelSort(dist.begin(), dist.end(),
[this](std::pair<data_size_t, double> a, std::pair<data_size_t, double> b) {
// if scores are equal, put j class first
if (std::fabs(a.second - b.second) < kEpsilon) {
return label_[a.first] > label_[b.first];
} else if (a.second < b.second) {
return true;
} else {
return false;
}
});
// calculate AUC
double num_j = 0;
double last_j_dist = 0;
double num_current_j = 0;
if (weights_ == nullptr) {
for (size_t k = 0; k < dist.size(); ++k) {
data_size_t a = dist[k].first;
double curr_dist = dist[k].second;
if (label_[a] == i) {
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
S[i][j] += num_j - 0.5 * num_current_j; // members of class j with same distance as a contribute 0.5
} else {
S[i][j] += num_j;
}
} else {
++num_j;
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
++num_current_j;
} else {
last_j_dist = dist[k].second;
num_current_j = 1;
}
}
}
} else {
for (size_t k = 0; k < dist.size(); ++k) {
data_size_t a = dist[k].first;
double curr_dist = dist[k].second;
double curr_weight = weights_[a];
if (label_[a] == i) {
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
S[i][j] += curr_weight * (num_j - 0.5 * num_current_j); // members of class j with same distance as a contribute 0.5
} else {
S[i][j] += curr_weight * num_j;
}
} else {
num_j += curr_weight;
if (std::fabs(curr_dist - last_j_dist) < kEpsilon) {
num_current_j += curr_weight;
} else {
last_j_dist = dist[k].second;
num_current_j = curr_weight;
}
}
}
}
j_start += class_sizes_[j];
}
i_start += class_sizes_[i];
}
double ans = 0;
for (int i = 0; i < num_class_; ++i) {
for (int j = i + 1; j < num_class_; ++j) {
if (weights_ == nullptr) {
ans += (S[i][j] / class_sizes_[i]) / class_sizes_[j];
} else {
ans += (S[i][j] / class_data_weights_[i]) / class_data_weights_[j];
}
}
}
ans = (2.0 * ans / num_class_) / (num_class_ - 1);
return std::vector<double>(1, ans);
}
private:
/*! \brief Number of data*/
data_size_t num_data_;
/*! \brief Pointer to label*/
const label_t* label_;
/*! \brief Name of this metric*/
std::vector<std::string> name_;
/*! \brief Number of classes*/
int num_class_;
/*! \brief Class auc-mu weights*/
std::vector<std::vector<double>> class_weights_;
/*! \brief Data weights */
const label_t* weights_;
/*! \brief Sum of data weights */
double sum_weights_;
/*! \brief Sum of data weights in each class*/
std::vector<double> class_data_weights_;
/*! \brief Number of data in each class*/
std::vector<data_size_t> class_sizes_;
/*! \brief config parameters*/
Config config_;
/*! \brief index to data, sorted by true class*/
std::vector<data_size_t> sorted_data_idx_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_METRIC_MULTICLASS_METRIC_HPP_
src/metric/rank_metric.hpp
+170
View File
@@ -0,0 +1,170 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_METRIC_RANK_METRIC_HPP_
#define LIGHTGBM_SRC_METRIC_RANK_METRIC_HPP_
#include <LightGBM/metric.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/log.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <string>
#include <sstream>
#include <vector>
namespace LightGBM {
class NDCGMetric:public Metric {
public:
explicit NDCGMetric(const Config& config) {
// get eval position
eval_at_ = config.eval_at;
auto label_gain = config.label_gain;
DCGCalculator::DefaultEvalAt(&eval_at_);
DCGCalculator::DefaultLabelGain(&label_gain);
// initialize DCG calculator
DCGCalculator::Init(label_gain);
}
~NDCGMetric() {
}
void Init(const Metadata& metadata, data_size_t num_data) override {
for (auto k : eval_at_) {
name_.emplace_back(std::string("ndcg@") + std::to_string(k));
}
num_data_ = num_data;
// get label
label_ = metadata.label();
num_queries_ = metadata.num_queries();
DCGCalculator::CheckMetadata(metadata, num_queries_);
DCGCalculator::CheckLabel(label_, num_data_);
// get query boundaries
query_boundaries_ = metadata.query_boundaries();
if (query_boundaries_ == nullptr) {
Log::Fatal("The NDCG metric requires query information");
}
// get query weights
query_weights_ = metadata.query_weights();
if (query_weights_ == nullptr) {
sum_query_weights_ = static_cast<double>(num_queries_);
} else {
sum_query_weights_ = 0.0f;
for (data_size_t i = 0; i < num_queries_; ++i) {
sum_query_weights_ += query_weights_[i];
}
}
inverse_max_dcgs_.resize(num_queries_);
// cache the inverse max DCG for all queries, used to calculate NDCG
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_queries_; ++i) {
inverse_max_dcgs_[i].resize(eval_at_.size(), 0.0f);
DCGCalculator::CalMaxDCG(eval_at_, label_ + query_boundaries_[i],
query_boundaries_[i + 1] - query_boundaries_[i],
&inverse_max_dcgs_[i]);
for (size_t j = 0; j < inverse_max_dcgs_[i].size(); ++j) {
if (inverse_max_dcgs_[i][j] > 0.0f) {
inverse_max_dcgs_[i][j] = 1.0f / inverse_max_dcgs_[i][j];
} else {
// marking negative for all negative queries.
// if one meet this query, it's ndcg will be set as -1.
inverse_max_dcgs_[i][j] = -1.0f;
}
}
}
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return 1.0f;
}
std::vector<double> Eval(const double* score, const ObjectiveFunction*) const override {
int num_threads = OMP_NUM_THREADS();
// some buffers for multi-threading sum up
std::vector<std::vector<double>> result_buffer_;
for (int i = 0; i < num_threads; ++i) {
result_buffer_.emplace_back(eval_at_.size(), 0.0f);
}
std::vector<double> tmp_dcg(eval_at_.size(), 0.0f);
if (query_weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) firstprivate(tmp_dcg)
for (data_size_t i = 0; i < num_queries_; ++i) {
const int tid = omp_get_thread_num();
// if all doc in this query are all negative, let its NDCG=1
if (inverse_max_dcgs_[i][0] <= 0.0f) {
for (size_t j = 0; j < eval_at_.size(); ++j) {
result_buffer_[tid][j] += 1.0f;
}
} else {
// calculate DCG
DCGCalculator::CalDCG(eval_at_, label_ + query_boundaries_[i],
score + query_boundaries_[i],
query_boundaries_[i + 1] - query_boundaries_[i], &tmp_dcg);
// calculate NDCG
for (size_t j = 0; j < eval_at_.size(); ++j) {
result_buffer_[tid][j] += tmp_dcg[j] * inverse_max_dcgs_[i][j];
}
}
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) firstprivate(tmp_dcg)
for (data_size_t i = 0; i < num_queries_; ++i) {
const int tid = omp_get_thread_num();
// if all doc in this query are all negative, let its NDCG=1
if (inverse_max_dcgs_[i][0] <= 0.0f) {
for (size_t j = 0; j < eval_at_.size(); ++j) {
result_buffer_[tid][j] += 1.0f;
}
} else {
// calculate DCG
DCGCalculator::CalDCG(eval_at_, label_ + query_boundaries_[i],
score + query_boundaries_[i],
query_boundaries_[i + 1] - query_boundaries_[i], &tmp_dcg);
// calculate NDCG
for (size_t j = 0; j < eval_at_.size(); ++j) {
result_buffer_[tid][j] += tmp_dcg[j] * inverse_max_dcgs_[i][j] * query_weights_[i];
}
}
}
}
// Get final average NDCG
std::vector<double> result(eval_at_.size(), 0.0f);
for (size_t j = 0; j < result.size(); ++j) {
for (int i = 0; i < num_threads; ++i) {
result[j] += result_buffer_[i][j];
}
result[j] /= sum_query_weights_;
}
return result;
}
private:
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Name of test set */
std::vector<std::string> name_;
/*! \brief Query boundaries information */
const data_size_t* query_boundaries_;
/*! \brief Number of queries */
data_size_t num_queries_;
/*! \brief Weights of queries */
const label_t* query_weights_;
/*! \brief Sum weights of queries */
double sum_query_weights_;
/*! \brief Evaluate position of NDCG */
std::vector<data_size_t> eval_at_;
/*! \brief Cache the inverse max dcg for all queries */
std::vector<std::vector<double>> inverse_max_dcgs_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_METRIC_RANK_METRIC_HPP_
src/metric/regression_metric.hpp
+433
View File
@@ -0,0 +1,433 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_METRIC_REGRESSION_METRIC_HPP_
#define LIGHTGBM_SRC_METRIC_REGRESSION_METRIC_HPP_
#include <LightGBM/metric.h>
#include <LightGBM/utils/log.h>
#include <string>
#include <algorithm>
#include <cmath>
#include <vector>
namespace LightGBM {
/*!
* \brief Metric for regression task.
* Use static class "PointWiseLossCalculator" to calculate loss point-wise
*/
template<typename PointWiseLossCalculator>
class RegressionMetric: public Metric {
public:
explicit RegressionMetric(const Config& config) :config_(config) {
}
virtual ~RegressionMetric() {
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return -1.0f;
}
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back(PointWiseLossCalculator::Name());
num_data_ = num_data;
// get label
label_ = metadata.label();
// get weights
weights_ = metadata.weights();
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
} else {
sum_weights_ = 0.0f;
for (data_size_t i = 0; i < num_data_; ++i) {
sum_weights_ += weights_[i];
}
}
for (data_size_t i = 0; i < num_data_; ++i) {
PointWiseLossCalculator::CheckLabel(label_[i]);
}
}
std::vector<double> Eval(const double* score, const ObjectiveFunction* objective) const override {
double sum_loss = 0.0f;
if (objective == nullptr) {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], score[i], config_);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
// add loss
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], score[i], config_) * weights_[i];
}
}
} else {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
// add loss
double t = 0;
objective->ConvertOutput(&score[i], &t);
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], t, config_);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
// add loss
double t = 0;
objective->ConvertOutput(&score[i], &t);
sum_loss += PointWiseLossCalculator::LossOnPoint(label_[i], t, config_) * weights_[i];
}
}
}
double loss = PointWiseLossCalculator::AverageLoss(sum_loss, sum_weights_);
return std::vector<double>(1, loss);
}
inline static double AverageLoss(double sum_loss, double sum_weights) {
return sum_loss / sum_weights;
}
inline static void CheckLabel(label_t) {
}
protected:
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Pointer of weighs */
const label_t* weights_;
/*! \brief Sum weights */
double sum_weights_;
/*! \brief Name of this test set */
Config config_;
std::vector<std::string> name_;
};
/*! \brief RMSE loss for regression task */
class RMSEMetric: public RegressionMetric<RMSEMetric> {
public:
explicit RMSEMetric(const Config& config) :RegressionMetric<RMSEMetric>(config) {}
inline static double LossOnPoint(label_t label, double score, const Config&) {
return (score - label)*(score - label);
}
inline static double AverageLoss(double sum_loss, double sum_weights) {
// need sqrt the result for RMSE loss
return std::sqrt(sum_loss / sum_weights);
}
inline static const char* Name() {
return "rmse";
}
};
/*! \brief L2 loss for regression task */
class L2Metric: public RegressionMetric<L2Metric> {
public:
explicit L2Metric(const Config& config) :RegressionMetric<L2Metric>(config) {}
inline static double LossOnPoint(label_t label, double score, const Config&) {
return (score - label)*(score - label);
}
inline static const char* Name() {
return "l2";
}
};
/*! \brief Quantile loss for regression task */
class QuantileMetric : public RegressionMetric<QuantileMetric> {
public:
explicit QuantileMetric(const Config& config) :RegressionMetric<QuantileMetric>(config) {
}
inline static double LossOnPoint(label_t label, double score, const Config& config) {
double delta = label - score;
if (delta < 0) {
return (config.alpha - 1.0f) * delta;
} else {
return config.alpha * delta;
}
}
inline static const char* Name() {
return "quantile";
}
};
/*! \brief L1 loss for regression task */
class L1Metric: public RegressionMetric<L1Metric> {
public:
explicit L1Metric(const Config& config) :RegressionMetric<L1Metric>(config) {}
inline static double LossOnPoint(label_t label, double score, const Config&) {
return std::fabs(score - label);
}
inline static const char* Name() {
return "l1";
}
};
/*! \brief Huber loss for regression task */
class HuberLossMetric: public RegressionMetric<HuberLossMetric> {
public:
explicit HuberLossMetric(const Config& config) :RegressionMetric<HuberLossMetric>(config) {
}
inline static double LossOnPoint(label_t label, double score, const Config& config) {
const double diff = score - label;
if (std::abs(diff) <= config.alpha) {
return 0.5f * diff * diff;
} else {
return config.alpha * (std::abs(diff) - 0.5f * config.alpha);
}
}
inline static const char* Name() {
return "huber";
}
};
/*! \brief Fair loss for regression task */
// http://research.microsoft.com/en-us/um/people/zhang/INRIA/Publis/Tutorial-Estim/node24.html
class FairLossMetric: public RegressionMetric<FairLossMetric> {
public:
explicit FairLossMetric(const Config& config) :RegressionMetric<FairLossMetric>(config) {
}
inline static double LossOnPoint(label_t label, double score, const Config& config) {
const double x = std::fabs(score - label);
const double c = config.fair_c;
return c * x - c * c * std::log1p(x / c);
}
inline static const char* Name() {
return "fair";
}
};
/*! \brief Poisson regression loss for regression task */
class PoissonMetric: public RegressionMetric<PoissonMetric> {
public:
explicit PoissonMetric(const Config& config) :RegressionMetric<PoissonMetric>(config) {
}
inline static double LossOnPoint(label_t label, double score, const Config&) {
const double eps = 1e-10f;
if (score < eps) {
score = eps;
}
return score - label * std::log(score);
}
inline static const char* Name() {
return "poisson";
}
};
/*! \brief MAPE regression loss for regression task */
class MAPEMetric : public RegressionMetric<MAPEMetric> {
public:
explicit MAPEMetric(const Config& config) :RegressionMetric<MAPEMetric>(config) {
}
inline static double LossOnPoint(label_t label, double score, const Config&) {
return std::fabs((label - score)) / std::max(1.0f, std::fabs(label));
}
inline static const char* Name() {
return "mape";
}
};
class GammaMetric : public RegressionMetric<GammaMetric> {
public:
explicit GammaMetric(const Config& config) :RegressionMetric<GammaMetric>(config) {
}
inline static double LossOnPoint(label_t label, double score, const Config&) {
const double psi = 1.0;
const double theta = -1.0 / score;
const double a = psi;
const double b = -Common::SafeLog(-theta);
const double c = 1. / psi * Common::SafeLog(label / psi) - Common::SafeLog(label) - 0; // 0 = std::lgamma(1.0 / psi) = std::lgamma(1.0);
return -((label * theta - b) / a + c);
}
inline static const char* Name() {
return "gamma";
}
inline static void CheckLabel(label_t label) {
CHECK_GT(label, 0);
}
};
class GammaDevianceMetric : public RegressionMetric<GammaDevianceMetric> {
public:
explicit GammaDevianceMetric(const Config& config) :RegressionMetric<GammaDevianceMetric>(config) {
}
inline static double LossOnPoint(label_t label, double score, const Config&) {
const double epsilon = 1.0e-9;
const double tmp = label / (score + epsilon);
return tmp - Common::SafeLog(tmp) - 1;
}
inline static const char* Name() {
return "gamma_deviance";
}
inline static double AverageLoss(double sum_loss, double) {
return sum_loss * 2;
}
inline static void CheckLabel(label_t label) {
CHECK_GT(label, 0);
}
};
class TweedieMetric : public RegressionMetric<TweedieMetric> {
public:
explicit TweedieMetric(const Config& config) :RegressionMetric<TweedieMetric>(config) {
}
inline static double LossOnPoint(label_t label, double score, const Config& config) {
const double rho = config.tweedie_variance_power;
const double eps = 1e-10f;
if (score < eps) {
score = eps;
}
const double a = label * std::exp((1 - rho) * std::log(score)) / (1 - rho);
const double b = std::exp((2 - rho) * std::log(score)) / (2 - rho);
return -a + b;
}
inline static const char* Name() {
return "tweedie";
}
};
class R2Metric: public Metric {
public:
explicit R2Metric(const Config& config) :config_(config) {}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return 1.0f;
}
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back("r2");
num_data_ = num_data;
label_ = metadata.label();
weights_ = metadata.weights();
double sum_label = 0.0f;
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_label)
for (data_size_t i = 0; i < num_data_; ++i) {
sum_label += label_[i];
}
} else {
double local_sum_weights = 0.0f;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:local_sum_weights, sum_label)
for (data_size_t i = 0; i < num_data_; ++i) {
local_sum_weights += weights_[i];
sum_label += label_[i] * weights_[i];
}
sum_weights_ = local_sum_weights;
}
label_mean_ = sum_label / sum_weights_;
total_sum_squares_ = 0.0f;
double local_total_sum_squares = 0.0f;
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:local_total_sum_squares)
for (data_size_t i = 0; i < num_data_; ++i) {
double diff = label_[i] - label_mean_;
local_total_sum_squares += diff * diff;
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:local_total_sum_squares)
for (data_size_t i = 0; i < num_data_; ++i) {
double diff = label_[i] - label_mean_;
local_total_sum_squares += diff * diff * weights_[i];
}
}
total_sum_squares_ = local_total_sum_squares;
}
std::vector<double> Eval(const double* score, const ObjectiveFunction* objective) const override {
double residual_sum_squares = 0.0f;
if (objective == nullptr) {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:residual_sum_squares)
for (data_size_t i = 0; i < num_data_; ++i) {
double diff = label_[i] - score[i];
residual_sum_squares += diff * diff;
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:residual_sum_squares)
for (data_size_t i = 0; i < num_data_; ++i) {
double diff = label_[i] - score[i];
residual_sum_squares += diff * diff * weights_[i];
}
}
} else {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:residual_sum_squares)
for (data_size_t i = 0; i < num_data_; ++i) {
double t = 0;
objective->ConvertOutput(&score[i], &t);
double diff = label_[i] - t;
residual_sum_squares += diff * diff;
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:residual_sum_squares)
for (data_size_t i = 0; i < num_data_; ++i) {
double t = 0;
objective->ConvertOutput(&score[i], &t);
double diff = label_[i] - t;
residual_sum_squares += diff * diff * weights_[i];
}
}
}
double r2 = 1.0 - (residual_sum_squares / total_sum_squares_);
if (std::fabs(total_sum_squares_) < kZeroThreshold) {
return std::vector<double>(1, std::fabs(residual_sum_squares) < kZeroThreshold ? 1.0 : 0.0);
}
return std::vector<double>(1, r2);
}
protected:
data_size_t num_data_;
const label_t* label_;
const label_t* weights_;
double sum_weights_;
Config config_;
std::vector<std::string> name_;
// Custom members for R2 calculation
double label_mean_;
double total_sum_squares_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_METRIC_REGRESSION_METRIC_HPP_
src/metric/xentropy_metric.hpp
+359
View File
@@ -0,0 +1,359 @@
/*!
* Copyright (c) 2017-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2017-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_METRIC_XENTROPY_METRIC_HPP_
#define LIGHTGBM_SRC_METRIC_XENTROPY_METRIC_HPP_
#include <LightGBM/meta.h>
#include <LightGBM/metric.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/log.h>
#include <string>
#include <algorithm>
#include <sstream>
#include <vector>
/*
* Implements three related metrics:
*
* (1) standard cross-entropy that can be used for continuous labels in [0, 1]
* (2) "intensity-weighted" cross-entropy, also for continuous labels in [0, 1]
* (3) Kullback-Leibler divergence, also for continuous labels in [0, 1]
*
* (3) adds an offset term to (1); the entropy of the label
*
* See xentropy_objective.hpp for further details.
*
*/
namespace LightGBM {
// label should be in interval [0, 1];
// prob should be in interval (0, 1); prob is clipped if needed
inline static double XentLoss(label_t label, double prob) {
const double log_arg_epsilon = 1.0e-12;
double a = label;
if (prob > log_arg_epsilon) {
a *= std::log(prob);
} else {
a *= std::log(log_arg_epsilon);
}
double b = 1.0f - label;
if (1.0f - prob > log_arg_epsilon) {
b *= std::log(1.0f - prob);
} else {
b *= std::log(log_arg_epsilon);
}
return - (a + b);
}
// hhat >(=) 0 assumed; and weight > 0 required; but not checked here
inline static double XentLambdaLoss(label_t label, label_t weight, double hhat) {
return XentLoss(label, 1.0f - std::exp(-weight * hhat));
}
// Computes the (negative) entropy for label p; p should be in interval [0, 1];
// This is used to presum the KL-divergence offset term (to be _added_ to the cross-entropy loss).
// NOTE: x*log(x) = 0 for x=0,1; so only add when in (0, 1); avoid log(0)*0
inline static double YentLoss(double p) {
double hp = 0.0;
if (p > 0) hp += p * std::log(p);
double q = 1.0f - p;
if (q > 0) hp += q * std::log(q);
return hp;
}
//
// CrossEntropyMetric : "xentropy" : (optional) weights are used linearly
//
class CrossEntropyMetric : public Metric {
public:
explicit CrossEntropyMetric(const Config&) {}
virtual ~CrossEntropyMetric() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back("cross_entropy");
num_data_ = num_data;
label_ = metadata.label();
weights_ = metadata.weights();
CHECK_NOTNULL(label_);
// ensure that labels are in interval [0, 1], interval ends included
Common::CheckElementsIntervalClosed<label_t>(label_, 0.0f, 1.0f, num_data_, GetName()[0].c_str());
Log::Info("[%s:%s]: (metric) labels passed interval [0, 1] check", GetName()[0].c_str(), __func__);
// check that weights are non-negative and sum is positive
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
} else {
label_t minw;
Common::ObtainMinMaxSum(weights_, num_data_, &minw, static_cast<label_t*>(nullptr), &sum_weights_);
if (minw < 0.0f) {
Log::Fatal("[%s:%s]: (metric) weights not allowed to be negative", GetName()[0].c_str(), __func__);
}
}
// check weight sum (may fail to be zero)
if (sum_weights_ <= 0.0f) {
Log::Fatal("[%s:%s]: sum-of-weights = %f is non-positive", __func__, GetName()[0].c_str(), sum_weights_);
}
Log::Info("[%s:%s]: sum-of-weights = %f", GetName()[0].c_str(), __func__, sum_weights_);
}
std::vector<double> Eval(const double* score, const ObjectiveFunction* objective) const override {
double sum_loss = 0.0f;
if (objective == nullptr) {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
sum_loss += XentLoss(label_[i], score[i]); // NOTE: does not work unless score is a probability
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
sum_loss += XentLoss(label_[i], score[i]) * weights_[i]; // NOTE: does not work unless score is a probability
}
}
} else {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double p = 0;
objective->ConvertOutput(&score[i], &p);
sum_loss += XentLoss(label_[i], p);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double p = 0;
objective->ConvertOutput(&score[i], &p);
sum_loss += XentLoss(label_[i], p) * weights_[i];
}
}
}
double loss = sum_loss / sum_weights_;
return std::vector<double>(1, loss);
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return -1.0f; // negative means smaller loss is better, positive means larger loss is better
}
private:
/*! \brief Number of data points */
data_size_t num_data_;
/*! \brief Pointer to label */
const label_t* label_;
/*! \brief Pointer to weights */
const label_t* weights_;
/*! \brief Sum of weights */
double sum_weights_;
/*! \brief Name of this metric */
std::vector<std::string> name_;
};
//
// CrossEntropyLambdaMetric : "xentlambda" : (optional) weights have a different meaning than for "xentropy"
// ATTENTION: Supposed to be used when the objective also is "xentlambda"
//
class CrossEntropyLambdaMetric : public Metric {
public:
explicit CrossEntropyLambdaMetric(const Config&) {}
virtual ~CrossEntropyLambdaMetric() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back("cross_entropy_lambda");
num_data_ = num_data;
label_ = metadata.label();
weights_ = metadata.weights();
CHECK_NOTNULL(label_);
Common::CheckElementsIntervalClosed<label_t>(label_, 0.0f, 1.0f, num_data_, GetName()[0].c_str());
Log::Info("[%s:%s]: (metric) labels passed interval [0, 1] check", GetName()[0].c_str(), __func__);
// check all weights are strictly positive; throw error if not
if (weights_ != nullptr) {
label_t minw;
Common::ObtainMinMaxSum(weights_, num_data_, &minw, static_cast<label_t*>(nullptr), static_cast<label_t*>(nullptr));
if (minw <= 0.0f) {
Log::Fatal("[%s:%s]: (metric) all weights must be positive", GetName()[0].c_str(), __func__);
}
}
}
std::vector<double> Eval(const double* score, const ObjectiveFunction* objective) const override {
double sum_loss = 0.0f;
if (objective == nullptr) {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double hhat = std::log1p(std::exp(score[i])); // auto-convert
sum_loss += XentLambdaLoss(label_[i], 1.0f, hhat);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double hhat = std::log1p(std::exp(score[i])); // auto-convert
sum_loss += XentLambdaLoss(label_[i], weights_[i], hhat);
}
}
} else {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double hhat = 0;
objective->ConvertOutput(&score[i], &hhat); // NOTE: this only works if objective = "xentlambda"
sum_loss += XentLambdaLoss(label_[i], 1.0f, hhat);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double hhat = 0;
objective->ConvertOutput(&score[i], &hhat); // NOTE: this only works if objective = "xentlambda"
sum_loss += XentLambdaLoss(label_[i], weights_[i], hhat);
}
}
}
return std::vector<double>(1, sum_loss / static_cast<double>(num_data_));
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return -1.0f;
}
private:
/*! \brief Number of data points */
data_size_t num_data_;
/*! \brief Pointer to label */
const label_t* label_;
/*! \brief Pointer to weights */
const label_t* weights_;
/*! \brief Name of this metric */
std::vector<std::string> name_;
};
//
// KullbackLeiblerDivergence : "kldiv" : (optional) weights are used linearly
//
class KullbackLeiblerDivergence : public Metric {
public:
explicit KullbackLeiblerDivergence(const Config&) {}
virtual ~KullbackLeiblerDivergence() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
name_.emplace_back("kullback_leibler");
num_data_ = num_data;
label_ = metadata.label();
weights_ = metadata.weights();
CHECK_NOTNULL(label_);
Common::CheckElementsIntervalClosed<label_t>(label_, 0.0f, 1.0f, num_data_, GetName()[0].c_str());
Log::Info("[%s:%s]: (metric) labels passed interval [0, 1] check", GetName()[0].c_str(), __func__);
if (weights_ == nullptr) {
sum_weights_ = static_cast<double>(num_data_);
} else {
label_t minw;
Common::ObtainMinMaxSum(weights_, num_data_, &minw, static_cast<label_t*>(nullptr), &sum_weights_);
if (minw < 0.0f) {
Log::Fatal("[%s:%s]: (metric) at least one weight is negative", GetName()[0].c_str(), __func__);
}
}
// check weight sum
if (sum_weights_ <= 0.0f) {
Log::Fatal("[%s:%s]: sum-of-weights = %f is non-positive", GetName()[0].c_str(), __func__, sum_weights_);
}
Log::Info("[%s:%s]: sum-of-weights = %f", GetName()[0].c_str(), __func__, sum_weights_);
// evaluate offset term
presum_label_entropy_ = 0.0f;
if (weights_ == nullptr) {
for (data_size_t i = 0; i < num_data; ++i) {
presum_label_entropy_ += YentLoss(label_[i]);
}
} else {
for (data_size_t i = 0; i < num_data; ++i) {
presum_label_entropy_ += YentLoss(label_[i]) * weights_[i];
}
}
presum_label_entropy_ /= sum_weights_;
// communicate the value of the offset term to be added
Log::Info("%s offset term = %f", GetName()[0].c_str(), presum_label_entropy_);
}
std::vector<double> Eval(const double* score, const ObjectiveFunction* objective) const override {
double sum_loss = 0.0f;
if (objective == nullptr) {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
sum_loss += XentLoss(label_[i], score[i]); // NOTE: does not work unless score is a probability
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
sum_loss += XentLoss(label_[i], score[i]) * weights_[i]; // NOTE: does not work unless score is a probability
}
}
} else {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double p = 0;
objective->ConvertOutput(&score[i], &p);
sum_loss += XentLoss(label_[i], p);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:sum_loss)
for (data_size_t i = 0; i < num_data_; ++i) {
double p = 0;
objective->ConvertOutput(&score[i], &p);
sum_loss += XentLoss(label_[i], p) * weights_[i];
}
}
}
double loss = presum_label_entropy_ + sum_loss / sum_weights_;
return std::vector<double>(1, loss);
}
const std::vector<std::string>& GetName() const override {
return name_;
}
double factor_to_bigger_better() const override {
return -1.0f;
}
private:
/*! \brief Number of data points */
data_size_t num_data_;
/*! \brief Pointer to label */
const label_t* label_;
/*! \brief Pointer to weights */
const label_t* weights_;
/*! \brief Sum of weights */
double sum_weights_;
/*! \brief Offset term to cross-entropy; precomputed during init */
double presum_label_entropy_;
/*! \brief Name of this metric */
std::vector<std::string> name_;
};
} // end namespace LightGBM
#endif // LIGHTGBM_SRC_METRIC_XENTROPY_METRIC_HPP_
src/network/linker_topo.cpp
+182
View File
@@ -0,0 +1,182 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/network.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/log.h>
#include <string>
#include <unordered_map>
#include <vector>
namespace LightGBM {
BruckMap::BruckMap() {
k = 0;
}
BruckMap::BruckMap(int n) {
k = n;
// default set to -1
for (int i = 0; i < n; ++i) {
in_ranks.push_back(-1);
out_ranks.push_back(-1);
}
}
BruckMap BruckMap::Construct(int rank, int num_machines) {
// distance at k-th communication, distance[k] = 2^k
std::vector<int> distance;
int k = 0;
for (k = 0; (1 << k) < num_machines; ++k) {
distance.push_back(1 << k);
}
BruckMap bruckMap(k);
for (int j = 0; j < k; ++j) {
// set incoming rank at k-th communication
const int in_rank = (rank + distance[j]) % num_machines;
bruckMap.in_ranks[j] = in_rank;
// set outgoing rank at k-th communication
const int out_rank = (rank - distance[j] + num_machines) % num_machines;
bruckMap.out_ranks[j] = out_rank;
}
return bruckMap;
}
RecursiveHalvingMap::RecursiveHalvingMap() {
k = 0;
}
RecursiveHalvingMap::RecursiveHalvingMap(int in_k, RecursiveHalvingNodeType _type, bool _is_power_of_2) {
type = _type;
k = in_k;
is_power_of_2 = _is_power_of_2;
if (type != RecursiveHalvingNodeType::Other) {
for (int i = 0; i < k; ++i) {
// default set as -1
ranks.push_back(-1);
send_block_start.push_back(-1);
send_block_len.push_back(-1);
recv_block_start.push_back(-1);
recv_block_len.push_back(-1);
}
}
}
RecursiveHalvingMap RecursiveHalvingMap::Construct(int rank, int num_machines) {
// construct all recursive halving map for all machines
int k = 0;
while ((1 << k) <= num_machines) {
++k;
}
// let 1 << k <= num_machines
--k;
// distance of each communication
std::vector<int> distance;
for (int i = 0; i < k; ++i) {
distance.push_back(1 << (k - 1 - i));
}
if ((1 << k) == num_machines) {
RecursiveHalvingMap rec_map(k, RecursiveHalvingNodeType::Normal, true);
// if num_machines = 2^k, don't need to group machines
for (int i = 0; i < k; ++i) {
// communication direction, %2 == 0 is positive
const int dir = ((rank / distance[i]) % 2 == 0) ? 1 : -1;
// neighbor at k-th communication
const int next_node_idx = rank + dir * distance[i];
rec_map.ranks[i] = next_node_idx;
// receive data block at k-th communication
const int recv_block_start = rank / distance[i];
rec_map.recv_block_start[i] = recv_block_start * distance[i];
rec_map.recv_block_len[i] = distance[i];
// send data block at k-th communication
const int send_block_start = next_node_idx / distance[i];
rec_map.send_block_start[i] = send_block_start * distance[i];
rec_map.send_block_len[i] = distance[i];
}
return rec_map;
} else {
// if num_machines != 2^k, need to group machines
int lower_power_of_2 = 1 << k;
int rest = num_machines - lower_power_of_2;
std::vector<RecursiveHalvingNodeType> node_type(num_machines);
for (int i = 0; i < num_machines; ++i) {
node_type[i] = RecursiveHalvingNodeType::Normal;
}
// group, two machine in one group, total "rest" groups will have 2 machines.
for (int i = 0; i < rest; ++i) {
int right = num_machines - i * 2 - 1;
int left = num_machines - i * 2 - 2;
// let left machine as group leader
node_type[left] = RecursiveHalvingNodeType::GroupLeader;
node_type[right] = RecursiveHalvingNodeType::Other;
}
int group_cnt = 0;
// cache block information for groups, group with 2 machines will have double block size
std::vector<int> group_block_start(lower_power_of_2);
std::vector<int> group_block_len(lower_power_of_2, 0);
// convert from group to node leader
std::vector<int> group_to_node(lower_power_of_2);
// convert from node to group
std::vector<int> node_to_group(num_machines);
for (int i = 0; i < num_machines; ++i) {
// meet new group
if (node_type[i] == RecursiveHalvingNodeType::Normal || node_type[i] == RecursiveHalvingNodeType::GroupLeader) {
group_to_node[group_cnt++] = i;
}
node_to_group[i] = group_cnt - 1;
// add block len for this group
group_block_len[group_cnt - 1]++;
}
// calculate the group block start
group_block_start[0] = 0;
for (int i = 1; i < lower_power_of_2; ++i) {
group_block_start[i] = group_block_start[i - 1] + group_block_len[i - 1];
}
RecursiveHalvingMap rec_map(k, node_type[rank], false);
if (node_type[rank] == RecursiveHalvingNodeType::Other) {
rec_map.neighbor = rank - 1;
// not need to construct
return rec_map;
}
if (node_type[rank] == RecursiveHalvingNodeType::GroupLeader) {
rec_map.neighbor = rank + 1;
}
const int cur_group_idx = node_to_group[rank];
for (int i = 0; i < k; ++i) {
const int dir = ((cur_group_idx / distance[i]) % 2 == 0) ? 1 : -1;
const int next_node_idx = group_to_node[(cur_group_idx + dir * distance[i])];
rec_map.ranks[i] = next_node_idx;
// get receive block information
const int recv_block_start = cur_group_idx / distance[i];
rec_map.recv_block_start[i] = group_block_start[static_cast<size_t>(recv_block_start) * distance[i]];
int recv_block_len = 0;
// accumulate block len
for (int j = 0; j < distance[i]; ++j) {
recv_block_len += group_block_len[recv_block_start * distance[i] + j];
}
rec_map.recv_block_len[i] = recv_block_len;
// get send block information
const int send_block_start = (cur_group_idx + dir * distance[i]) / distance[i];
rec_map.send_block_start[i] = group_block_start[static_cast<size_t>(send_block_start) * distance[i]];
int send_block_len = 0;
// accumulate block len
for (int j = 0; j < distance[i]; ++j) {
send_block_len += group_block_len[send_block_start * distance[i] + j];
}
rec_map.send_block_len[i] = send_block_len;
}
return rec_map;
}
}
} // namespace LightGBM
src/network/linkers.h
+329
View File
@@ -0,0 +1,329 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_NETWORK_LINKERS_H_
#define LIGHTGBM_SRC_NETWORK_LINKERS_H_
#include <LightGBM/config.h>
#include <LightGBM/meta.h>
#include <LightGBM/network.h>
#include <LightGBM/utils/common.h>
#include <string>
#include <algorithm>
#include <chrono>
#include <ctime>
#include <memory>
#include <thread>
#include <vector>
#ifdef USE_SOCKET
#include "socket_wrapper.hpp"
#endif
#ifdef USE_MPI
#include <mpi.h>
#define MPI_SAFE_CALL(mpi_return) CHECK((mpi_return) == MPI_SUCCESS)
#endif
namespace LightGBM {
/*!
* \brief A network basic communication wrapper.
* Will wrap low level communication methods, e.g. mpi, socket and so on.
* This class will wrap all linkers to other machines if needs
*/
class Linkers {
public:
Linkers() {
is_init_ = false;
}
/*!
* \brief Constructor
* \param config Config of network settings
*/
explicit Linkers(Config config);
/*!
* \brief Destructor
*/
~Linkers();
/*!
* \brief Recv data, blocking
* \param rank Which rank will send data to local machine
* \param data Pointer of receive data
* \param len Recv size, will block until receive len size of data
*/
inline void Recv(int rank, char* data, int len) const;
inline void Recv(int rank, char* data, int64_t len) const;
/*!
* \brief Send data, blocking
* \param rank Which rank local machine will send to
* \param data Pointer of send data
* \param len Send size
*/
inline void Send(int rank, char* data, int len) const;
inline void Send(int rank, char* data, int64_t len) const;
/*!
* \brief Send and Recv at same time, blocking
* \param send_rank
* \param send_data
* \param send_len
* \param recv_rank
* \param recv_data
* \param recv_len
*/
inline void SendRecv(int send_rank, char* send_data, int send_len,
int recv_rank, char* recv_data, int recv_len);
inline void SendRecv(int send_rank, char* send_data, int64_t send_len,
int recv_rank, char* recv_data, int64_t recv_len);
/*!
* \brief Get rank of local machine
*/
inline int rank();
/*!
* \brief Get total number of machines
*/
inline int num_machines();
/*!
* \brief Get Bruck map of this network
*/
inline const BruckMap& bruck_map();
/*!
* \brief Get Recursive Halving map of this network
*/
inline const RecursiveHalvingMap& recursive_halving_map();
#ifdef USE_SOCKET
/*!
* \brief Bind local listen to port
* \param port Local listen port
*/
void TryBind(int port);
/*!
* \brief Set socket to rank
* \param rank
* \param socket
*/
void SetLinker(int rank, const TcpSocket& socket);
/*!
* \brief Thread for listening
* \param incoming_cnt Number of incoming machines
*/
void ListenThread(int incoming_cnt);
/*!
* \brief Construct network topo
*/
void Construct();
/*!
* \brief Parser machines information from file
* \param machines
* \param filename
*/
void ParseMachineList(const std::string& machines, const std::string& filename);
/*!
* \brief Check one linker is connected or not
* \param rank
* \return True if linker is connected
*/
bool CheckLinker(int rank);
/*!
* \brief Print connected linkers
*/
void PrintLinkers();
#endif // USE_SOCKET
#ifdef USE_MPI
/*!
* \brief Check if MPI has been initialized
*/
static bool IsMpiInitialized();
/*!
* \brief Finalize the MPI session if it was initialized
*/
static void MpiFinalizeIfIsParallel();
/*!
* \brief Abort the MPI session if it was initialized (called in case there was a error that needs abrupt ending)
*/
static void MpiAbortIfIsParallel();
#endif
private:
/*! \brief Rank of local machine */
int rank_;
/*! \brief Total number machines */
int num_machines_;
/*! \brief Bruck map */
BruckMap bruck_map_;
/*! \brief Recursive Halving map */
RecursiveHalvingMap recursive_halving_map_;
std::chrono::duration<double, std::milli> network_time_;
bool is_init_;
#ifdef USE_SOCKET
/*! \brief use to store client ips */
std::vector<std::string> client_ips_;
/*! \brief use to store client ports */
std::vector<int> client_ports_;
/*! \brief time out for sockets, in minutes */
int socket_timeout_;
/*! \brief Local listen ports */
int local_listen_port_;
/*! \brief Linkers */
std::vector<std::unique_ptr<TcpSocket>> linkers_;
/*! \brief Local socket listener */
std::unique_ptr<TcpSocket> listener_;
#endif // USE_SOCKET
};
inline int Linkers::rank() {
return rank_;
}
inline int Linkers::num_machines() {
return num_machines_;
}
inline const BruckMap& Linkers::bruck_map() {
return bruck_map_;
}
inline const RecursiveHalvingMap& Linkers::recursive_halving_map() {
return recursive_halving_map_;
}
inline void Linkers::Recv(int rank, char* data, int64_t len) const {
int64_t used = 0;
do {
int cur_size = static_cast<int>(std::min<int64_t>(len - used, INT32_MAX));
Recv(rank, data + used, cur_size);
used += cur_size;
} while (used < len);
}
inline void Linkers::Send(int rank, char* data, int64_t len) const {
int64_t used = 0;
do {
int cur_size = static_cast<int>(std::min<int64_t>(len - used, INT32_MAX));
Send(rank, data + used, cur_size);
used += cur_size;
} while (used < len);
}
inline void Linkers::SendRecv(int send_rank, char* send_data, int64_t send_len,
int recv_rank, char* recv_data, int64_t recv_len) {
auto start_time = std::chrono::high_resolution_clock::now();
std::thread send_worker(
[this, send_rank, send_data, send_len]() {
Send(send_rank, send_data, send_len);
});
Recv(recv_rank, recv_data, recv_len);
send_worker.join();
// wait for send complete
auto end_time = std::chrono::high_resolution_clock::now();
// output used time on each iteration
network_time_ += std::chrono::duration<double, std::milli>(end_time - start_time);
}
#ifdef USE_SOCKET
inline void Linkers::Recv(int rank, char* data, int len) const {
int recv_cnt = 0;
while (recv_cnt < len) {
recv_cnt += linkers_[rank]->Recv(data + recv_cnt,
// len - recv_cnt
std::min(len - recv_cnt, SocketConfig::kMaxReceiveSize));
}
}
inline void Linkers::Send(int rank, char* data, int len) const {
if (len <= 0) {
return;
}
int send_cnt = 0;
while (send_cnt < len) {
send_cnt += linkers_[rank]->Send(data + send_cnt, len - send_cnt);
}
}
inline void Linkers::SendRecv(int send_rank, char* send_data, int send_len,
int recv_rank, char* recv_data, int recv_len) {
auto start_time = std::chrono::high_resolution_clock::now();
if (send_len < SocketConfig::kSocketBufferSize) {
// if buffer is enough, send will non-blocking
Send(send_rank, send_data, send_len);
Recv(recv_rank, recv_data, recv_len);
} else {
// if buffer is not enough, use another thread to send, since send will be blocking
std::thread send_worker(
[this, send_rank, send_data, send_len]() {
Send(send_rank, send_data, send_len);
});
Recv(recv_rank, recv_data, recv_len);
send_worker.join();
}
// wait for send complete
auto end_time = std::chrono::high_resolution_clock::now();
// output used time on each iteration
network_time_ += std::chrono::duration<double, std::milli>(end_time - start_time);
}
#endif // USE_SOCKET
#ifdef USE_MPI
inline void Linkers::Recv(int rank, char* data, int len) const {
MPI_Status status;
int read_cnt = 0;
while (read_cnt < len) {
MPI_SAFE_CALL(MPI_Recv(data + read_cnt, len - read_cnt, MPI_BYTE, rank, MPI_ANY_TAG, MPI_COMM_WORLD, &status));
int cur_cnt;
MPI_SAFE_CALL(MPI_Get_count(&status, MPI_BYTE, &cur_cnt));
read_cnt += cur_cnt;
}
}
inline void Linkers::Send(int rank, char* data, int len) const {
if (len <= 0) {
return;
}
MPI_Status status;
MPI_Request send_request;
MPI_SAFE_CALL(MPI_Isend(data, len, MPI_BYTE, rank, 0, MPI_COMM_WORLD, &send_request));
MPI_SAFE_CALL(MPI_Wait(&send_request, &status));
}
inline void Linkers::SendRecv(int send_rank, char* send_data, int send_len,
int recv_rank, char* recv_data, int recv_len) {
MPI_Request send_request;
// send first, non-blocking
MPI_SAFE_CALL(MPI_Isend(send_data, send_len, MPI_BYTE, send_rank, 0, MPI_COMM_WORLD, &send_request));
// then receive, blocking
MPI_Status status;
int read_cnt = 0;
while (read_cnt < recv_len) {
MPI_SAFE_CALL(MPI_Recv(recv_data + read_cnt, recv_len - read_cnt, MPI_BYTE, recv_rank, 0, MPI_COMM_WORLD, &status));
int cur_cnt;
MPI_SAFE_CALL(MPI_Get_count(&status, MPI_BYTE, &cur_cnt));
read_cnt += cur_cnt;
}
// wait for send complete
MPI_SAFE_CALL(MPI_Wait(&send_request, &status));
}
#endif // USE_MPI
} // namespace LightGBM
#endif // LIGHTGBM_SRC_NETWORK_LINKERS_H_
src/network/linkers_mpi.cpp
+64
View File
@@ -0,0 +1,64 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_MPI
#include "linkers.h"
#include <iostream>
namespace LightGBM {
Linkers::Linkers(Config) {
is_init_ = false;
int argc = 0;
char**argv = nullptr;
int flag = 0;
MPI_SAFE_CALL(MPI_Initialized(&flag)); // test if MPI has been initialized
if (!flag) { // if MPI not started, start it
MPI_SAFE_CALL(MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &flag));
}
MPI_SAFE_CALL(MPI_Comm_size(MPI_COMM_WORLD, &num_machines_));
MPI_SAFE_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &rank_));
// wait for all client start up
MPI_SAFE_CALL(MPI_Barrier(MPI_COMM_WORLD));
bruck_map_ = BruckMap::Construct(rank_, num_machines_);
recursive_halving_map_ = RecursiveHalvingMap::Construct(rank_, num_machines_);
is_init_ = true;
}
Linkers::~Linkers() {
// Don't call MPI_Finalize() here: If the destructor was called because only this node had an exception, calling MPI_Finalize() will cause all nodes to hang.
// Instead we will handle finalize/abort for MPI in main().
}
bool Linkers::IsMpiInitialized() {
int is_mpi_init;
MPI_SAFE_CALL(MPI_Initialized(&is_mpi_init));
return is_mpi_init;
}
void Linkers::MpiFinalizeIfIsParallel() {
if (IsMpiInitialized()) {
Log::Debug("Finalizing MPI session.");
MPI_SAFE_CALL(MPI_Finalize());
}
}
void Linkers::MpiAbortIfIsParallel() {
try {
if (IsMpiInitialized()) {
std::cerr << "Aborting MPI communication." << std::endl << std::flush;
MPI_SAFE_CALL(MPI_Abort(MPI_COMM_WORLD, -1));;
}
}
catch (...) {
std::cerr << "Exception was raised before aborting MPI. Aborting process..." << std::endl << std::flush;
abort();
}
}
} // namespace LightGBM
#endif // USE_MPI
src/network/linkers_socket.cpp
+241
View File
@@ -0,0 +1,241 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_SOCKET
#include <LightGBM/config.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/text_reader.h>
#include <algorithm>
#include <chrono>
#include <cstring>
#include <memory>
#include <string>
#include <thread>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "linkers.h"
namespace LightGBM {
Linkers::Linkers(Config config) {
is_init_ = false;
// start up socket
TcpSocket::Startup();
network_time_ = std::chrono::duration<double, std::milli>(0);
num_machines_ = config.num_machines;
local_listen_port_ = config.local_listen_port;
socket_timeout_ = config.time_out;
rank_ = -1;
// parse clients from file
ParseMachineList(config.machines, config.machine_list_filename);
if (rank_ == -1) {
// get ip list of local machine
std::unordered_set<std::string> local_ip_list = TcpSocket::GetLocalIpList();
// get local rank
for (size_t i = 0; i < client_ips_.size(); ++i) {
if (local_ip_list.count(client_ips_[i]) > 0 && client_ports_[i] == local_listen_port_) {
rank_ = static_cast<int>(i);
break;
}
}
}
if (rank_ == -1) {
Log::Fatal("Machine list file doesn't contain the local machine");
}
// construct listener
listener_ = std::unique_ptr<TcpSocket>(new TcpSocket());
TryBind(local_listen_port_);
for (int i = 0; i < num_machines_; ++i) {
linkers_.push_back(nullptr);
}
// construct communication topo
bruck_map_ = BruckMap::Construct(rank_, num_machines_);
recursive_halving_map_ = RecursiveHalvingMap::Construct(rank_, num_machines_);
// construct linkers
Construct();
// free listener
listener_->Close();
is_init_ = true;
}
Linkers::~Linkers() {
if (is_init_) {
for (size_t i = 0; i < linkers_.size(); ++i) {
if (linkers_[i] != nullptr) {
linkers_[i]->Close();
}
}
TcpSocket::Finalize();
Log::Info("Finished linking network in %f seconds", network_time_ * 1e-3);
}
}
void Linkers::ParseMachineList(const std::string& machines, const std::string& filename) {
std::vector<std::string> lines;
if (machines.empty()) {
TextReader<size_t> machine_list_reader(filename.c_str(), false);
machine_list_reader.ReadAllLines();
if (machine_list_reader.Lines().empty()) {
Log::Fatal("Machine list file %s doesn't exist", filename.c_str());
}
lines = machine_list_reader.Lines();
} else {
lines = Common::Split(machines.c_str(), ',');
}
for (auto& line : lines) {
line = Common::Trim(line);
if (line.find("rank=") != std::string::npos) {
std::vector<std::string> str_after_split = Common::Split(line.c_str(), '=');
Common::Atoi(str_after_split[1].c_str(), &rank_);
continue;
}
std::vector<std::string> str_after_split = Common::Split(line.c_str(), ' ');
if (str_after_split.size() != 2) {
str_after_split = Common::Split(line.c_str(), ':');
if (str_after_split.size() != 2) {
continue;
}
}
if (client_ips_.size() >= static_cast<size_t>(num_machines_)) {
Log::Warning("machine_list size is larger than the parameter num_machines, ignoring redundant entries");
break;
}
str_after_split[0] = Common::Trim(str_after_split[0]);
str_after_split[1] = Common::Trim(str_after_split[1]);
client_ips_.push_back(str_after_split[0]);
client_ports_.push_back(atoi(str_after_split[1].c_str()));
}
if (client_ips_.empty()) {
Log::Fatal("Cannot find any ip and port.\n"
"Please check machine_list_filename or machines parameter");
}
if (client_ips_.size() != static_cast<size_t>(num_machines_)) {
Log::Warning("World size is larger than the machine_list size, change world size to %zu", client_ips_.size());
num_machines_ = static_cast<int>(client_ips_.size());
}
}
void Linkers::TryBind(int port) {
Log::Info("Trying to bind port %d...", port);
if (listener_->Bind(port)) {
Log::Info("Binding port %d succeeded", port);
} else {
Log::Fatal("Binding port %d failed", port);
}
}
void Linkers::SetLinker(int rank, const TcpSocket& socket) {
linkers_[rank].reset(new TcpSocket(socket));
// set timeout
linkers_[rank]->SetTimeout(socket_timeout_ * 1000 * 60);
}
void Linkers::ListenThread(int incoming_cnt) {
Log::Info("Listening...");
char buffer[100];
int connected_cnt = 0;
while (connected_cnt < incoming_cnt) {
// accept incoming socket
TcpSocket handler = listener_->Accept();
if (handler.IsClosed()) {
continue;
}
// receive rank
int read_cnt = 0;
int size_of_int = static_cast<int>(sizeof(int));
while (read_cnt < size_of_int) {
int cur_read_cnt = handler.Recv(buffer + read_cnt, size_of_int - read_cnt);
read_cnt += cur_read_cnt;
}
int* ptr_in_rank = reinterpret_cast<int*>(buffer);
int in_rank = *ptr_in_rank;
if (in_rank < 0 || in_rank >= num_machines_) {
Log::Fatal("Invalid rank %d found during initialization of linkers. The world size is %d.", in_rank, num_machines_);
}
// add new socket
SetLinker(in_rank, handler);
++connected_cnt;
}
}
void Linkers::Construct() {
// save ranks that need to connect with
std::unordered_map<int, int> need_connect;
for (int i = 0; i < num_machines_; ++i) {
if (i != rank_) {
need_connect[i] = 1;
}
}
int incoming_cnt = 0;
for (auto it = need_connect.begin(); it != need_connect.end(); ++it) {
int machine_rank = it->first;
if (machine_rank < rank_) {
++incoming_cnt;
}
}
// start listener
listener_->SetTimeout(socket_timeout_ * 1000 * 60);
listener_->Listen(incoming_cnt);
std::thread listen_thread(&Linkers::ListenThread, this, incoming_cnt);
const int connect_fail_constant_factor = 20;
const int connect_fail_retries_scale_factor = static_cast<int>(num_machines_ / connect_fail_constant_factor);
const int connect_fail_retry_cnt = std::max(connect_fail_constant_factor, connect_fail_retries_scale_factor);
const int connect_fail_retry_first_delay_interval = 200; // 0.2 s
const float connect_fail_retry_delay_factor = 1.3f;
// start connect
for (auto it = need_connect.begin(); it != need_connect.end(); ++it) {
int out_rank = it->first;
// let smaller rank connect to larger rank
if (out_rank > rank_) {
int connect_fail_delay_time = connect_fail_retry_first_delay_interval;
for (int i = 0; i < connect_fail_retry_cnt; ++i) {
TcpSocket cur_socket;
if (cur_socket.Connect(client_ips_[out_rank].c_str(), client_ports_[out_rank])) {
// send local rank
cur_socket.Send(reinterpret_cast<const char*>(&rank_), sizeof(rank_));
SetLinker(out_rank, cur_socket);
break;
} else {
Log::Warning("Connecting to rank %d failed, waiting for %d milliseconds", out_rank, connect_fail_delay_time);
cur_socket.Close();
std::this_thread::sleep_for(std::chrono::milliseconds(connect_fail_delay_time));
connect_fail_delay_time = static_cast<int>(connect_fail_delay_time * connect_fail_retry_delay_factor);
}
}
}
}
// wait for listener
listen_thread.join();
// print connected linkers
PrintLinkers();
}
bool Linkers::CheckLinker(int rank) {
if (linkers_[rank] == nullptr || linkers_[rank]->IsClosed()) {
return false;
}
return true;
}
void Linkers::PrintLinkers() {
for (int i = 0; i < num_machines_; ++i) {
if (CheckLinker(i)) {
Log::Info("Connected to rank %d", i);
}
}
}
} // namespace LightGBM
#endif // USE_SOCKET
src/network/network.cpp
+332
View File
@@ -0,0 +1,332 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/network.h>
#include <LightGBM/utils/common.h>
#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <memory>
#include <vector>
#include "linkers.h"
namespace LightGBM {
// static member definition
THREAD_LOCAL int Network::num_machines_ = 1;
THREAD_LOCAL int Network::rank_ = 0;
THREAD_LOCAL std::unique_ptr<Linkers> Network::linkers_;
THREAD_LOCAL BruckMap Network::bruck_map_;
THREAD_LOCAL RecursiveHalvingMap Network::recursive_halving_map_;
THREAD_LOCAL std::vector<comm_size_t> Network::block_start_;
THREAD_LOCAL std::vector<comm_size_t> Network::block_len_;
THREAD_LOCAL comm_size_t Network::buffer_size_ = 0;
THREAD_LOCAL std::vector<char> Network::buffer_;
THREAD_LOCAL ReduceScatterFunction Network::reduce_scatter_ext_fun_ = nullptr;
THREAD_LOCAL AllgatherFunction Network::allgather_ext_fun_ = nullptr;
void Network::Init(Config config) {
if (config.num_machines > 1) {
linkers_.reset(new Linkers(config));
rank_ = linkers_->rank();
num_machines_ = linkers_->num_machines();
bruck_map_ = linkers_->bruck_map();
recursive_halving_map_ = linkers_->recursive_halving_map();
block_start_ = std::vector<comm_size_t>(num_machines_);
block_len_ = std::vector<comm_size_t>(num_machines_);
buffer_size_ = 1024 * 1024;
buffer_.resize(buffer_size_);
Log::Info("Local rank: %d, total number of machines: %d", rank_, num_machines_);
}
}
void Network::Init(int num_machines, int rank,
ReduceScatterFunction reduce_scatter_ext_fun, AllgatherFunction allgather_ext_fun) {
if (num_machines > 1) {
rank_ = rank;
num_machines_ = num_machines;
block_start_ = std::vector<comm_size_t>(num_machines_);
block_len_ = std::vector<comm_size_t>(num_machines_);
buffer_size_ = 1024 * 1024;
buffer_.resize(buffer_size_);
reduce_scatter_ext_fun_ = reduce_scatter_ext_fun;
allgather_ext_fun_ = allgather_ext_fun;
Log::Info("Local rank: %d, total number of machines: %d", rank_, num_machines_);
}
}
void Network::Dispose() {
num_machines_ = 1;
rank_ = 0;
linkers_.reset(new Linkers());
reduce_scatter_ext_fun_ = nullptr;
allgather_ext_fun_ = nullptr;
}
void Network::Allreduce(char* input, comm_size_t input_size, int type_size, char* output, const ReduceFunction& reducer) {
if (num_machines_ <= 1) {
Log::Fatal("Please initialize the network interface first");
}
comm_size_t count = input_size / type_size;
// if small package or small count , do it by all gather.(reduce the communication times.)
if (count < num_machines_ || input_size < 4096) {
AllreduceByAllGather(input, input_size, type_size, output, reducer);
return;
}
// assign the blocks to every rank.
comm_size_t step = (count + num_machines_ - 1) / num_machines_;
if (step < 1) {
step = 1;
}
block_start_[0] = 0;
for (int i = 0; i < num_machines_ - 1; ++i) {
block_len_[i] = std::min<comm_size_t>(step * type_size, input_size - block_start_[i]);
block_start_[i + 1] = block_start_[i] + block_len_[i];
}
block_len_[num_machines_ - 1] = input_size - block_start_[num_machines_ - 1];
// do reduce scatter
ReduceScatter(input, input_size, type_size, block_start_.data(), block_len_.data(), output, input_size, reducer);
// do all gather
Allgather(output, block_start_.data(), block_len_.data(), output, input_size);
}
void Network::AllreduceByAllGather(char* input, comm_size_t input_size, int type_size, char* output, const ReduceFunction& reducer) {
if (num_machines_ <= 1) {
Log::Fatal("Please initialize the network interface first");
}
// assign blocks
comm_size_t all_size = input_size * num_machines_;
block_start_[0] = 0;
block_len_[0] = input_size;
for (int i = 1; i < num_machines_; ++i) {
block_start_[i] = block_start_[i - 1] + block_len_[i - 1];
block_len_[i] = input_size;
}
// need use buffer here, since size of "output" is smaller than size after all gather
if (input_size*num_machines_ > buffer_size_) {
buffer_size_ = input_size*num_machines_;
buffer_.resize(buffer_size_);
}
Allgather(input, block_start_.data(), block_len_.data(), buffer_.data(), all_size);
for (int i = 1; i < num_machines_; ++i) {
reducer(buffer_.data() + block_start_[i], buffer_.data() + block_start_[0], type_size, input_size);
}
// copy back
std::memcpy(output, buffer_.data(), input_size);
}
void Network::Allgather(char* input, comm_size_t send_size, char* output) {
if (num_machines_ <= 1) {
Log::Fatal("Please initialize the network interface first");
return;
}
// assign blocks
block_start_[0] = 0;
block_len_[0] = send_size;
for (int i = 1; i < num_machines_; ++i) {
block_start_[i] = block_start_[i - 1] + block_len_[i - 1];
block_len_[i] = send_size;
}
// start all gather
Allgather(input, block_start_.data(), block_len_.data(), output, send_size * num_machines_);
}
void Network::Allgather(char* input, const comm_size_t* block_start, const comm_size_t* block_len, char* output, comm_size_t all_size) {
if (num_machines_ <= 1) {
Log::Fatal("Please initialize the network interface first");
}
if (allgather_ext_fun_ != nullptr) {
return allgather_ext_fun_(input, block_len[rank_], block_start, block_len, num_machines_, output, all_size);
}
const comm_size_t kRingThreshold = 10 * 1024 * 1024; // 10MB
const int kRingNodeThreshold = 64;
if (all_size > kRingThreshold && num_machines_ < kRingNodeThreshold) {
// when num_machines is small and data is large
AllgatherRing(input, block_start, block_len, output, all_size);
} else if (recursive_halving_map_.is_power_of_2) {
AllgatherRecursiveDoubling(input, block_start, block_len, output, all_size);
} else {
AllgatherBruck(input, block_start, block_len, output, all_size);
}
}
void Network::AllgatherBruck(char* input, const comm_size_t* block_start, const comm_size_t* block_len, char* output, comm_size_t all_size) {
comm_size_t write_pos = 0;
// use output as receive buffer
std::memcpy(output, input, block_len[rank_]);
write_pos += block_len[rank_];
int accumulated_block = 1;
for (int i = 0; i < bruck_map_.k; ++i) {
// get current local block size
int cur_block_size = std::min(1 << i, num_machines_ - accumulated_block);
// get out rank
int out_rank = bruck_map_.out_ranks[i];
// get in rank
int in_rank = bruck_map_.in_ranks[i];
// get send information
comm_size_t need_send_len = 0;
// get recv information
comm_size_t need_recv_len = 0;
for (int j = 0; j < cur_block_size; ++j) {
need_send_len += block_len[(rank_ + j) % num_machines_];
need_recv_len += block_len[(rank_ + accumulated_block + j) % num_machines_];
}
// send and recv at same time
linkers_->SendRecv(out_rank, output, need_send_len, in_rank, output + write_pos, need_recv_len);
write_pos += need_recv_len;
accumulated_block += cur_block_size;
}
// rotate in-place
std::reverse<char*>(output, output + all_size);
std::reverse<char*>(output, output + block_start[rank_]);
std::reverse<char*>(output + block_start[rank_], output + all_size);
}
void Network::AllgatherRecursiveDoubling(char* input, const comm_size_t* block_start, const comm_size_t* block_len, char* output, comm_size_t) {
// use output as receive buffer
std::memcpy(output + block_start[rank_], input, block_len[rank_]);
for (int i = 0; i < bruck_map_.k; ++i) {
// get current local block size
int cur_step = 1 << i;
const int vgroup = rank_ / cur_step;
const int vrank = vgroup * cur_step;
int target = rank_ + cur_step;
int target_vrank = (vgroup + 1) * cur_step;
if (vgroup & 1) {
target = rank_ - cur_step;
target_vrank = (vgroup - 1) * cur_step;
}
// get send information
comm_size_t need_send_len = 0;
// get recv information
comm_size_t need_recv_len = 0;
for (int j = 0; j < cur_step; ++j) {
need_send_len += block_len[(vrank + j)];
need_recv_len += block_len[(target_vrank + j)];
}
// send and recv at same time
linkers_->SendRecv(target, output + block_start[vrank], need_send_len,
target, output + block_start[target_vrank], need_recv_len);
}
}
void Network::AllgatherRing(char* input, const comm_size_t* block_start, const comm_size_t* block_len, char* output, comm_size_t) {
// use output as receive buffer
std::memcpy(output + block_start[rank_], input, block_len[rank_]);
int out_rank = (rank_ + 1) % num_machines_;
int in_rank = (rank_ - 1 + num_machines_) % num_machines_;
int out_block = rank_;
int in_block = in_rank;
for (int i = 1; i < num_machines_; ++i) {
// send and recv at same time
linkers_->SendRecv(out_rank, output + block_start[out_block], block_len[out_block],
in_rank, output + block_start[in_block], block_len[in_block]);
out_block = (out_block - 1 + num_machines_) % num_machines_;
in_block = (in_block - 1 + num_machines_) % num_machines_;
}
}
void Network::ReduceScatter(char* input, comm_size_t input_size, int type_size,
const comm_size_t* block_start, const comm_size_t* block_len, char* output,
comm_size_t output_size, const ReduceFunction& reducer) {
if (num_machines_ <= 1) {
Log::Fatal("Please initialize the network interface first");
}
if (reduce_scatter_ext_fun_ != nullptr) {
return reduce_scatter_ext_fun_(input, input_size, type_size, block_start, block_len, num_machines_, output, output_size, reducer);
}
const comm_size_t kRingThreshold = 10 * 1024 * 1024; // 10MB
if (recursive_halving_map_.is_power_of_2 || input_size < kRingThreshold) {
ReduceScatterRecursiveHalving(input, input_size, type_size, block_start, block_len, output, output_size, reducer);
} else {
ReduceScatterRing(input, input_size, type_size, block_start, block_len, output, output_size, reducer);
}
}
void Network::ReduceScatterRecursiveHalving(char* input, comm_size_t input_size, int type_size,
const comm_size_t* block_start, const comm_size_t* block_len, char* output,
comm_size_t, const ReduceFunction& reducer) {
if (!recursive_halving_map_.is_power_of_2) {
if (recursive_halving_map_.type == RecursiveHalvingNodeType::Other) {
// send local data to neighbor first
linkers_->Send(recursive_halving_map_.neighbor, input, input_size);
} else if (recursive_halving_map_.type == RecursiveHalvingNodeType::GroupLeader) {
// receive neighbor data first
int need_recv_cnt = input_size;
linkers_->Recv(recursive_halving_map_.neighbor, output, need_recv_cnt);
// reduce
reducer(output, input, type_size, input_size);
}
}
if (recursive_halving_map_.type != RecursiveHalvingNodeType::Other) {
for (int i = 0; i < recursive_halving_map_.k; ++i) {
// get target
int target = recursive_halving_map_.ranks[i];
comm_size_t send_block_start = recursive_halving_map_.send_block_start[i];
comm_size_t recv_block_start = recursive_halving_map_.recv_block_start[i];
// get send information
comm_size_t send_size = 0;
for (int j = 0; j < recursive_halving_map_.send_block_len[i]; ++j) {
send_size += block_len[send_block_start + j];
}
// get recv information
comm_size_t need_recv_cnt = 0;
for (int j = 0; j < recursive_halving_map_.recv_block_len[i]; ++j) {
need_recv_cnt += block_len[recv_block_start + j];
}
// send and recv at same time
linkers_->SendRecv(target, input + block_start[send_block_start], send_size, target, output, need_recv_cnt);
// reduce
reducer(output, input + block_start[recv_block_start], type_size, need_recv_cnt);
}
}
if (!recursive_halving_map_.is_power_of_2) {
if (recursive_halving_map_.type == RecursiveHalvingNodeType::GroupLeader) {
// send result to neighbor
linkers_->Send(recursive_halving_map_.neighbor,
input + block_start[recursive_halving_map_.neighbor],
block_len[recursive_halving_map_.neighbor]);
} else if (recursive_halving_map_.type == RecursiveHalvingNodeType::Other) {
// receive result from neighbor
int need_recv_cnt = block_len[rank_];
linkers_->Recv(recursive_halving_map_.neighbor, output, need_recv_cnt);
return;
}
}
// copy result
std::memcpy(output, input + block_start[rank_], block_len[rank_]);
}
void Network::ReduceScatterRing(char* input, comm_size_t, int type_size,
const comm_size_t* block_start, const comm_size_t* block_len, char* output,
comm_size_t, const ReduceFunction& reducer) {
const int out_rank = (rank_ + 1) % num_machines_;
const int in_rank = (rank_ - 1 + num_machines_) % num_machines_;
int out_block = in_rank;
int in_block = (in_rank - 1 + num_machines_) % num_machines_;
for (int i = 1; i < num_machines_; ++i) {
linkers_->SendRecv(out_rank, input + block_start[out_block], block_len[out_block],
in_rank, output, block_len[in_block]);
reducer(output, input + block_start[in_block], type_size, block_len[in_block]);
out_block = (out_block - 1 + num_machines_) % num_machines_;
in_block = (in_block - 1 + num_machines_) % num_machines_;
}
std::memcpy(output, input + block_start[rank_], block_len[rank_]);
}
int Network::rank() {
return rank_;
}
int Network::num_machines() {
return num_machines_;
}
} // namespace LightGBM
src/network/socket_wrapper.hpp
+333
View File
@@ -0,0 +1,333 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_NETWORK_SOCKET_WRAPPER_HPP_
#define LIGHTGBM_SRC_NETWORK_SOCKET_WRAPPER_HPP_
#ifdef USE_SOCKET
#include <LightGBM/utils/log.h>
#include <string>
#include <cerrno>
#include <cstdlib>
#include <unordered_set>
#if defined(_WIN32)
#ifdef _MSC_VER
#define NOMINMAX
#endif
#include <winsock2.h>
#include <ws2tcpip.h>
#include <iphlpapi.h>
#else
#include <arpa/inet.h>
#include <fcntl.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>
#include <ifaddrs.h>
#include <sys/time.h>
#endif // defined(_WIN32)
#ifdef _MSC_VER
#pragma comment(lib, "Ws2_32.lib")
#pragma comment(lib, "IPHLPAPI.lib")
#endif
namespace LightGBM {
#ifndef _WIN32
typedef int SOCKET;
const int INVALID_SOCKET = -1;
#define SOCKET_ERROR -1
#endif
#ifdef _WIN32
#define MALLOC(x) HeapAlloc(GetProcessHeap(), 0, (x))
#define FREE(x) HeapFree(GetProcessHeap(), 0, (x))
// existence of inet_pton is checked in CMakeLists.txt and configure.win, then stored in WIN_HAS_INET_PTON
#ifndef WIN_HAS_INET_PTON
inline int inet_pton(int af, const char *src, void *dst) {
struct sockaddr_storage ss;
int size = sizeof(ss);
char src_copy[INET6_ADDRSTRLEN + 1];
ZeroMemory(&ss, sizeof(ss));
/* stupid non-const API */
strncpy(src_copy, src, INET6_ADDRSTRLEN + 1);
src_copy[INET6_ADDRSTRLEN] = 0;
if (WSAStringToAddress(src_copy, af, NULL, (struct sockaddr *)&ss, &size) == 0) {
switch (af) {
case AF_INET:
*(struct in_addr *)dst = ((struct sockaddr_in *)&ss)->sin_addr;
return 1;
case AF_INET6:
*(struct in6_addr *)dst = ((struct sockaddr_in6 *)&ss)->sin6_addr;
return 1;
}
}
return 0;
}
#endif
#endif
namespace SocketConfig {
const int kSocketBufferSize = 100 * 1000;
const int kMaxReceiveSize = 100 * 1000;
const int kNoDelay = 1;
}
class TcpSocket {
public:
TcpSocket() {
sockfd_ = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (sockfd_ == INVALID_SOCKET) {
Log::Fatal("Socket construction error");
return;
}
ConfigSocket();
}
explicit TcpSocket(SOCKET socket) {
sockfd_ = socket;
if (sockfd_ == INVALID_SOCKET) {
Log::Fatal("Passed socket error");
return;
}
ConfigSocket();
}
TcpSocket(const TcpSocket &object) {
sockfd_ = object.sockfd_;
ConfigSocket();
}
~TcpSocket() {
}
inline void SetTimeout(int timeout_ms) {
#if defined(_WIN32)
DWORD timeout = static_cast<DWORD>(timeout_ms);
setsockopt(sockfd_, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast<const char*>(&timeout), sizeof(timeout));
#else
struct timeval tv;
tv.tv_sec = timeout_ms / 1000;
tv.tv_usec = (timeout_ms % 1000) * 1000;
setsockopt(sockfd_, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
#endif
}
inline void ConfigSocket() {
if (sockfd_ == INVALID_SOCKET) {
return;
}
if (setsockopt(sockfd_, SOL_SOCKET, SO_RCVBUF, reinterpret_cast<const char*>(&SocketConfig::kSocketBufferSize), sizeof(SocketConfig::kSocketBufferSize)) != 0) {
Log::Warning("Set SO_RCVBUF failed, please increase your net.core.rmem_max to 100k at least");
}
if (setsockopt(sockfd_, SOL_SOCKET, SO_SNDBUF, reinterpret_cast<const char*>(&SocketConfig::kSocketBufferSize), sizeof(SocketConfig::kSocketBufferSize)) != 0) {
Log::Warning("Set SO_SNDBUF failed, please increase your net.core.wmem_max to 100k at least");
}
if (setsockopt(sockfd_, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<const char*>(&SocketConfig::kNoDelay), sizeof(SocketConfig::kNoDelay)) != 0) {
Log::Warning("Set TCP_NODELAY failed");
}
}
inline static void Startup() {
#if defined(_WIN32)
WSADATA wsa_data;
if (WSAStartup(MAKEWORD(2, 2), &wsa_data) == -1) {
Log::Fatal("Socket error: WSAStartup error");
}
if (LOBYTE(wsa_data.wVersion) != 2 || HIBYTE(wsa_data.wVersion) != 2) {
WSACleanup();
Log::Fatal("Socket error: Winsock.dll version error");
}
#else
#endif
}
inline static void Finalize() {
#if defined(_WIN32)
WSACleanup();
#endif
}
inline static int GetLastError() {
#if defined(_WIN32)
return WSAGetLastError();
#else
return errno;
#endif
}
#if defined(_WIN32)
inline static std::unordered_set<std::string> GetLocalIpList() {
std::unordered_set<std::string> ip_list;
char buffer[512];
// get hostName
if (gethostname(buffer, sizeof(buffer)) == SOCKET_ERROR) {
Log::Fatal("Error code %d, when getting local host name", WSAGetLastError());
}
// push local ip
PIP_ADAPTER_INFO pAdapterInfo;
PIP_ADAPTER_INFO pAdapter = NULL;
DWORD dwRetVal = 0;
ULONG ulOutBufLen = sizeof(IP_ADAPTER_INFO);
pAdapterInfo = reinterpret_cast<IP_ADAPTER_INFO *>(MALLOC(sizeof(IP_ADAPTER_INFO)));
if (pAdapterInfo == NULL) {
Log::Fatal("GetAdaptersinfo error: allocating memory");
}
// Make an initial call to GetAdaptersInfo to get
// the necessary size into the ulOutBufLen variable
if (GetAdaptersInfo(pAdapterInfo, &ulOutBufLen) == ERROR_BUFFER_OVERFLOW) {
FREE(pAdapterInfo);
pAdapterInfo = reinterpret_cast<IP_ADAPTER_INFO *>(MALLOC(ulOutBufLen));
if (pAdapterInfo == NULL) {
Log::Fatal("GetAdaptersinfo error: allocating memory");
}
}
if ((dwRetVal = GetAdaptersInfo(pAdapterInfo, &ulOutBufLen)) == NO_ERROR) {
pAdapter = pAdapterInfo;
while (pAdapter) {
ip_list.insert(pAdapter->IpAddressList.IpAddress.String);
pAdapter = pAdapter->Next;
}
} else {
Log::Fatal("GetAdaptersinfo error: code %d", dwRetVal);
}
if (pAdapterInfo)
FREE(pAdapterInfo);
return ip_list;
}
#else
inline static std::unordered_set<std::string> GetLocalIpList() {
std::unordered_set<std::string> ip_list;
struct ifaddrs * ifAddrStruct = NULL;
struct ifaddrs * ifa = NULL;
void * tmpAddrPtr = NULL;
getifaddrs(&ifAddrStruct);
for (ifa = ifAddrStruct; ifa != NULL; ifa = ifa->ifa_next) {
if (!ifa->ifa_addr) {
continue;
}
if (ifa->ifa_addr->sa_family == AF_INET) {
// NOLINTNEXTLINE
tmpAddrPtr = &((struct sockaddr_in *)ifa->ifa_addr)->sin_addr;
char addressBuffer[INET_ADDRSTRLEN];
inet_ntop(AF_INET, tmpAddrPtr, addressBuffer, INET_ADDRSTRLEN);
ip_list.insert(std::string(addressBuffer));
}
}
if (ifAddrStruct != NULL) freeifaddrs(ifAddrStruct);
return ip_list;
}
#endif
inline static sockaddr_in GetAddress(const char* url, int port) {
sockaddr_in addr = sockaddr_in();
std::memset(&addr, 0, sizeof(sockaddr_in));
inet_pton(AF_INET, url, &addr.sin_addr);
addr.sin_family = AF_INET;
addr.sin_port = htons(static_cast<u_short>(port));
return addr;
}
inline bool Bind(int port) {
sockaddr_in local_addr = GetAddress("0.0.0.0", port);
if (bind(sockfd_, reinterpret_cast<const sockaddr*>(&local_addr), sizeof(sockaddr_in)) == 0) {
return true;
}
return false;
}
inline bool Connect(const char *url, int port) {
sockaddr_in server_addr = GetAddress(url, port);
if (connect(sockfd_, reinterpret_cast<const sockaddr*>(&server_addr), sizeof(sockaddr_in)) == 0) {
return true;
}
return false;
}
inline void Listen(int backlog = 128) {
listen(sockfd_, backlog);
}
inline TcpSocket Accept() {
SOCKET newfd = accept(sockfd_, NULL, NULL);
if (newfd == INVALID_SOCKET) {
int err_code = GetLastError();
#if defined(_WIN32)
Log::Fatal("Socket accept error (code: %d)", err_code);
#else
Log::Fatal("Socket accept error, %s (code: %d)", std::strerror(err_code), err_code);
#endif
}
return TcpSocket(newfd);
}
inline int Send(const char *buf_, int len, int flag = 0) {
int cur_cnt = send(sockfd_, buf_, len, flag);
if (cur_cnt == SOCKET_ERROR) {
int err_code = GetLastError();
#if defined(_WIN32)
Log::Fatal("Socket send error (code: %d)", err_code);
#else
Log::Fatal("Socket send error, %s (code: %d)", std::strerror(err_code), err_code);
#endif
}
return cur_cnt;
}
inline int Recv(char *buf_, int len, int flags = 0) {
int cur_cnt = recv(sockfd_, buf_ , len , flags);
if (cur_cnt == SOCKET_ERROR) {
int err_code = GetLastError();
#if defined(_WIN32)
Log::Fatal("Socket recv error (code: %d)", err_code);
#else
Log::Fatal("Socket recv error, %s (code: %d)", std::strerror(err_code), err_code);
#endif
}
return cur_cnt;
}
inline bool IsClosed() {
return sockfd_ == INVALID_SOCKET;
}
inline void Close() {
if (!IsClosed()) {
#if defined(_WIN32)
closesocket(sockfd_);
#else
close(sockfd_);
#endif
sockfd_ = INVALID_SOCKET;
}
}
private:
SOCKET sockfd_;
};
} // namespace LightGBM
#endif // USE_SOCKET
#endif // LIGHTGBM_SRC_NETWORK_SOCKET_WRAPPER_HPP_
src/objective/binary_objective.hpp
+217
View File
@@ -0,0 +1,217 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_OBJECTIVE_BINARY_OBJECTIVE_HPP_
#define LIGHTGBM_SRC_OBJECTIVE_BINARY_OBJECTIVE_HPP_
#include <LightGBM/network.h>
#include <LightGBM/objective_function.h>
#include <string>
#include <algorithm>
#include <cmath>
#include <cstring>
#include <vector>
namespace LightGBM {
/*!
* \brief Objective function for binary classification
*/
class BinaryLogloss: public ObjectiveFunction {
public:
explicit BinaryLogloss(const Config& config,
std::function<bool(label_t)> is_pos = nullptr)
: deterministic_(config.deterministic) {
sigmoid_ = static_cast<double>(config.sigmoid);
if (sigmoid_ <= 0.0) {
Log::Fatal("Sigmoid parameter %f should be greater than zero", sigmoid_);
}
is_unbalance_ = config.is_unbalance;
scale_pos_weight_ = static_cast<double>(config.scale_pos_weight);
if (is_unbalance_ && std::fabs(scale_pos_weight_ - 1.0f) > 1e-6) {
Log::Fatal("Cannot set is_unbalance and scale_pos_weight at the same time");
}
is_pos_ = is_pos;
if (is_pos_ == nullptr) {
is_pos_ = [](label_t label) { return label > 0; };
}
}
explicit BinaryLogloss(const std::vector<std::string>& strs)
: deterministic_(false) {
sigmoid_ = -1;
for (auto str : strs) {
auto tokens = Common::Split(str.c_str(), ':');
if (tokens.size() == 2) {
if (tokens[0] == std::string("sigmoid")) {
Common::Atof(tokens[1].c_str(), &sigmoid_);
}
}
}
if (sigmoid_ <= 0.0) {
Log::Fatal("Sigmoid parameter %f should be greater than zero", sigmoid_);
}
}
~BinaryLogloss() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
num_data_ = num_data;
label_ = metadata.label();
weights_ = metadata.weights();
data_size_t cnt_positive = 0;
data_size_t cnt_negative = 0;
// count for positive and negative samples
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:cnt_positive, cnt_negative)
for (data_size_t i = 0; i < num_data_; ++i) {
if (is_pos_(label_[i])) {
++cnt_positive;
} else {
++cnt_negative;
}
}
num_pos_data_ = cnt_positive;
if (Network::num_machines() > 1) {
cnt_positive = Network::GlobalSyncUpBySum(cnt_positive);
cnt_negative = Network::GlobalSyncUpBySum(cnt_negative);
}
need_train_ = true;
if (cnt_negative == 0 || cnt_positive == 0) {
Log::Warning("Contains only one class");
// not need to boost.
need_train_ = false;
}
Log::Info("Number of positive: %d, number of negative: %d", cnt_positive, cnt_negative);
// use -1 for negative class, and 1 for positive class
label_val_[0] = -1;
label_val_[1] = 1;
// weight for label
label_weights_[0] = 1.0f;
label_weights_[1] = 1.0f;
// if using unbalance, change the labels weight
if (is_unbalance_ && cnt_positive > 0 && cnt_negative > 0) {
if (cnt_positive > cnt_negative) {
label_weights_[1] = 1.0f;
label_weights_[0] = static_cast<double>(cnt_positive) / cnt_negative;
} else {
label_weights_[1] = static_cast<double>(cnt_negative) / cnt_positive;
label_weights_[0] = 1.0f;
}
}
label_weights_[1] *= scale_pos_weight_;
}
void GetGradients(const double* score, score_t* gradients, score_t* hessians) const override {
if (!need_train_) {
return;
}
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
// get label and label weights
const int is_pos = is_pos_(label_[i]);
const int label = label_val_[is_pos];
const double label_weight = label_weights_[is_pos];
// calculate gradients and hessians
const double response = -label * sigmoid_ / (1.0f + std::exp(label * sigmoid_ * score[i]));
const double abs_response = fabs(response);
gradients[i] = static_cast<score_t>(response * label_weight);
hessians[i] = static_cast<score_t>(abs_response * (sigmoid_ - abs_response) * label_weight);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
// get label and label weights
const int is_pos = is_pos_(label_[i]);
const int label = label_val_[is_pos];
const double label_weight = label_weights_[is_pos];
// calculate gradients and hessians
const double response = -label * sigmoid_ / (1.0f + std::exp(label * sigmoid_ * score[i]));
const double abs_response = fabs(response);
gradients[i] = static_cast<score_t>(response * label_weight * weights_[i]);
hessians[i] = static_cast<score_t>(abs_response * (sigmoid_ - abs_response) * label_weight * weights_[i]);
}
}
}
// implement custom average to boost from (if enabled among options)
double BoostFromScore(int) const override {
double suml = 0.0f;
double sumw = 0.0f;
if (weights_ != nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:suml, sumw) if (!deterministic_)
for (data_size_t i = 0; i < num_data_; ++i) {
suml += is_pos_(label_[i]) * weights_[i];
sumw += weights_[i];
}
} else {
sumw = static_cast<double>(num_data_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:suml) if (!deterministic_)
for (data_size_t i = 0; i < num_data_; ++i) {
suml += is_pos_(label_[i]);
}
}
if (Network::num_machines() > 1) {
suml = Network::GlobalSyncUpBySum(suml);
sumw = Network::GlobalSyncUpBySum(sumw);
}
double pavg = suml / sumw;
pavg = std::min(pavg, 1.0 - kEpsilon);
pavg = std::max<double>(pavg, kEpsilon);
double initscore = std::log(pavg / (1.0f - pavg)) / sigmoid_;
Log::Info("[%s:%s]: pavg=%f -> initscore=%f", GetName(), __func__, pavg, initscore);
return initscore;
}
bool ClassNeedTrain(int /*class_id*/) const override {
return need_train_;
}
const char* GetName() const override {
return "binary";
}
void ConvertOutput(const double* input, double* output) const override {
output[0] = 1.0f / (1.0f + std::exp(-sigmoid_ * input[0]));
}
std::string ToString() const override {
std::stringstream str_buf;
str_buf << GetName() << " ";
str_buf << "sigmoid:" << sigmoid_;
return str_buf.str();
}
bool SkipEmptyClass() const override { return true; }
bool NeedAccuratePrediction() const override { return false; }
data_size_t NumPositiveData() const override { return num_pos_data_; }
protected:
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Number of positive samples */
data_size_t num_pos_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief True if using unbalance training */
bool is_unbalance_;
/*! \brief Sigmoid parameter */
double sigmoid_;
/*! \brief Values for positive and negative labels */
int label_val_[2];
/*! \brief Weights for positive and negative labels */
double label_weights_[2];
/*! \brief Weights for data */
const label_t* weights_;
double scale_pos_weight_;
std::function<bool(label_t)> is_pos_;
bool need_train_;
const bool deterministic_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_OBJECTIVE_BINARY_OBJECTIVE_HPP_
src/objective/cuda/cuda_binary_objective.cpp
+58
View File
@@ -0,0 +1,58 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include "cuda_binary_objective.hpp"
#include <string>
#include <vector>
namespace LightGBM {
CUDABinaryLogloss::CUDABinaryLogloss(const Config& config):
CUDAObjectiveInterface<BinaryLogloss>(config), ova_class_id_(-1) {
cuda_label_ = nullptr;
cuda_weights_ = nullptr;
}
CUDABinaryLogloss::CUDABinaryLogloss(const Config& config, const int ova_class_id):
CUDAObjectiveInterface<BinaryLogloss>(config), ova_class_id_(ova_class_id) {
is_pos_ = [ova_class_id](label_t label) { return static_cast<int>(label) == ova_class_id; };
}
CUDABinaryLogloss::CUDABinaryLogloss(const std::vector<std::string>& strs): CUDAObjectiveInterface<BinaryLogloss>(strs) {}
CUDABinaryLogloss::~CUDABinaryLogloss() {}
void CUDABinaryLogloss::Init(const Metadata& metadata, data_size_t num_data) {
CUDAObjectiveInterface<BinaryLogloss>::Init(metadata, num_data);
if (ova_class_id_ == -1) {
cuda_label_ = metadata.cuda_metadata()->cuda_label();
cuda_ova_label_.Clear();
} else {
cuda_ova_label_.Resize(static_cast<size_t>(num_data));
CopyFromHostToCUDADevice<label_t>(cuda_ova_label_.RawData(), metadata.cuda_metadata()->cuda_label(), static_cast<size_t>(num_data), __FILE__, __LINE__);
LaunchResetOVACUDALabelKernel();
cuda_label_ = cuda_ova_label_.RawData();
}
cuda_weights_ = metadata.cuda_metadata()->cuda_weights();
cuda_boost_from_score_.Resize(1);
SetCUDAMemory<double>(cuda_boost_from_score_.RawData(), 0, 1, __FILE__, __LINE__);
cuda_sum_weights_.Resize(1);
SetCUDAMemory<double>(cuda_sum_weights_.RawData(), 0, 1, __FILE__, __LINE__);
if (label_weights_[0] != 1.0f || label_weights_[1] != 1.0f) {
cuda_label_weights_.Resize(2);
CopyFromHostToCUDADevice<double>(cuda_label_weights_.RawData(), label_weights_, 2, __FILE__, __LINE__);
} else {
cuda_label_weights_.Clear();
}
}
} // namespace LightGBM
#endif // USE_CUDA
src/objective/cuda/cuda_binary_objective.cu
+225
View File
@@ -0,0 +1,225 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
* Modifications Copyright(C) 2023 Advanced Micro Devices, Inc. All rights reserved.
*/
#ifdef USE_CUDA
#include "cuda_binary_objective.hpp"
#include <LightGBM/cuda/cuda_rocm_interop.h>
#include <algorithm>
namespace LightGBM {
template <bool USE_WEIGHT>
__global__ void BoostFromScoreKernel_1_BinaryLogloss(const label_t* cuda_labels, const data_size_t num_data, double* out_cuda_sum_labels,
double* out_cuda_sum_weights, const label_t* cuda_weights) {
__shared__ double shared_buffer[WARPSIZE];
const uint32_t mask = 0xffffffff;
const uint32_t warpLane = threadIdx.x % warpSize;
const uint32_t warpID = threadIdx.x / warpSize;
const uint32_t num_warp = blockDim.x / warpSize;
const data_size_t index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
double label_value = 0.0;
double weight_value = 0.0;
if (index < num_data) {
if (USE_WEIGHT) {
const label_t cuda_label = cuda_labels[index];
const double sample_weight = cuda_weights[index];
const label_t label = cuda_label > 0 ? 1 : 0;
label_value = label * sample_weight;
weight_value = sample_weight;
} else {
const label_t cuda_label = cuda_labels[index];
label_value = cuda_label > 0 ? 1 : 0;
}
}
for (uint32_t offset = warpSize / 2; offset >= 1; offset >>= 1) {
label_value += __shfl_down_sync(mask, label_value, offset);
}
if (warpLane == 0) {
shared_buffer[warpID] = label_value;
}
__syncthreads();
if (warpID == 0) {
label_value = (warpLane < num_warp ? shared_buffer[warpLane] : 0);
for (uint32_t offset = warpSize / 2; offset >= 1; offset >>= 1) {
label_value += __shfl_down_sync(mask, label_value, offset);
}
}
__syncthreads();
if (USE_WEIGHT) {
for (uint32_t offset = warpSize / 2; offset >= 1; offset >>= 1) {
weight_value += __shfl_down_sync(mask, weight_value, offset);
}
if (warpLane == 0) {
shared_buffer[warpID] = weight_value;
}
__syncthreads();
if (warpID == 0) {
weight_value = (warpLane < num_warp ? shared_buffer[warpLane] : 0);
for (uint32_t offset = warpSize / 2; offset >= 1; offset >>= 1) {
weight_value += __shfl_down_sync(mask, weight_value, offset);
}
}
__syncthreads();
}
if (threadIdx.x == 0) {
atomicAdd_system(out_cuda_sum_labels, label_value);
if (USE_WEIGHT) {
atomicAdd_system(out_cuda_sum_weights, weight_value);
}
}
}
template <bool USE_WEIGHT>
__global__ void BoostFromScoreKernel_2_BinaryLogloss(double* out_cuda_sum_labels, double* out_cuda_sum_weights,
const data_size_t num_data, const double sigmoid) {
const double suml = *out_cuda_sum_labels;
const double sumw = USE_WEIGHT ? *out_cuda_sum_weights : static_cast<double>(num_data);
double pavg = suml / sumw;
pavg = min(pavg, 1.0 - kEpsilon);
pavg = max(pavg, kEpsilon);
const double init_score = log(pavg / (1.0f - pavg)) / sigmoid;
*out_cuda_sum_weights = pavg;
*out_cuda_sum_labels = init_score;
}
double CUDABinaryLogloss::LaunchCalcInitScoreKernel(const int /*class_id*/) const {
const int num_blocks = (num_data_ + CALC_INIT_SCORE_BLOCK_SIZE_BINARY - 1) / CALC_INIT_SCORE_BLOCK_SIZE_BINARY;
SetCUDAMemory<double>(cuda_boost_from_score_.RawData(), 0, 1, __FILE__, __LINE__);
if (cuda_weights_ == nullptr) {
BoostFromScoreKernel_1_BinaryLogloss<false><<<num_blocks, CALC_INIT_SCORE_BLOCK_SIZE_BINARY>>>
(cuda_label_, num_data_, cuda_boost_from_score_.RawData(), cuda_sum_weights_.RawData(), cuda_weights_);
} else {
BoostFromScoreKernel_1_BinaryLogloss<true><<<num_blocks, CALC_INIT_SCORE_BLOCK_SIZE_BINARY>>>
(cuda_label_, num_data_, cuda_boost_from_score_.RawData(), cuda_sum_weights_.RawData(), cuda_weights_);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
if (cuda_weights_ == nullptr) {
if (nccl_communicator_ == nullptr) {
BoostFromScoreKernel_2_BinaryLogloss<false><<<1, 1>>>(cuda_boost_from_score_.RawData(), cuda_sum_weights_.RawData(), num_data_, sigmoid_);
} else {
NCCLAllReduce<double>(cuda_boost_from_score_.RawData(), cuda_boost_from_score_.RawData(), 1, ncclFloat64, ncclSum, nccl_communicator_);
const data_size_t global_num_data = NCCLAllReduce<data_size_t>(num_data_, ncclInt32, ncclSum, nccl_communicator_);
BoostFromScoreKernel_2_BinaryLogloss<false><<<1, 1>>>(cuda_boost_from_score_.RawData(), cuda_sum_weights_.RawData(), global_num_data, sigmoid_);
}
} else {
if (nccl_communicator_ == nullptr) {
BoostFromScoreKernel_2_BinaryLogloss<true><<<1, 1>>>(cuda_boost_from_score_.RawData(), cuda_sum_weights_.RawData(), num_data_, sigmoid_);
} else {
NCCLAllReduce<double>(cuda_boost_from_score_.RawData(), cuda_boost_from_score_.RawData(), 1, ncclFloat64, ncclSum, nccl_communicator_);
NCCLAllReduce<double>(cuda_sum_weights_.RawData(), cuda_sum_weights_.RawData(), 1, ncclFloat64, ncclSum, nccl_communicator_);
BoostFromScoreKernel_2_BinaryLogloss<true><<<1, 1>>>(cuda_boost_from_score_.RawData(), cuda_sum_weights_.RawData(), num_data_, sigmoid_);
}
}
SynchronizeCUDADevice(__FILE__, __LINE__);
double boost_from_score = 0.0f;
CopyFromCUDADeviceToHost<double>(&boost_from_score, cuda_boost_from_score_.RawData(), 1, __FILE__, __LINE__);
double pavg = 0.0f;
CopyFromCUDADeviceToHost<double>(&pavg, cuda_sum_weights_.RawData(), 1, __FILE__, __LINE__);
// for some test cases in test_utilities.py which check the log output
Log::Info("[%s:%s]: pavg=%f -> initscore=%f", GetName(), "BoostFromScore", pavg, boost_from_score);
return boost_from_score;
}
template <bool USE_LABEL_WEIGHT, bool USE_WEIGHT>
__global__ void GetGradientsKernel_BinaryLogloss(const double* cuda_scores, const label_t* cuda_labels,
const double* cuda_label_weights, const label_t* cuda_weights,
const double sigmoid, const data_size_t num_data,
score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
const data_size_t data_index = static_cast<data_size_t>(blockDim.x * blockIdx.x + threadIdx.x);
if (data_index < num_data) {
const label_t cuda_label = static_cast<int>(cuda_labels[data_index]);
const int label = cuda_label > 0 ? 1 : -1;
const double response = -label * sigmoid / (1.0f + exp(label * sigmoid * cuda_scores[data_index]));
const double abs_response = fabs(response);
if (!USE_WEIGHT) {
if (USE_LABEL_WEIGHT) {
const double label_weight = cuda_label_weights[label];
cuda_out_gradients[data_index] = static_cast<score_t>(response * label_weight);
cuda_out_hessians[data_index] = static_cast<score_t>(abs_response * (sigmoid - abs_response) * label_weight);
} else {
cuda_out_gradients[data_index] = static_cast<score_t>(response);
cuda_out_hessians[data_index] = static_cast<score_t>(abs_response * (sigmoid - abs_response));
}
} else {
const double sample_weight = cuda_weights[data_index];
if (USE_LABEL_WEIGHT) {
const double label_weight = cuda_label_weights[label];
cuda_out_gradients[data_index] = static_cast<score_t>(response * label_weight * sample_weight);
cuda_out_hessians[data_index] = static_cast<score_t>(abs_response * (sigmoid - abs_response) * label_weight * sample_weight);
} else {
cuda_out_gradients[data_index] = static_cast<score_t>(response * sample_weight);
cuda_out_hessians[data_index] = static_cast<score_t>(abs_response * (sigmoid - abs_response) * sample_weight);
}
}
}
}
#define GetGradientsKernel_BinaryLogloss_ARGS \
scores, \
cuda_label_, \
cuda_label_weights_.RawData(), \
cuda_weights_, \
sigmoid_, \
num_data_, \
gradients, \
hessians
void CUDABinaryLogloss::LaunchGetGradientsKernel(const double* scores, score_t* gradients, score_t* hessians) const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_BINARY - 1) / GET_GRADIENTS_BLOCK_SIZE_BINARY;
if (cuda_label_weights_.Size() == 0) {
if (cuda_weights_ == nullptr) {
GetGradientsKernel_BinaryLogloss<false, false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_BINARY>>>(GetGradientsKernel_BinaryLogloss_ARGS);
} else {
GetGradientsKernel_BinaryLogloss<false, true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_BINARY>>>(GetGradientsKernel_BinaryLogloss_ARGS);
}
} else {
if (cuda_weights_ == nullptr) {
GetGradientsKernel_BinaryLogloss<true, false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_BINARY>>>(GetGradientsKernel_BinaryLogloss_ARGS);
} else {
GetGradientsKernel_BinaryLogloss<true, true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_BINARY>>>(GetGradientsKernel_BinaryLogloss_ARGS);
}
}
}
#undef GetGradientsKernel_BinaryLogloss_ARGS
__global__ void ConvertOutputCUDAKernel_BinaryLogloss(const double sigmoid, const data_size_t num_data, const double* input, double* output) {
const data_size_t data_index = static_cast<data_size_t>(blockIdx.x * blockDim.x + threadIdx.x);
if (data_index < num_data) {
output[data_index] = 1.0f / (1.0f + exp(-sigmoid * input[data_index]));
}
}
const double* CUDABinaryLogloss::LaunchConvertOutputCUDAKernel(const data_size_t num_data, const double* input, double* output) const {
const int num_blocks = (num_data + GET_GRADIENTS_BLOCK_SIZE_BINARY - 1) / GET_GRADIENTS_BLOCK_SIZE_BINARY;
ConvertOutputCUDAKernel_BinaryLogloss<<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_BINARY>>>(sigmoid_, num_data, input, output);
return output;
}
__global__ void ResetOVACUDALabelKernel(
const int ova_class_id,
const data_size_t num_data,
label_t* cuda_label) {
const data_size_t data_index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
if (data_index < num_data) {
const int int_label = static_cast<int>(cuda_label[data_index]);
cuda_label[data_index] = (int_label == ova_class_id ? 1.0f : 0.0f);
}
}
void CUDABinaryLogloss::LaunchResetOVACUDALabelKernel() const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_BINARY - 1) / GET_GRADIENTS_BLOCK_SIZE_BINARY;
ResetOVACUDALabelKernel<<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_BINARY>>>(ova_class_id_, num_data_, cuda_ova_label_.RawData());
}
} // namespace LightGBM
#endif // USE_CUDA
src/objective/cuda/cuda_binary_objective.hpp
+64
View File
@@ -0,0 +1,64 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_BINARY_OBJECTIVE_HPP_
#define LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_BINARY_OBJECTIVE_HPP_
#ifdef USE_CUDA
#define GET_GRADIENTS_BLOCK_SIZE_BINARY (1024)
#define CALC_INIT_SCORE_BLOCK_SIZE_BINARY (1024)
#include <LightGBM/cuda/cuda_objective_function.hpp>
#include <string>
#include <vector>
#include "../binary_objective.hpp"
namespace LightGBM {
class CUDABinaryLogloss : public CUDAObjectiveInterface<BinaryLogloss> {
public:
explicit CUDABinaryLogloss(const Config& config);
explicit CUDABinaryLogloss(const Config& config, const int ova_class_id);
explicit CUDABinaryLogloss(const std::vector<std::string>& strs);
~CUDABinaryLogloss();
void Init(const Metadata& metadata, data_size_t num_data) override;
bool NeedConvertOutputCUDA() const override { return true; }
private:
void LaunchGetGradientsKernel(const double* scores, score_t* gradients, score_t* hessians) const override;
double LaunchCalcInitScoreKernel(const int class_id) const override;
const double* LaunchConvertOutputCUDAKernel(const data_size_t num_data, const double* input, double* output) const override;
void LaunchResetOVACUDALabelKernel() const;
// CUDA memory, held by other objects
const label_t* cuda_label_;
CUDAVector<label_t> cuda_ova_label_;
const label_t* cuda_weights_;
// CUDA memory, held by this object
CUDAVector<double> cuda_boost_from_score_;
CUDAVector<double> cuda_sum_weights_;
CUDAVector<double> cuda_label_weights_;
const int ova_class_id_ = -1;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_BINARY_OBJECTIVE_HPP_
src/objective/cuda/cuda_multiclass_objective.cpp
+63
View File
@@ -0,0 +1,63 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_CUDA
#include "cuda_multiclass_objective.hpp"
#include <string>
#include <vector>
namespace LightGBM {
CUDAMulticlassSoftmax::CUDAMulticlassSoftmax(const Config& config): CUDAObjectiveInterface<MulticlassSoftmax>(config) {}
CUDAMulticlassSoftmax::CUDAMulticlassSoftmax(const std::vector<std::string>& strs): CUDAObjectiveInterface<MulticlassSoftmax>(strs) {}
CUDAMulticlassSoftmax::~CUDAMulticlassSoftmax() {}
void CUDAMulticlassSoftmax::Init(const Metadata& metadata, data_size_t num_data) {
CUDAObjectiveInterface<MulticlassSoftmax>::Init(metadata, num_data);
cuda_softmax_buffer_.Resize(static_cast<size_t>(num_data) * static_cast<size_t>(num_class_));
SynchronizeCUDADevice(__FILE__, __LINE__);
}
CUDAMulticlassOVA::CUDAMulticlassOVA(const Config& config): CUDAObjectiveInterface<MulticlassOVA>(config) {
for (int i = 0; i < num_class_; ++i) {
cuda_binary_loss_.emplace_back(new CUDABinaryLogloss(config, i));
}
}
CUDAMulticlassOVA::CUDAMulticlassOVA(const std::vector<std::string>& strs): CUDAObjectiveInterface<MulticlassOVA>(strs) {}
CUDAMulticlassOVA::~CUDAMulticlassOVA() {}
void CUDAMulticlassOVA::Init(const Metadata& metadata, data_size_t num_data) {
MulticlassOVA::Init(metadata, num_data);
for (int i = 0; i < num_class_; ++i) {
cuda_binary_loss_[i]->Init(metadata, num_data);
}
}
void CUDAMulticlassOVA::GetGradients(const double* score, score_t* gradients, score_t* hessians) const {
for (int i = 0; i < num_class_; ++i) {
int64_t offset = static_cast<int64_t>(num_data_) * i;
cuda_binary_loss_[i]->GetGradients(score + offset, gradients + offset, hessians + offset);
}
}
const double* CUDAMulticlassOVA::ConvertOutputCUDA(const data_size_t num_data, const double* input, double* output) const {
for (int i = 0; i < num_class_; ++i) {
cuda_binary_loss_[i]->ConvertOutputCUDA(num_data, input + i * num_data, output + i * num_data);
}
return output;
}
} // namespace LightGBM
#endif // USE_CUDA
src/objective/cuda/cuda_multiclass_objective.cu
+109
View File
@@ -0,0 +1,109 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifdef USE_CUDA
#include <algorithm>
#include "cuda_multiclass_objective.hpp"
namespace LightGBM {
__device__ void SoftmaxCUDA(double* softmax_buffer, int len) {
double wmax = softmax_buffer[0];
for (int i = 1; i < len; ++i) {
wmax = max(softmax_buffer[i], wmax);
}
double wsum = 0.0f;
for (int i = 0; i < len; ++i) {
softmax_buffer[i] = exp(softmax_buffer[i] - wmax);
wsum += softmax_buffer[i];
}
for (int i = 0; i < len; ++i) {
softmax_buffer[i] /= static_cast<double>(wsum);
}
}
template <bool USE_WEIGHT>
__global__ void GetGradientsKernel_MulticlassSoftmax(
const double* cuda_scores, const label_t* cuda_labels, const label_t* cuda_weights,
const double factor, const int num_class, const data_size_t num_data,
double* cuda_softmax_buffer, score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
const data_size_t data_index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
if (data_index < num_data) {
const data_size_t offset = data_index * num_class;
double* softmax_result = cuda_softmax_buffer + offset;
for (int k = 0; k < num_class; ++k) {
softmax_result[k] = cuda_scores[k * num_data + data_index];
}
SoftmaxCUDA(softmax_result, num_class);
if (!USE_WEIGHT) {
for (int k = 0; k < num_class; ++k) {
const double p = softmax_result[k];
size_t idx = static_cast<size_t>(num_data) * k + data_index;
if (static_cast<int>(cuda_labels[data_index]) == k) {
cuda_out_gradients[idx] = static_cast<score_t>(p - 1.0f);
} else {
cuda_out_gradients[idx] = static_cast<score_t>(p);
}
cuda_out_hessians[idx] = static_cast<score_t>(factor * p * (1.0f - p));
}
} else {
for (int k = 0; k < num_class; ++k) {
const double p = softmax_result[k];
const double weight = cuda_weights[data_index];
size_t idx = static_cast<size_t>(num_data) * k + data_index;
if (static_cast<int>(cuda_labels[data_index]) == k) {
cuda_out_gradients[idx] = static_cast<score_t>((p - 1.0f) * weight);
} else {
cuda_out_gradients[idx] = static_cast<score_t>(p * weight);
}
cuda_out_hessians[idx] = static_cast<score_t>((factor * p * (1.0f - p)) * weight);
}
}
}
}
void CUDAMulticlassSoftmax::LaunchGetGradientsKernel(const double* scores, score_t* gradients, score_t* hessians) const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_MULTICLASS - 1) / GET_GRADIENTS_BLOCK_SIZE_MULTICLASS;
if (cuda_weights_ == nullptr) {
GetGradientsKernel_MulticlassSoftmax<false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_MULTICLASS>>>(
scores, cuda_labels_, cuda_weights_, factor_, num_class_, num_data_,
cuda_softmax_buffer_.RawData(), gradients, hessians);
} else {
GetGradientsKernel_MulticlassSoftmax<true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_MULTICLASS>>>(
scores, cuda_labels_, cuda_weights_, factor_, num_class_, num_data_,
cuda_softmax_buffer_.RawData(), gradients, hessians);
}
}
__global__ void ConvertOutputCUDAKernel_MulticlassSoftmax(
const int num_class, const data_size_t num_data, const double* input, double* cuda_softmax_buffer, double* output) {
const data_size_t data_index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
if (data_index < num_data) {
const data_size_t offset = data_index * num_class;
double* cuda_softmax_buffer_ptr = cuda_softmax_buffer + offset;
for (int class_index = 0; class_index < num_class; ++class_index) {
cuda_softmax_buffer_ptr[class_index] = input[class_index * num_data + data_index];
}
SoftmaxCUDA(cuda_softmax_buffer_ptr, num_class);
for (int class_index = 0; class_index < num_class; ++class_index) {
output[class_index * num_data + data_index] = cuda_softmax_buffer_ptr[class_index];
}
}
}
const double* CUDAMulticlassSoftmax::LaunchConvertOutputCUDAKernel(
const data_size_t num_data, const double* input, double* output) const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_MULTICLASS - 1) / GET_GRADIENTS_BLOCK_SIZE_MULTICLASS;
ConvertOutputCUDAKernel_MulticlassSoftmax<<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_MULTICLASS>>>(
num_class_, num_data, input, cuda_softmax_buffer_.RawData(), output);
return output;
}
} // namespace LightGBM
#endif // USE_CUDA
src/objective/cuda/cuda_multiclass_objective.hpp
+79
View File
@@ -0,0 +1,79 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_MULTICLASS_OBJECTIVE_HPP_
#define LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_MULTICLASS_OBJECTIVE_HPP_
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_objective_function.hpp>
#include <memory>
#include <string>
#include <vector>
#include "cuda_binary_objective.hpp"
#include "../multiclass_objective.hpp"
#define GET_GRADIENTS_BLOCK_SIZE_MULTICLASS (1024)
namespace LightGBM {
class CUDAMulticlassSoftmax: public CUDAObjectiveInterface<MulticlassSoftmax> {
public:
explicit CUDAMulticlassSoftmax(const Config& config);
explicit CUDAMulticlassSoftmax(const std::vector<std::string>& strs);
~CUDAMulticlassSoftmax();
void Init(const Metadata& metadata, data_size_t num_data) override;
private:
void LaunchGetGradientsKernel(const double* scores, score_t* gradients, score_t* hessians) const;
const double* LaunchConvertOutputCUDAKernel(const data_size_t num_data, const double* input, double* output) const;
// CUDA memory, held by this object
CUDAVector<double> cuda_softmax_buffer_;
};
class CUDAMulticlassOVA: public CUDAObjectiveInterface<MulticlassOVA> {
public:
explicit CUDAMulticlassOVA(const Config& config);
explicit CUDAMulticlassOVA(const std::vector<std::string>& strs);
void Init(const Metadata& metadata, data_size_t num_data) override;
void GetGradients(const double* score, score_t* gradients, score_t* hessians) const override;
const double* ConvertOutputCUDA(const data_size_t num_data, const double* input, double* output) const override;
double BoostFromScore(int class_id) const override {
return cuda_binary_loss_[class_id]->BoostFromScore(0);
}
bool ClassNeedTrain(int class_id) const override {
return cuda_binary_loss_[class_id]->ClassNeedTrain(0);
}
~CUDAMulticlassOVA();
bool IsCUDAObjective() const override { return true; }
private:
void LaunchGetGradientsKernel(const double* /*scores*/, score_t* /*gradients*/, score_t* /*hessians*/) const {}
std::vector<std::unique_ptr<CUDABinaryLogloss>> cuda_binary_loss_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_MULTICLASS_OBJECTIVE_HPP_
src/objective/cuda/cuda_rank_objective.cpp
+68
View File
@@ -0,0 +1,68 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include <string>
#include <vector>
#include "cuda_rank_objective.hpp"
namespace LightGBM {
CUDALambdarankNDCG::CUDALambdarankNDCG(const Config& config): CUDALambdaRankObjectiveInterface<LambdarankNDCG>(config) {}
CUDALambdarankNDCG::CUDALambdarankNDCG(const std::vector<std::string>& strs): CUDALambdaRankObjectiveInterface<LambdarankNDCG>(strs) {}
CUDALambdarankNDCG::~CUDALambdarankNDCG() {}
void CUDALambdarankNDCG::Init(const Metadata& metadata, data_size_t num_data) {
CUDALambdaRankObjectiveInterface<LambdarankNDCG>::Init(metadata, num_data);
cuda_inverse_max_dcgs_.Resize(this->inverse_max_dcgs_.size());
CopyFromHostToCUDADevice(cuda_inverse_max_dcgs_.RawData(), this->inverse_max_dcgs_.data(), this->inverse_max_dcgs_.size(), __FILE__, __LINE__);
cuda_label_gain_.Resize(this->label_gain_.size());
CopyFromHostToCUDADevice(cuda_label_gain_.RawData(), this->label_gain_.data(), this->label_gain_.size(), __FILE__, __LINE__);
}
CUDARankXENDCG::CUDARankXENDCG(const Config& config): CUDALambdaRankObjectiveInterface<RankXENDCG>(config) {}
CUDARankXENDCG::CUDARankXENDCG(const std::vector<std::string>& strs): CUDALambdaRankObjectiveInterface<RankXENDCG>(strs) {}
CUDARankXENDCG::~CUDARankXENDCG() {}
void CUDARankXENDCG::Init(const Metadata& metadata, data_size_t num_data) {
CUDALambdaRankObjectiveInterface<RankXENDCG>::Init(metadata, num_data);
for (data_size_t i = 0; i < num_queries_; ++i) {
rands_.emplace_back(seed_ + i);
}
item_rands_.resize(num_data, 0.0f);
cuda_item_rands_.Resize(static_cast<size_t>(num_data));
if (max_items_in_query_aligned_ >= 2048) {
cuda_params_buffer_.Resize(static_cast<size_t>(num_data_));
}
}
void CUDARankXENDCG::GenerateItemRands() const {
const int num_threads = OMP_NUM_THREADS();
OMP_INIT_EX();
#pragma omp parallel for schedule(static) num_threads(num_threads)
for (data_size_t i = 0; i < num_queries_; ++i) {
OMP_LOOP_EX_BEGIN();
const data_size_t start = query_boundaries_[i];
const data_size_t end = query_boundaries_[i + 1];
for (data_size_t j = start; j < end; ++j) {
item_rands_[j] = rands_[i].NextFloat();
}
OMP_LOOP_EX_END();
}
OMP_THROW_EX();
}
} // namespace LightGBM
#endif // USE_CUDA
src/objective/cuda/cuda_rank_objective.cu
+666
View File
@@ -0,0 +1,666 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
* Modifications Copyright(C) 2023 Advanced Micro Devices, Inc. All rights reserved.
*/
#ifdef USE_CUDA
#include "cuda_rank_objective.hpp"
#include <LightGBM/cuda/cuda_algorithms.hpp>
#include <random>
#include <algorithm>
namespace LightGBM {
template <bool MAX_ITEM_GREATER_THAN_1024, data_size_t NUM_RANK_LABEL>
__global__ void GetGradientsKernel_LambdarankNDCG(const double* cuda_scores, const label_t* cuda_labels, const data_size_t num_data,
const data_size_t num_queries, const data_size_t* cuda_query_boundaries, const double* cuda_inverse_max_dcgs,
const bool norm, const double sigmoid, const int truncation_level, const double* cuda_label_gain, const data_size_t num_rank_label,
score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
__shared__ score_t shared_scores[MAX_ITEM_GREATER_THAN_1024 ? 2048 : 1024];
__shared__ uint16_t shared_indices[MAX_ITEM_GREATER_THAN_1024 ? 2048 : 1024];
__shared__ score_t shared_lambdas[MAX_ITEM_GREATER_THAN_1024 ? 2048 : 1024];
__shared__ score_t shared_hessians[MAX_ITEM_GREATER_THAN_1024 ? 2048 : 1024];
__shared__ double shared_label_gain[NUM_RANK_LABEL > 1024 ? 1 : NUM_RANK_LABEL];
const double* label_gain_ptr = nullptr;
if (NUM_RANK_LABEL <= 1024) {
for (uint32_t i = threadIdx.x; i < num_rank_label; i += blockDim.x) {
shared_label_gain[i] = cuda_label_gain[i];
}
__syncthreads();
label_gain_ptr = shared_label_gain;
} else {
label_gain_ptr = cuda_label_gain;
}
const data_size_t query_index_start = static_cast<data_size_t>(blockIdx.x) * NUM_QUERY_PER_BLOCK;
const data_size_t query_index_end = min(query_index_start + NUM_QUERY_PER_BLOCK, num_queries);
for (data_size_t query_index = query_index_start; query_index < query_index_end; ++query_index) {
const double inverse_max_dcg = cuda_inverse_max_dcgs[query_index];
const data_size_t query_start = cuda_query_boundaries[query_index];
const data_size_t query_end = cuda_query_boundaries[query_index + 1];
const data_size_t query_item_count = query_end - query_start;
const double* cuda_scores_pointer = cuda_scores + query_start;
score_t* cuda_out_gradients_pointer = cuda_out_gradients + query_start;
score_t* cuda_out_hessians_pointer = cuda_out_hessians + query_start;
const label_t* cuda_label_pointer = cuda_labels + query_start;
if (threadIdx.x < query_item_count) {
shared_scores[threadIdx.x] = cuda_scores_pointer[threadIdx.x];
shared_indices[threadIdx.x] = static_cast<uint16_t>(threadIdx.x);
shared_lambdas[threadIdx.x] = 0.0f;
shared_hessians[threadIdx.x] = 0.0f;
} else {
shared_scores[threadIdx.x] = kMinScore;
shared_indices[threadIdx.x] = static_cast<uint16_t>(threadIdx.x);
}
if (MAX_ITEM_GREATER_THAN_1024) {
if (query_item_count > 1024) {
const unsigned int threadIdx_x_plus_1024 = threadIdx.x + 1024;
if (threadIdx_x_plus_1024 < query_item_count) {
shared_scores[threadIdx_x_plus_1024] = cuda_scores_pointer[threadIdx_x_plus_1024];
shared_indices[threadIdx_x_plus_1024] = static_cast<uint16_t>(threadIdx_x_plus_1024);
shared_lambdas[threadIdx_x_plus_1024] = 0.0f;
shared_hessians[threadIdx_x_plus_1024] = 0.0f;
} else {
shared_scores[threadIdx_x_plus_1024] = kMinScore;
shared_indices[threadIdx_x_plus_1024] = static_cast<uint16_t>(threadIdx_x_plus_1024);
}
}
}
__syncthreads();
if (MAX_ITEM_GREATER_THAN_1024) {
if (query_item_count > 1024) {
BitonicArgSort_2048<score_t, uint16_t, false>(shared_scores, shared_indices);
} else {
BitonicArgSort_1024<score_t, uint16_t, false>(shared_scores, shared_indices, static_cast<uint16_t>(query_item_count));
}
} else {
BitonicArgSort_1024<score_t, uint16_t, false>(shared_scores, shared_indices, static_cast<uint16_t>(query_item_count));
}
__syncthreads();
// get best and worst score
const double best_score = shared_scores[shared_indices[0]];
data_size_t worst_idx = query_item_count - 1;
if (worst_idx > 0 && shared_scores[shared_indices[worst_idx]] == kMinScore) {
worst_idx -= 1;
}
const double worst_score = shared_scores[shared_indices[worst_idx]];
__shared__ double sum_lambdas;
if (threadIdx.x == 0) {
sum_lambdas = 0.0f;
}
__syncthreads();
// start accumulate lambdas by pairs that contain at least one document above truncation level
const data_size_t num_items_i = min(query_item_count - 1, truncation_level);
const data_size_t num_j_per_i = query_item_count - 1;
const data_size_t s = num_j_per_i - num_items_i + 1;
const data_size_t num_pairs = (num_j_per_i + s) * num_items_i / 2;
double thread_sum_lambdas = 0.0f;
for (data_size_t pair_index = static_cast<data_size_t>(threadIdx.x); pair_index < num_pairs; pair_index += static_cast<data_size_t>(blockDim.x)) {
const double square = 2 * static_cast<double>(pair_index) + s * s - s;
const double sqrt_result = floor(sqrt(square));
const data_size_t row_index = static_cast<data_size_t>(floor(sqrt(square - sqrt_result)) + 1 - s);
const data_size_t i = num_items_i - 1 - row_index;
const data_size_t j = num_j_per_i - (pair_index - (2 * s + row_index - 1) * row_index / 2);
if (cuda_label_pointer[shared_indices[i]] != cuda_label_pointer[shared_indices[j]] && shared_scores[shared_indices[j]] != kMinScore) {
data_size_t high_rank, low_rank;
if (cuda_label_pointer[shared_indices[i]] > cuda_label_pointer[shared_indices[j]]) {
high_rank = i;
low_rank = j;
} else {
high_rank = j;
low_rank = i;
}
const data_size_t high = shared_indices[high_rank];
const int high_label = static_cast<int>(cuda_label_pointer[high]);
const double high_score = shared_scores[high];
const double high_label_gain = label_gain_ptr[high_label];
const double high_discount = log2(2.0f + high_rank);
const data_size_t low = shared_indices[low_rank];
const int low_label = static_cast<int>(cuda_label_pointer[low]);
const double low_score = shared_scores[low];
const double low_label_gain = label_gain_ptr[low_label];
const double low_discount = log2(2.0f + low_rank);
const double delta_score = high_score - low_score;
// get dcg gap
const double dcg_gap = high_label_gain - low_label_gain;
// get discount of this pair
const double paired_discount = fabs(high_discount - low_discount);
// get delta NDCG
double delta_pair_NDCG = dcg_gap * paired_discount * inverse_max_dcg;
// regular the delta_pair_NDCG by score distance
if (norm && best_score != worst_score) {
delta_pair_NDCG /= (0.01f + fabs(delta_score));
}
// calculate lambda for this pair
double p_lambda = 1.0f / (1.0f + exp(sigmoid * delta_score));
double p_hessian = p_lambda * (1.0f - p_lambda);
// update
p_lambda *= -sigmoid * delta_pair_NDCG;
p_hessian *= sigmoid * sigmoid * delta_pair_NDCG;
atomicAdd_block(shared_lambdas + low, -static_cast<score_t>(p_lambda));
atomicAdd_block(shared_hessians + low, static_cast<score_t>(p_hessian));
atomicAdd_block(shared_lambdas + high, static_cast<score_t>(p_lambda));
atomicAdd_block(shared_hessians + high, static_cast<score_t>(p_hessian));
// lambda is negative, so use minus to accumulate
thread_sum_lambdas -= 2 * p_lambda;
}
}
atomicAdd_block(&sum_lambdas, thread_sum_lambdas);
__syncthreads();
if (norm && sum_lambdas > 0) {
const double norm_factor = log2(1 + sum_lambdas) / sum_lambdas;
if (threadIdx.x < static_cast<unsigned int>(query_item_count)) {
cuda_out_gradients_pointer[threadIdx.x] = static_cast<score_t>(shared_lambdas[threadIdx.x] * norm_factor);
cuda_out_hessians_pointer[threadIdx.x] = static_cast<score_t>(shared_hessians[threadIdx.x] * norm_factor);
}
if (MAX_ITEM_GREATER_THAN_1024) {
if (query_item_count > 1024) {
const unsigned int threadIdx_x_plus_1024 = threadIdx.x + 1024;
if (threadIdx_x_plus_1024 < static_cast<unsigned int>(query_item_count)) {
cuda_out_gradients_pointer[threadIdx_x_plus_1024] = static_cast<score_t>(shared_lambdas[threadIdx_x_plus_1024] * norm_factor);
cuda_out_hessians_pointer[threadIdx_x_plus_1024] = static_cast<score_t>(shared_hessians[threadIdx_x_plus_1024] * norm_factor);
}
}
}
} else {
if (threadIdx.x < static_cast<unsigned int>(query_item_count)) {
cuda_out_gradients_pointer[threadIdx.x] = static_cast<score_t>(shared_lambdas[threadIdx.x]);
cuda_out_hessians_pointer[threadIdx.x] = static_cast<score_t>(shared_hessians[threadIdx.x]);
}
if (MAX_ITEM_GREATER_THAN_1024) {
if (query_item_count > 1024) {
const unsigned int threadIdx_x_plus_1024 = threadIdx.x + 1024;
if (threadIdx_x_plus_1024 < static_cast<unsigned int>(query_item_count)) {
cuda_out_gradients_pointer[threadIdx_x_plus_1024] = static_cast<score_t>(shared_lambdas[threadIdx_x_plus_1024]);
cuda_out_hessians_pointer[threadIdx_x_plus_1024] = static_cast<score_t>(shared_hessians[threadIdx_x_plus_1024]);
}
}
}
}
__syncthreads();
}
}
template <data_size_t NUM_RANK_LABEL>
__global__ void GetGradientsKernel_LambdarankNDCG_Sorted(
const double* cuda_scores, const int* cuda_item_indices_buffer, const label_t* cuda_labels, const data_size_t num_data,
const data_size_t num_queries, const data_size_t* cuda_query_boundaries, const double* cuda_inverse_max_dcgs,
const bool norm, const double sigmoid, const int truncation_level, const double* cuda_label_gain, const data_size_t num_rank_label,
score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
__shared__ double shared_label_gain[NUM_RANK_LABEL > 1024 ? 1 : NUM_RANK_LABEL];
const double* label_gain_ptr = nullptr;
if (NUM_RANK_LABEL <= 1024) {
for (uint32_t i = threadIdx.x; i < static_cast<uint32_t>(num_rank_label); i += blockDim.x) {
shared_label_gain[i] = cuda_label_gain[i];
}
__syncthreads();
label_gain_ptr = shared_label_gain;
} else {
label_gain_ptr = cuda_label_gain;
}
const data_size_t query_index_start = static_cast<data_size_t>(blockIdx.x) * NUM_QUERY_PER_BLOCK;
const data_size_t query_index_end = min(query_index_start + NUM_QUERY_PER_BLOCK, num_queries);
for (data_size_t query_index = query_index_start; query_index < query_index_end; ++query_index) {
const double inverse_max_dcg = cuda_inverse_max_dcgs[query_index];
const data_size_t query_start = cuda_query_boundaries[query_index];
const data_size_t query_end = cuda_query_boundaries[query_index + 1];
const data_size_t query_item_count = query_end - query_start;
const double* cuda_scores_pointer = cuda_scores + query_start;
const int* cuda_item_indices_buffer_pointer = cuda_item_indices_buffer + query_start;
score_t* cuda_out_gradients_pointer = cuda_out_gradients + query_start;
score_t* cuda_out_hessians_pointer = cuda_out_hessians + query_start;
const label_t* cuda_label_pointer = cuda_labels + query_start;
// get best and worst score
const double best_score = cuda_scores_pointer[cuda_item_indices_buffer_pointer[0]];
data_size_t worst_idx = query_item_count - 1;
if (worst_idx > 0 && cuda_scores_pointer[cuda_item_indices_buffer_pointer[worst_idx]] == kMinScore) {
worst_idx -= 1;
}
const double worst_score = cuda_scores_pointer[cuda_item_indices_buffer_pointer[worst_idx]];
__shared__ double sum_lambdas;
if (threadIdx.x == 0) {
sum_lambdas = 0.0f;
}
for (int item_index = static_cast<int>(threadIdx.x); item_index < query_item_count; item_index += static_cast<int>(blockDim.x)) {
cuda_out_gradients_pointer[item_index] = 0.0f;
cuda_out_hessians_pointer[item_index] = 0.0f;
}
__syncthreads();
// start accumulate lambdas by pairs that contain at least one document above truncation level
const data_size_t num_items_i = min(query_item_count - 1, truncation_level);
const data_size_t num_j_per_i = query_item_count - 1;
const data_size_t s = num_j_per_i - num_items_i + 1;
const data_size_t num_pairs = (num_j_per_i + s) * num_items_i / 2;
double thread_sum_lambdas = 0.0f;
for (data_size_t pair_index = static_cast<data_size_t>(threadIdx.x); pair_index < num_pairs; pair_index += static_cast<data_size_t>(blockDim.x)) {
const double square = 2 * static_cast<double>(pair_index) + s * s - s;
const double sqrt_result = floor(sqrt(square));
const data_size_t row_index = static_cast<data_size_t>(floor(sqrt(square - sqrt_result)) + 1 - s);
const data_size_t i = num_items_i - 1 - row_index;
const data_size_t j = num_j_per_i - (pair_index - (2 * s + row_index - 1) * row_index / 2);
if (j > i) {
// skip pairs with the same labels
if (cuda_label_pointer[cuda_item_indices_buffer_pointer[i]] != cuda_label_pointer[cuda_item_indices_buffer_pointer[j]] && cuda_scores_pointer[cuda_item_indices_buffer_pointer[j]] != kMinScore) {
data_size_t high_rank, low_rank;
if (cuda_label_pointer[cuda_item_indices_buffer_pointer[i]] > cuda_label_pointer[cuda_item_indices_buffer_pointer[j]]) {
high_rank = i;
low_rank = j;
} else {
high_rank = j;
low_rank = i;
}
const data_size_t high = cuda_item_indices_buffer_pointer[high_rank];
const int high_label = static_cast<int>(cuda_label_pointer[high]);
const double high_score = cuda_scores_pointer[high];
const double high_label_gain = label_gain_ptr[high_label];
const double high_discount = log2(2.0f + high_rank);
const data_size_t low = cuda_item_indices_buffer_pointer[low_rank];
const int low_label = static_cast<int>(cuda_label_pointer[low]);
const double low_score = cuda_scores_pointer[low];
const double low_label_gain = label_gain_ptr[low_label];
const double low_discount = log2(2.0f + low_rank);
const double delta_score = high_score - low_score;
// get dcg gap
const double dcg_gap = high_label_gain - low_label_gain;
// get discount of this pair
const double paired_discount = fabs(high_discount - low_discount);
// get delta NDCG
double delta_pair_NDCG = dcg_gap * paired_discount * inverse_max_dcg;
// regular the delta_pair_NDCG by score distance
if (norm && best_score != worst_score) {
delta_pair_NDCG /= (0.01f + fabs(delta_score));
}
// calculate lambda for this pair
double p_lambda = 1.0f / (1.0f + exp(sigmoid * delta_score));
double p_hessian = p_lambda * (1.0f - p_lambda);
// update
p_lambda *= -sigmoid * delta_pair_NDCG;
p_hessian *= sigmoid * sigmoid * delta_pair_NDCG;
atomicAdd_block(cuda_out_gradients_pointer + low, -static_cast<score_t>(p_lambda));
atomicAdd_block(cuda_out_hessians_pointer + low, static_cast<score_t>(p_hessian));
atomicAdd_block(cuda_out_gradients_pointer + high, static_cast<score_t>(p_lambda));
atomicAdd_block(cuda_out_hessians_pointer + high, static_cast<score_t>(p_hessian));
// lambda is negative, so use minus to accumulate
thread_sum_lambdas -= 2 * p_lambda;
}
}
}
atomicAdd_block(&sum_lambdas, thread_sum_lambdas);
__syncthreads();
if (norm && sum_lambdas > 0) {
const double norm_factor = log2(1 + sum_lambdas) / sum_lambdas;
for (int item_index = static_cast<int>(threadIdx.x); item_index < query_item_count; item_index += static_cast<int>(blockDim.x)) {
cuda_out_gradients_pointer[item_index] *= norm_factor;
cuda_out_hessians_pointer[item_index] *= norm_factor;
}
}
__syncthreads();
}
}
void CUDALambdarankNDCG::LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const {
const int num_blocks = (num_queries_ + NUM_QUERY_PER_BLOCK - 1) / NUM_QUERY_PER_BLOCK;
const data_size_t num_rank_label = static_cast<int>(label_gain_.size());
const int device_index = GetCUDADevice(__FILE__, __LINE__);
cudaDeviceProp device_prop;
CUDASUCCESS_OR_FATAL(cudaGetDeviceProperties(&device_prop, device_index));
#define GetGradientsKernel_LambdarankNDCG_ARGS \
score, cuda_labels_, num_data_, \
num_queries_, cuda_query_boundaries_, cuda_inverse_max_dcgs_.RawData(), \
norm_, sigmoid_, truncation_level_, cuda_label_gain_.RawData(), num_rank_label, \
gradients, hessians
#define GetGradientsKernel_LambdarankNDCG_Sorted_ARGS \
score, cuda_item_indices_buffer_.RawData(), cuda_labels_, num_data_, \
num_queries_, cuda_query_boundaries_, cuda_inverse_max_dcgs_.RawData(), \
norm_, sigmoid_, truncation_level_, cuda_label_gain_.RawData(), num_rank_label, \
gradients, hessians
if (max_items_in_query_aligned_ <= 1024) {
if (num_rank_label <= 32 && device_prop.warpSize == 32) {
GetGradientsKernel_LambdarankNDCG<false, 32><<<num_blocks, max_items_in_query_aligned_>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 64) {
GetGradientsKernel_LambdarankNDCG<false, 64><<<num_blocks, max_items_in_query_aligned_>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 128) {
GetGradientsKernel_LambdarankNDCG<false, 128><<<num_blocks, max_items_in_query_aligned_>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 256) {
GetGradientsKernel_LambdarankNDCG<false, 256><<<num_blocks, max_items_in_query_aligned_>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 512) {
GetGradientsKernel_LambdarankNDCG<false, 512><<<num_blocks, max_items_in_query_aligned_>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 1024) {
GetGradientsKernel_LambdarankNDCG<false, 1024><<<num_blocks, max_items_in_query_aligned_>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else {
GetGradientsKernel_LambdarankNDCG<false, 2048><<<num_blocks, max_items_in_query_aligned_>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
}
} else if (max_items_in_query_aligned_ <= 2048) {
if (num_rank_label <= 32 && device_prop.warpSize == 32) {
GetGradientsKernel_LambdarankNDCG<true, 32><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 64) {
GetGradientsKernel_LambdarankNDCG<true, 64><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 128) {
GetGradientsKernel_LambdarankNDCG<true, 128><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 256) {
GetGradientsKernel_LambdarankNDCG<true, 256><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 512) {
GetGradientsKernel_LambdarankNDCG<true, 512><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else if (num_rank_label <= 1024) {
GetGradientsKernel_LambdarankNDCG<true, 1024><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
} else {
GetGradientsKernel_LambdarankNDCG<true, 2048><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_ARGS);
}
} else {
BitonicArgSortItemsGlobal(score, num_queries_, cuda_query_boundaries_, cuda_item_indices_buffer_.RawData());
if (num_rank_label <= 32 && device_prop.warpSize == 32) {
GetGradientsKernel_LambdarankNDCG_Sorted<32><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_Sorted_ARGS);
} else if (num_rank_label <= 64) {
GetGradientsKernel_LambdarankNDCG_Sorted<64><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_Sorted_ARGS);
} else if (num_rank_label <= 128) {
GetGradientsKernel_LambdarankNDCG_Sorted<128><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_Sorted_ARGS);
} else if (num_rank_label <= 256) {
GetGradientsKernel_LambdarankNDCG_Sorted<256><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_Sorted_ARGS);
} else if (num_rank_label <= 512) {
GetGradientsKernel_LambdarankNDCG_Sorted<512><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_Sorted_ARGS);
} else if (num_rank_label <= 1024) {
GetGradientsKernel_LambdarankNDCG_Sorted<1024><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_Sorted_ARGS);
} else {
GetGradientsKernel_LambdarankNDCG_Sorted<2048><<<num_blocks, 1024>>>(GetGradientsKernel_LambdarankNDCG_Sorted_ARGS);
}
}
SynchronizeCUDADevice(__FILE__, __LINE__);
#undef GetGradientsKernel_LambdarankNDCG_ARGS
#undef GetGradientsKernel_LambdarankNDCG_Sorted_ARGS
}
__device__ __forceinline__ double CUDAPhi(const label_t l, double g) {
return pow(2.0f, static_cast<double>(l)) - g;
}
template <size_t SHARED_MEMORY_SIZE>
__global__ void GetGradientsKernel_RankXENDCG_SharedMemory(
const double* cuda_scores,
const label_t* cuda_labels,
const double* cuda_item_rands,
const data_size_t num_data,
const data_size_t num_queries,
const data_size_t* cuda_query_boundaries,
score_t* cuda_out_gradients,
score_t* cuda_out_hessians) {
const data_size_t query_index_start = static_cast<data_size_t>(blockIdx.x) * NUM_QUERY_PER_BLOCK;
const data_size_t query_index_end = min(query_index_start + NUM_QUERY_PER_BLOCK, num_queries);
for (data_size_t query_index = query_index_start; query_index < query_index_end; ++query_index) {
const data_size_t item_index_start = cuda_query_boundaries[query_index];
const data_size_t item_index_end = cuda_query_boundaries[query_index + 1];
const data_size_t query_item_count = item_index_end - item_index_start;
score_t* cuda_out_gradients_pointer = cuda_out_gradients + item_index_start;
score_t* cuda_out_hessians_pointer = cuda_out_hessians + item_index_start;
const label_t* cuda_labels_pointer = cuda_labels + item_index_start;
const double* cuda_scores_pointer = cuda_scores + item_index_start;
const double* cuda_item_rands_pointer = cuda_item_rands + item_index_start;
const data_size_t block_reduce_size = query_item_count >= 1024 ? 1024 : query_item_count;
__shared__ double shared_rho[SHARED_MEMORY_SIZE];
// assert that warpSize == 32
__shared__ double shared_buffer[1024 / WARPSIZE];
__shared__ double shared_params[SHARED_MEMORY_SIZE];
__shared__ score_t shared_lambdas[SHARED_MEMORY_SIZE];
__shared__ double reduce_result;
if (query_item_count <= 1) {
for (data_size_t i = 0; i <= query_item_count; ++i) {
cuda_out_gradients_pointer[i] = 0.0f;
cuda_out_hessians_pointer[i] = 0.0f;
}
__syncthreads();
} else {
// compute softmax
double thread_reduce_result = kMinScore;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double rho = cuda_scores_pointer[i];
shared_rho[i] = rho;
if (rho > thread_reduce_result) {
thread_reduce_result = rho;
}
}
__syncthreads();
thread_reduce_result = ShuffleReduceMax<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
}
__syncthreads();
thread_reduce_result = 0.0f;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double exp_value = exp(shared_rho[i] - reduce_result);
shared_rho[i] = exp_value;
thread_reduce_result += exp_value;
}
thread_reduce_result = ShuffleReduceSum<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
}
__syncthreads();
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
shared_rho[i] /= reduce_result;
}
__syncthreads();
// compute params
thread_reduce_result = 0.0f;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double param_value = CUDAPhi(cuda_labels_pointer[i], cuda_item_rands_pointer[i]);
shared_params[i] = param_value;
thread_reduce_result += param_value;
}
thread_reduce_result = ShuffleReduceSum<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
reduce_result = 1.0f / max(kEpsilon, reduce_result);
}
__syncthreads();
const double inv_denominator = reduce_result;
thread_reduce_result = 0.0f;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double term = -shared_params[i] * inv_denominator + shared_rho[i];
shared_lambdas[i] = static_cast<score_t>(term);
shared_params[i] = term / (1.0f - shared_rho[i]);
thread_reduce_result += shared_params[i];
}
thread_reduce_result = ShuffleReduceSum<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
}
__syncthreads();
const double sum_l1 = reduce_result;
thread_reduce_result = 0.0f;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double term = shared_rho[i] * (sum_l1 - shared_params[i]);
shared_lambdas[i] += static_cast<score_t>(term);
shared_params[i] = term / (1.0f - shared_rho[i]);
thread_reduce_result += shared_params[i];
}
thread_reduce_result = ShuffleReduceSum<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
}
__syncthreads();
const double sum_l2 = reduce_result;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
shared_lambdas[i] += static_cast<score_t>(shared_rho[i] * (sum_l2 - shared_params[i]));
cuda_out_hessians_pointer[i] = static_cast<score_t>(shared_rho[i] * (1.0f - shared_rho[i]));
}
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
cuda_out_gradients_pointer[i] = shared_lambdas[i];
}
__syncthreads();
}
}
}
__global__ void GetGradientsKernel_RankXENDCG_GlobalMemory(
const double* cuda_scores,
const label_t* cuda_labels,
const double* cuda_item_rands,
const data_size_t num_data,
const data_size_t num_queries,
const data_size_t* cuda_query_boundaries,
double* cuda_params_buffer,
score_t* cuda_out_gradients,
score_t* cuda_out_hessians) {
const data_size_t query_index_start = static_cast<data_size_t>(blockIdx.x) * NUM_QUERY_PER_BLOCK;
const data_size_t query_index_end = min(query_index_start + NUM_QUERY_PER_BLOCK, num_queries);
for (data_size_t query_index = query_index_start; query_index < query_index_end; ++query_index) {
const data_size_t item_index_start = cuda_query_boundaries[query_index];
const data_size_t item_index_end = cuda_query_boundaries[query_index + 1];
const data_size_t query_item_count = item_index_end - item_index_start;
score_t* cuda_out_gradients_pointer = cuda_out_gradients + item_index_start;
score_t* cuda_out_hessians_pointer = cuda_out_hessians + item_index_start;
const label_t* cuda_labels_pointer = cuda_labels + item_index_start;
const double* cuda_scores_pointer = cuda_scores + item_index_start;
const double* cuda_item_rands_pointer = cuda_item_rands + item_index_start;
double* cuda_params_buffer_pointer = cuda_params_buffer + item_index_start;
const data_size_t block_reduce_size = query_item_count > 1024 ? 1024 : query_item_count;
// assert that warpSize == 32, so we use buffer size 1024 / 32 = 32
__shared__ double shared_buffer[1024 / WARPSIZE];
__shared__ double reduce_result;
if (query_item_count <= 1) {
for (data_size_t i = 0; i <= query_item_count; ++i) {
cuda_out_gradients_pointer[i] = 0.0f;
cuda_out_hessians_pointer[i] = 0.0f;
}
__syncthreads();
} else {
// compute softmax
double thread_reduce_result = kMinScore;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double rho = cuda_scores_pointer[i];
if (rho > thread_reduce_result) {
thread_reduce_result = rho;
}
}
__syncthreads();
thread_reduce_result = ShuffleReduceMax<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
}
__syncthreads();
thread_reduce_result = 0.0f;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double exp_value = exp(cuda_scores_pointer[i] - reduce_result);
cuda_out_hessians_pointer[i] = exp_value;
thread_reduce_result += exp_value;
}
thread_reduce_result = ShuffleReduceSum<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
}
__syncthreads();
// store probability into hessians
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
cuda_out_hessians_pointer[i] /= reduce_result;
}
__syncthreads();
// compute params
thread_reduce_result = 0.0f;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double param_value = CUDAPhi(cuda_labels_pointer[i], cuda_item_rands_pointer[i]);
cuda_params_buffer_pointer[i] = param_value;
thread_reduce_result += param_value;
}
thread_reduce_result = ShuffleReduceSum<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
reduce_result = 1.0f / max(kEpsilon, reduce_result);
}
__syncthreads();
const double inv_denominator = reduce_result;
thread_reduce_result = 0.0f;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double term = -cuda_params_buffer_pointer[i] * inv_denominator + cuda_out_hessians_pointer[i];
cuda_out_gradients_pointer[i] = static_cast<score_t>(term);
const double param = term / (1.0f - cuda_out_hessians_pointer[i]);
cuda_params_buffer_pointer[i] = param;
thread_reduce_result += param;
}
thread_reduce_result = ShuffleReduceSum<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
}
__syncthreads();
const double sum_l1 = reduce_result;
thread_reduce_result = 0.0f;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double term = cuda_out_hessians_pointer[i] * (sum_l1 - cuda_params_buffer_pointer[i]);
cuda_out_gradients_pointer[i] += static_cast<score_t>(term);
const double param = term / (1.0f - cuda_out_hessians_pointer[i]);
cuda_params_buffer_pointer[i] = param;
thread_reduce_result += param;
}
thread_reduce_result = ShuffleReduceSum<double>(thread_reduce_result, shared_buffer, block_reduce_size);
if (threadIdx.x == 0) {
reduce_result = thread_reduce_result;
}
__syncthreads();
const double sum_l2 = reduce_result;
for (data_size_t i = static_cast<data_size_t>(threadIdx.x); i < query_item_count; i += static_cast<data_size_t>(blockDim.x)) {
const double prob = cuda_out_hessians_pointer[i];
cuda_out_gradients_pointer[i] += static_cast<score_t>(prob * (sum_l2 - cuda_params_buffer_pointer[i]));
cuda_out_hessians_pointer[i] = static_cast<score_t>(prob * (1.0f - prob));
}
__syncthreads();
}
}
}
void CUDARankXENDCG::LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const {
GenerateItemRands();
CopyFromHostToCUDADevice<double>(cuda_item_rands_.RawData(), item_rands_.data(), item_rands_.size(), __FILE__, __LINE__);
const int num_blocks = (num_queries_ + NUM_QUERY_PER_BLOCK - 1) / NUM_QUERY_PER_BLOCK;
if (max_items_in_query_aligned_ <= 1024) {
GetGradientsKernel_RankXENDCG_SharedMemory<1024><<<num_blocks, max_items_in_query_aligned_>>>(
score,
cuda_labels_,
cuda_item_rands_.RawData(),
num_data_,
num_queries_,
cuda_query_boundaries_,
gradients,
hessians);
} else if (max_items_in_query_aligned_ <= 2 * 1024) {
GetGradientsKernel_RankXENDCG_SharedMemory<2 * 1024><<<num_blocks, 1024>>>(
score,
cuda_labels_,
cuda_item_rands_.RawData(),
num_data_,
num_queries_,
cuda_query_boundaries_,
gradients,
hessians);
} else {
GetGradientsKernel_RankXENDCG_GlobalMemory<<<num_blocks, 1024>>>(
score,
cuda_labels_,
cuda_item_rands_.RawData(),
num_data_,
num_queries_,
cuda_query_boundaries_,
cuda_params_buffer_.RawData(),
gradients,
hessians);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
}
} // namespace LightGBM
#endif // USE_CUDA
src/objective/cuda/cuda_rank_objective.hpp
+123
View File
@@ -0,0 +1,123 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_RANK_OBJECTIVE_HPP_
#define LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_RANK_OBJECTIVE_HPP_
#ifdef USE_CUDA
#define NUM_QUERY_PER_BLOCK (10)
#include <LightGBM/cuda/cuda_objective_function.hpp>
#include <LightGBM/utils/threading.h>
#include <fstream>
#include <string>
#include <vector>
#include "../rank_objective.hpp"
namespace LightGBM {
template <typename HOST_OBJECTIVE>
class CUDALambdaRankObjectiveInterface : public CUDAObjectiveInterface<HOST_OBJECTIVE> {
public:
explicit CUDALambdaRankObjectiveInterface(const Config& config): CUDAObjectiveInterface<HOST_OBJECTIVE>(config) {}
explicit CUDALambdaRankObjectiveInterface(const std::vector<std::string>& strs): CUDAObjectiveInterface<HOST_OBJECTIVE>(strs) {}
~CUDALambdaRankObjectiveInterface() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
CUDAObjectiveInterface<HOST_OBJECTIVE>::Init(metadata, num_data);
const int num_threads = OMP_NUM_THREADS();
std::vector<uint16_t> thread_max_num_items_in_query(num_threads);
Threading::For<data_size_t>(0, this->num_queries_, 1,
[this, &thread_max_num_items_in_query] (int thread_index, data_size_t start, data_size_t end) {
for (data_size_t query_index = start; query_index < end; ++query_index) {
const data_size_t query_item_count = this->query_boundaries_[query_index + 1] - this->query_boundaries_[query_index];
if (query_item_count > thread_max_num_items_in_query[thread_index]) {
thread_max_num_items_in_query[thread_index] = query_item_count;
}
}
});
data_size_t max_items_in_query = 0;
for (int thread_index = 0; thread_index < num_threads; ++thread_index) {
if (thread_max_num_items_in_query[thread_index] > max_items_in_query) {
max_items_in_query = thread_max_num_items_in_query[thread_index];
}
}
max_items_in_query_aligned_ = 1;
--max_items_in_query;
while (max_items_in_query > 0) {
max_items_in_query >>= 1;
max_items_in_query_aligned_ <<= 1;
}
if (max_items_in_query_aligned_ > 2048) {
cuda_item_indices_buffer_.Resize(static_cast<size_t>(metadata.query_boundaries()[metadata.num_queries()]));
}
this->cuda_labels_ = metadata.cuda_metadata()->cuda_label();
cuda_query_boundaries_ = metadata.cuda_metadata()->cuda_query_boundaries();
}
protected:
// CUDA memory, held by this object
CUDAVector<int> cuda_item_indices_buffer_;
// CUDA memory, held by other objects
const data_size_t* cuda_query_boundaries_;
// Host memory
int max_items_in_query_aligned_;
};
class CUDALambdarankNDCG: public CUDALambdaRankObjectiveInterface<LambdarankNDCG> {
public:
explicit CUDALambdarankNDCG(const Config& config);
explicit CUDALambdarankNDCG(const std::vector<std::string>& strs);
void Init(const Metadata& mdtadata, data_size_t num_data) override;
~CUDALambdarankNDCG();
private:
void LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const override;
// CUDA memory, held by this object
CUDAVector<double> cuda_inverse_max_dcgs_;
CUDAVector<double> cuda_label_gain_;
};
class CUDARankXENDCG : public CUDALambdaRankObjectiveInterface<RankXENDCG> {
public:
explicit CUDARankXENDCG(const Config& config);
explicit CUDARankXENDCG(const std::vector<std::string>& strs);
~CUDARankXENDCG();
void Init(const Metadata& metadata, data_size_t num_data) override;
protected:
void LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const;
void GenerateItemRands() const;
mutable std::vector<double> item_rands_;
CUDAVector<double> cuda_item_rands_;
CUDAVector<double> cuda_params_buffer_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_RANK_OBJECTIVE_HPP_
src/objective/cuda/cuda_regression_objective.cpp
+110
View File
@@ -0,0 +1,110 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include "cuda_regression_objective.hpp"
#include <string>
#include <vector>
namespace LightGBM {
CUDARegressionL2loss::CUDARegressionL2loss(const Config& config):
CUDARegressionObjectiveInterface<RegressionL2loss>(config) {}
CUDARegressionL2loss::CUDARegressionL2loss(const std::vector<std::string>& strs):
CUDARegressionObjectiveInterface<RegressionL2loss>(strs) {}
CUDARegressionL2loss::~CUDARegressionL2loss() {}
void CUDARegressionL2loss::Init(const Metadata& metadata, data_size_t num_data) {
CUDARegressionObjectiveInterface<RegressionL2loss>::Init(metadata, num_data);
}
CUDARegressionL1loss::CUDARegressionL1loss(const Config& config):
CUDARegressionObjectiveInterface<RegressionL1loss>(config) {}
CUDARegressionL1loss::CUDARegressionL1loss(const std::vector<std::string>& strs):
CUDARegressionObjectiveInterface<RegressionL1loss>(strs) {}
CUDARegressionL1loss::~CUDARegressionL1loss() {}
void CUDARegressionL1loss::Init(const Metadata& metadata, data_size_t num_data) {
CUDARegressionObjectiveInterface<RegressionL1loss>::Init(metadata, num_data);
cuda_data_indices_buffer_.Resize(static_cast<size_t>(num_data));
cuda_percentile_result_.Resize(1);
if (cuda_weights_ != nullptr) {
const int num_blocks = (num_data + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION + 1;
cuda_weights_prefix_sum_.Resize(static_cast<size_t>(num_data));
cuda_weights_prefix_sum_buffer_.Resize(static_cast<size_t>(num_blocks));
cuda_weight_by_leaf_buffer_.Resize(static_cast<size_t>(num_data));
}
cuda_residual_buffer_.Resize(static_cast<size_t>(num_data));
}
CUDARegressionHuberLoss::CUDARegressionHuberLoss(const Config& config):
CUDARegressionObjectiveInterface<RegressionHuberLoss>(config) {}
CUDARegressionHuberLoss::CUDARegressionHuberLoss(const std::vector<std::string>& strs):
CUDARegressionObjectiveInterface<RegressionHuberLoss>(strs) {}
CUDARegressionHuberLoss::~CUDARegressionHuberLoss() {}
CUDARegressionFairLoss::CUDARegressionFairLoss(const Config& config):
CUDARegressionObjectiveInterface<RegressionFairLoss>(config) {}
CUDARegressionFairLoss::CUDARegressionFairLoss(const std::vector<std::string>& strs):
CUDARegressionObjectiveInterface<RegressionFairLoss>(strs) {}
CUDARegressionFairLoss::~CUDARegressionFairLoss() {}
CUDARegressionPoissonLoss::CUDARegressionPoissonLoss(const Config& config):
CUDARegressionObjectiveInterface<RegressionPoissonLoss>(config) {}
CUDARegressionPoissonLoss::CUDARegressionPoissonLoss(const std::vector<std::string>& strs):
CUDARegressionObjectiveInterface<RegressionPoissonLoss>(strs) {}
CUDARegressionPoissonLoss::~CUDARegressionPoissonLoss() {}
void CUDARegressionPoissonLoss::Init(const Metadata& metadata, data_size_t num_data) {
CUDARegressionObjectiveInterface<RegressionPoissonLoss>::Init(metadata, num_data);
LaunchCheckLabelKernel();
}
double CUDARegressionPoissonLoss::LaunchCalcInitScoreKernel(const int class_id) const {
return Common::SafeLog(CUDARegressionObjectiveInterface<RegressionPoissonLoss>::LaunchCalcInitScoreKernel(class_id));
}
CUDARegressionQuantileloss::CUDARegressionQuantileloss(const Config& config):
CUDARegressionObjectiveInterface<RegressionQuantileloss>(config) {}
CUDARegressionQuantileloss::CUDARegressionQuantileloss(const std::vector<std::string>& strs):
CUDARegressionObjectiveInterface<RegressionQuantileloss>(strs) {}
CUDARegressionQuantileloss::~CUDARegressionQuantileloss() {}
void CUDARegressionQuantileloss::Init(const Metadata& metadata, data_size_t num_data) {
CUDARegressionObjectiveInterface<RegressionQuantileloss>::Init(metadata, num_data);
cuda_data_indices_buffer_.Resize(static_cast<size_t>(num_data));
cuda_percentile_result_.Resize(1);
if (cuda_weights_ != nullptr) {
const int num_blocks = (num_data + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION + 1;
cuda_weights_prefix_sum_.Resize(static_cast<size_t>(num_data));
cuda_weights_prefix_sum_buffer_.Resize(static_cast<size_t>(num_blocks));
cuda_weight_by_leaf_buffer_.Resize(static_cast<size_t>(num_data));
}
cuda_residual_buffer_.Resize(static_cast<size_t>(num_data));
}
} // namespace LightGBM
#endif // USE_CUDA
src/objective/cuda/cuda_regression_objective.cu
+482
View File
@@ -0,0 +1,482 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include "cuda_regression_objective.hpp"
#include <LightGBM/cuda/cuda_algorithms.hpp>
namespace LightGBM {
template <typename HOST_OBJECTIVE>
void CUDARegressionObjectiveInterface<HOST_OBJECTIVE>::Init(const Metadata& metadata, data_size_t num_data) {
CUDAObjectiveInterface<HOST_OBJECTIVE>::Init(metadata, num_data);
const data_size_t num_get_gradients_blocks = (this->num_data_ + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
cuda_block_buffer_.Resize(static_cast<size_t>(num_get_gradients_blocks));
if (this->sqrt_) {
cuda_trans_label_.Resize(this->trans_label_.size());
CopyFromHostToCUDADevice<label_t>(cuda_trans_label_.RawData(), this->trans_label_.data(), this->trans_label_.size(), __FILE__, __LINE__);
this->cuda_labels_ = cuda_trans_label_.RawData();
}
}
template void CUDARegressionObjectiveInterface<RegressionL2loss>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDARegressionObjectiveInterface<RegressionL1loss>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDARegressionObjectiveInterface<RegressionHuberLoss>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDARegressionObjectiveInterface<RegressionFairLoss>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDARegressionObjectiveInterface<RegressionPoissonLoss>::Init(const Metadata& metadata, data_size_t num_data);
template void CUDARegressionObjectiveInterface<RegressionQuantileloss>::Init(const Metadata& metadata, data_size_t num_data);
template <typename HOST_OBJECTIVE>
double CUDARegressionObjectiveInterface<HOST_OBJECTIVE>::LaunchCalcInitScoreKernel(const int /*class_id*/) const {
double label_sum = 0.0f, weight_sum = 0.0f;
if (this->cuda_weights_ == nullptr) {
ShuffleReduceSumGlobal<label_t, double>(this->cuda_labels_,
static_cast<size_t>(this->num_data_), cuda_block_buffer_.RawData());
CopyFromCUDADeviceToHost<double>(&label_sum, cuda_block_buffer_.RawData(), 1, __FILE__, __LINE__);
weight_sum = static_cast<double>(this->num_data_);
} else {
ShuffleReduceDotProdGlobal<label_t, double>(this->cuda_labels_,
this->cuda_weights_, static_cast<size_t>(this->num_data_), cuda_block_buffer_.RawData());
CopyFromCUDADeviceToHost<double>(&label_sum, cuda_block_buffer_.RawData(), 1, __FILE__, __LINE__);
ShuffleReduceSumGlobal<label_t, double>(this->cuda_weights_,
static_cast<size_t>(this->num_data_), cuda_block_buffer_.RawData());
CopyFromCUDADeviceToHost<double>(&weight_sum, cuda_block_buffer_.RawData(), 1, __FILE__, __LINE__);
}
return label_sum / weight_sum;
}
template double CUDARegressionObjectiveInterface<RegressionL2loss>::LaunchCalcInitScoreKernel(const int class_id) const;
template double CUDARegressionObjectiveInterface<RegressionL1loss>::LaunchCalcInitScoreKernel(const int class_id) const;
template double CUDARegressionObjectiveInterface<RegressionHuberLoss>::LaunchCalcInitScoreKernel(const int class_id) const;
template double CUDARegressionObjectiveInterface<RegressionFairLoss>::LaunchCalcInitScoreKernel(const int class_id) const;
template double CUDARegressionObjectiveInterface<RegressionPoissonLoss>::LaunchCalcInitScoreKernel(const int class_id) const;
template double CUDARegressionObjectiveInterface<RegressionQuantileloss>::LaunchCalcInitScoreKernel(const int class_id) const;
__global__ void ConvertOutputCUDAKernel_Regression(const bool sqrt, const data_size_t num_data, const double* input, double* output) {
const int data_index = static_cast<data_size_t>(blockIdx.x * blockDim.x + threadIdx.x);
if (data_index < num_data) {
if (sqrt) {
const double sign = input[data_index] >= 0.0f ? 1 : -1;
output[data_index] = sign * input[data_index] * input[data_index];
} else {
output[data_index] = input[data_index];
}
}
}
const double* CUDARegressionL2loss::LaunchConvertOutputCUDAKernel(const data_size_t num_data, const double* input, double* output) const {
const int num_blocks = (num_data + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
if (sqrt_) {
ConvertOutputCUDAKernel_Regression<<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(sqrt_, num_data, input, output);
return output;
} else {
return input;
}
}
template <bool USE_WEIGHT>
__global__ void GetGradientsKernel_RegressionL2(const double* cuda_scores, const label_t* cuda_labels, const label_t* cuda_weights, const data_size_t num_data,
score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
const data_size_t data_index = static_cast<data_size_t>(blockDim.x * blockIdx.x + threadIdx.x);
if (data_index < num_data) {
if (!USE_WEIGHT) {
cuda_out_gradients[data_index] = static_cast<score_t>(cuda_scores[data_index] - cuda_labels[data_index]);
cuda_out_hessians[data_index] = 1.0f;
} else {
const score_t weight = static_cast<score_t>(cuda_weights[data_index]);
cuda_out_gradients[data_index] = static_cast<score_t>(cuda_scores[data_index] - cuda_labels[data_index]) * weight;
cuda_out_hessians[data_index] = weight;
}
}
}
void CUDARegressionL2loss::LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
if (cuda_weights_ == nullptr) {
GetGradientsKernel_RegressionL2<false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, nullptr, num_data_, gradients, hessians);
} else {
GetGradientsKernel_RegressionL2<true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, cuda_weights_, num_data_, gradients, hessians);
}
}
double CUDARegressionL1loss::LaunchCalcInitScoreKernel(const int /*class_id*/) const {
const double alpha = 0.5f;
if (cuda_weights_ == nullptr) {
PercentileGlobal<label_t, data_size_t, label_t, double, false, false>(
cuda_labels_, nullptr, cuda_data_indices_buffer_.RawData(), nullptr, nullptr, alpha, num_data_, cuda_percentile_result_.RawData());
} else {
PercentileGlobal<label_t, data_size_t, label_t, double, false, true>(
cuda_labels_, cuda_weights_, cuda_data_indices_buffer_.RawData(), cuda_weights_prefix_sum_.RawData(),
cuda_weights_prefix_sum_buffer_.RawData(), alpha, num_data_, cuda_percentile_result_.RawData());
}
label_t percentile_result = 0.0f;
CopyFromCUDADeviceToHost<label_t>(&percentile_result, cuda_percentile_result_.RawData(), 1, __FILE__, __LINE__);
SynchronizeCUDADevice(__FILE__, __LINE__);
return static_cast<label_t>(percentile_result);
}
template <bool USE_WEIGHT>
__global__ void GetGradientsKernel_RegressionL1(const double* cuda_scores, const label_t* cuda_labels, const label_t* cuda_weights, const data_size_t num_data,
score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
const data_size_t data_index = static_cast<data_size_t>(blockDim.x * blockIdx.x + threadIdx.x);
if (data_index < num_data) {
if (!USE_WEIGHT) {
const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
cuda_out_gradients[data_index] = static_cast<score_t>((diff > 0.0f) - (diff < 0.0f));
cuda_out_hessians[data_index] = 1.0f;
} else {
const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
const score_t weight = static_cast<score_t>(cuda_weights[data_index]);
cuda_out_gradients[data_index] = static_cast<score_t>((diff > 0.0f) - (diff < 0.0f)) * weight;
cuda_out_hessians[data_index] = weight;
}
}
}
void CUDARegressionL1loss::LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
if (cuda_weights_ == nullptr) {
GetGradientsKernel_RegressionL1<false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, nullptr, num_data_, gradients, hessians);
} else {
GetGradientsKernel_RegressionL1<true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, cuda_weights_, num_data_, gradients, hessians);
}
}
template <bool USE_WEIGHT>
__global__ void RenewTreeOutputCUDAKernel_RegressionL1(
const double* score,
const label_t* label,
const label_t* weight,
double* residual_buffer,
label_t* weight_by_leaf,
double* weight_prefix_sum_buffer,
const data_size_t* data_indices_in_leaf,
const data_size_t* num_data_in_leaf,
const data_size_t* data_start_in_leaf,
data_size_t* data_indices_buffer,
double* leaf_value) {
const int leaf_index = static_cast<int>(blockIdx.x);
const data_size_t data_start = data_start_in_leaf[leaf_index];
const data_size_t num_data = num_data_in_leaf[leaf_index];
data_size_t* data_indices_buffer_pointer = data_indices_buffer + data_start;
const label_t* weight_by_leaf_pointer = weight_by_leaf + data_start;
double* weight_prefix_sum_buffer_pointer = weight_prefix_sum_buffer + data_start;
const double* residual_buffer_pointer = residual_buffer + data_start;
const double alpha = 0.5f;
for (data_size_t inner_data_index = data_start + static_cast<data_size_t>(threadIdx.x);
inner_data_index < data_start + num_data; inner_data_index += static_cast<data_size_t>(blockDim.x)) {
const data_size_t data_index = data_indices_in_leaf[inner_data_index];
const label_t data_label = label[data_index];
const double data_score = score[data_index];
residual_buffer[inner_data_index] = static_cast<double>(data_label) - data_score;
if (USE_WEIGHT) {
weight_by_leaf[inner_data_index] = weight[data_index];
}
}
__syncthreads();
const double renew_leaf_value = PercentileDevice<double, data_size_t, label_t, double, false, USE_WEIGHT>(
residual_buffer_pointer, weight_by_leaf_pointer, data_indices_buffer_pointer,
weight_prefix_sum_buffer_pointer, alpha, num_data);
if (threadIdx.x == 0) {
leaf_value[leaf_index] = renew_leaf_value;
}
}
void CUDARegressionL1loss::LaunchRenewTreeOutputCUDAKernel(
const double* score,
const data_size_t* data_indices_in_leaf,
const data_size_t* num_data_in_leaf,
const data_size_t* data_start_in_leaf,
const int num_leaves,
double* leaf_value) const {
if (cuda_weights_ == nullptr) {
RenewTreeOutputCUDAKernel_RegressionL1<false><<<num_leaves, GET_GRADIENTS_BLOCK_SIZE_REGRESSION / 2>>>(
score,
cuda_labels_,
cuda_weights_,
cuda_residual_buffer_.RawData(),
cuda_weight_by_leaf_buffer_.RawData(),
cuda_weights_prefix_sum_.RawData(),
data_indices_in_leaf,
num_data_in_leaf,
data_start_in_leaf,
cuda_data_indices_buffer_.RawData(),
leaf_value);
} else {
RenewTreeOutputCUDAKernel_RegressionL1<true><<<num_leaves, GET_GRADIENTS_BLOCK_SIZE_REGRESSION / 4>>>(
score,
cuda_labels_,
cuda_weights_,
cuda_residual_buffer_.RawData(),
cuda_weight_by_leaf_buffer_.RawData(),
cuda_weights_prefix_sum_.RawData(),
data_indices_in_leaf,
num_data_in_leaf,
data_start_in_leaf,
cuda_data_indices_buffer_.RawData(),
leaf_value);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
}
template <bool USE_WEIGHT>
__global__ void GetGradientsKernel_Huber(const double* cuda_scores, const label_t* cuda_labels, const label_t* cuda_weights, const data_size_t num_data,
const double alpha, score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
const data_size_t data_index = static_cast<data_size_t>(blockDim.x * blockIdx.x + threadIdx.x);
if (data_index < num_data) {
if (!USE_WEIGHT) {
const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
if (fabs(diff) <= alpha) {
cuda_out_gradients[data_index] = static_cast<score_t>(diff);
} else {
const score_t sign = static_cast<score_t>((diff > 0.0f) - (diff < 0.0f));
cuda_out_gradients[data_index] = static_cast<score_t>(sign * alpha);
}
cuda_out_hessians[data_index] = 1.0f;
} else {
const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
const score_t weight = static_cast<score_t>(cuda_weights[data_index]);
if (fabs(diff) <= alpha) {
cuda_out_gradients[data_index] = static_cast<score_t>(diff) * weight;
} else {
const score_t sign = static_cast<score_t>((diff > 0.0f) - (diff < 0.0f));
cuda_out_gradients[data_index] = static_cast<score_t>(sign * alpha) * weight;
}
cuda_out_hessians[data_index] = weight;
}
}
}
void CUDARegressionHuberLoss::LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
if (cuda_weights_ == nullptr) {
GetGradientsKernel_Huber<false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, nullptr, num_data_, alpha_, gradients, hessians);
} else {
GetGradientsKernel_Huber<true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, cuda_weights_, num_data_, alpha_, gradients, hessians);
}
}
template <bool USE_WEIGHT>
__global__ void GetGradientsKernel_Fair(const double* cuda_scores, const label_t* cuda_labels, const label_t* cuda_weights, const data_size_t num_data,
const double c, score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
const data_size_t data_index = static_cast<data_size_t>(blockDim.x * blockIdx.x + threadIdx.x);
if (data_index < num_data) {
if (!USE_WEIGHT) {
const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
cuda_out_gradients[data_index] = static_cast<score_t>(c * diff / (fabs(diff) + c));
cuda_out_hessians[data_index] = static_cast<score_t>(c * c / ((fabs(diff) + c) * (fabs(diff) + c)));
} else {
const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
const score_t weight = static_cast<score_t>(cuda_weights[data_index]);
cuda_out_gradients[data_index] = static_cast<score_t>(c * diff / (fabs(diff) + c) * weight);
cuda_out_hessians[data_index] = static_cast<score_t>(c * c / ((fabs(diff) + c) * (fabs(diff) + c)) * weight);
}
}
}
void CUDARegressionFairLoss::LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
if (cuda_weights_ == nullptr) {
GetGradientsKernel_Fair<false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, nullptr, num_data_, c_, gradients, hessians);
} else {
GetGradientsKernel_Fair<true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, cuda_weights_, num_data_, c_, gradients, hessians);
}
}
void CUDARegressionPoissonLoss::LaunchCheckLabelKernel() const {
ShuffleReduceSumGlobal<label_t, double>(cuda_labels_, static_cast<size_t>(num_data_), cuda_block_buffer_.RawData());
double label_sum = 0.0f;
CopyFromCUDADeviceToHost<double>(&label_sum, cuda_block_buffer_.RawData(), 1, __FILE__, __LINE__);
ShuffleReduceMinGlobal<label_t, double>(cuda_labels_, static_cast<size_t>(num_data_), cuda_block_buffer_.RawData());
double label_min = 0.0f;
CopyFromCUDADeviceToHost<double>(&label_min, cuda_block_buffer_.RawData(), 1, __FILE__, __LINE__);
if (label_min < 0.0f) {
Log::Fatal("[%s]: at least one target label is negative", GetName());
}
if (label_sum == 0.0f) {
Log::Fatal("[%s]: sum of labels is zero", GetName());
}
}
template <bool USE_WEIGHT>
__global__ void GetGradientsKernel_Poisson(const double* cuda_scores, const label_t* cuda_labels, const label_t* cuda_weights, const data_size_t num_data,
const double max_delta_step, score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
const data_size_t data_index = static_cast<data_size_t>(blockDim.x * blockIdx.x + threadIdx.x);
const double exp_max_delta_step = std::exp(max_delta_step);
if (data_index < num_data) {
if (!USE_WEIGHT) {
const double exp_score = exp(cuda_scores[data_index]);
cuda_out_gradients[data_index] = static_cast<score_t>(exp_score - cuda_labels[data_index]);
cuda_out_hessians[data_index] = static_cast<score_t>(exp_score * exp_max_delta_step);
} else {
const double exp_score = exp(cuda_scores[data_index]);
const score_t weight = static_cast<score_t>(cuda_weights[data_index]);
cuda_out_gradients[data_index] = static_cast<score_t>((exp_score - cuda_labels[data_index]) * weight);
cuda_out_hessians[data_index] = static_cast<score_t>(exp_score * exp_max_delta_step * weight);
}
}
}
void CUDARegressionPoissonLoss::LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
if (cuda_weights_ == nullptr) {
GetGradientsKernel_Poisson<false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(
score, cuda_labels_, nullptr, num_data_, max_delta_step_, gradients, hessians);
} else {
GetGradientsKernel_Poisson<true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(
score, cuda_labels_, cuda_weights_, num_data_, max_delta_step_, gradients, hessians);
}
}
__global__ void ConvertOutputCUDAKernel_Regression_Poisson(const data_size_t num_data, const double* input, double* output) {
const int data_index = static_cast<data_size_t>(blockIdx.x * blockDim.x + threadIdx.x);
if (data_index < num_data) {
output[data_index] = exp(input[data_index]);
}
}
const double* CUDARegressionPoissonLoss::LaunchConvertOutputCUDAKernel(const data_size_t num_data, const double* input, double* output) const {
const int num_blocks = (num_data + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
ConvertOutputCUDAKernel_Regression_Poisson<<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(num_data, input, output);
return output;
}
double CUDARegressionQuantileloss::LaunchCalcInitScoreKernel(const int /*class_id*/) const {
if (cuda_weights_ == nullptr) {
PercentileGlobal<label_t, data_size_t, label_t, double, false, false>(
cuda_labels_, nullptr, cuda_data_indices_buffer_.RawData(), nullptr, nullptr, alpha_, num_data_, cuda_percentile_result_.RawData());
} else {
PercentileGlobal<label_t, data_size_t, label_t, double, false, true>(
cuda_labels_, cuda_weights_, cuda_data_indices_buffer_.RawData(), cuda_weights_prefix_sum_.RawData(),
cuda_weights_prefix_sum_buffer_.RawData(), alpha_, num_data_, cuda_percentile_result_.RawData());
}
label_t percentile_result = 0.0f;
CopyFromCUDADeviceToHost<label_t>(&percentile_result, cuda_percentile_result_.RawData(), 1, __FILE__, __LINE__);
SynchronizeCUDADevice(__FILE__, __LINE__);
return static_cast<label_t>(percentile_result);
}
template <bool USE_WEIGHT>
__global__ void RenewTreeOutputCUDAKernel_RegressionQuantile(
const double* score,
const label_t* label,
const label_t* weight,
double* residual_buffer,
label_t* weight_by_leaf,
double* weight_prefix_sum_buffer,
const data_size_t* data_indices_in_leaf,
const data_size_t* num_data_in_leaf,
const data_size_t* data_start_in_leaf,
data_size_t* data_indices_buffer,
double* leaf_value,
const double alpha) {
const int leaf_index = static_cast<int>(blockIdx.x);
const data_size_t data_start = data_start_in_leaf[leaf_index];
const data_size_t num_data = num_data_in_leaf[leaf_index];
data_size_t* data_indices_buffer_pointer = data_indices_buffer + data_start;
const label_t* weight_by_leaf_pointer = weight_by_leaf + data_start;
double* weight_prefix_sum_buffer_pointer = weight_prefix_sum_buffer + data_start;
const double* residual_buffer_pointer = residual_buffer + data_start;
for (data_size_t inner_data_index = data_start + static_cast<data_size_t>(threadIdx.x); inner_data_index < data_start + num_data; inner_data_index += static_cast<data_size_t>(blockDim.x)) {
const data_size_t data_index = data_indices_in_leaf[inner_data_index];
const label_t data_label = label[data_index];
const double data_score = score[data_index];
residual_buffer[inner_data_index] = static_cast<double>(data_label) - data_score;
if (USE_WEIGHT) {
weight_by_leaf[inner_data_index] = weight[data_index];
}
}
__syncthreads();
const double renew_leaf_value = PercentileDevice<double, data_size_t, label_t, double, false, USE_WEIGHT>(
residual_buffer_pointer, weight_by_leaf_pointer, data_indices_buffer_pointer,
weight_prefix_sum_buffer_pointer, alpha, num_data);
if (threadIdx.x == 0) {
leaf_value[leaf_index] = renew_leaf_value;
}
}
void CUDARegressionQuantileloss::LaunchRenewTreeOutputCUDAKernel(
const double* score, const data_size_t* data_indices_in_leaf, const data_size_t* num_data_in_leaf,
const data_size_t* data_start_in_leaf, const int num_leaves, double* leaf_value) const {
if (cuda_weights_ == nullptr) {
RenewTreeOutputCUDAKernel_RegressionQuantile<false><<<num_leaves, GET_GRADIENTS_BLOCK_SIZE_REGRESSION / 2>>>(
score,
cuda_labels_,
cuda_weights_,
cuda_residual_buffer_.RawData(),
cuda_weight_by_leaf_buffer_.RawData(),
cuda_weights_prefix_sum_.RawData(),
data_indices_in_leaf,
num_data_in_leaf,
data_start_in_leaf,
cuda_data_indices_buffer_.RawData(),
leaf_value,
alpha_);
} else {
RenewTreeOutputCUDAKernel_RegressionQuantile<true><<<num_leaves, GET_GRADIENTS_BLOCK_SIZE_REGRESSION / 4>>>(
score,
cuda_labels_,
cuda_weights_,
cuda_residual_buffer_.RawData(),
cuda_weight_by_leaf_buffer_.RawData(),
cuda_weights_prefix_sum_.RawData(),
data_indices_in_leaf,
num_data_in_leaf,
data_start_in_leaf,
cuda_data_indices_buffer_.RawData(),
leaf_value,
alpha_);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
}
template <bool USE_WEIGHT>
__global__ void GetGradientsKernel_RegressionQuantile(const double* cuda_scores, const label_t* cuda_labels,
const label_t* cuda_weights, const data_size_t num_data, const double alpha,
score_t* cuda_out_gradients, score_t* cuda_out_hessians) {
const data_size_t data_index = static_cast<data_size_t>(blockDim.x * blockIdx.x + threadIdx.x);
if (data_index < num_data) {
if (!USE_WEIGHT) {
const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
if (diff >= 0.0f) {
cuda_out_gradients[data_index] = (1.0f - alpha);
} else {
cuda_out_gradients[data_index] = -alpha;
}
cuda_out_hessians[data_index] = 1.0f;
} else {
const double diff = cuda_scores[data_index] - static_cast<double>(cuda_labels[data_index]);
const score_t weight = static_cast<score_t>(cuda_weights[data_index]);
if (diff >= 0.0f) {
cuda_out_gradients[data_index] = (1.0f - alpha) * weight;
} else {
cuda_out_gradients[data_index] = -alpha * weight;
}
cuda_out_hessians[data_index] = weight;
}
}
}
void CUDARegressionQuantileloss::LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const {
const int num_blocks = (num_data_ + GET_GRADIENTS_BLOCK_SIZE_REGRESSION - 1) / GET_GRADIENTS_BLOCK_SIZE_REGRESSION;
if (cuda_weights_ == nullptr) {
GetGradientsKernel_RegressionQuantile<false><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, nullptr, num_data_, alpha_, gradients, hessians);
} else {
GetGradientsKernel_RegressionQuantile<true><<<num_blocks, GET_GRADIENTS_BLOCK_SIZE_REGRESSION>>>(score, cuda_labels_, cuda_weights_, num_data_, alpha_, gradients, hessians);
}
}
} // namespace LightGBM
#endif // USE_CUDA
src/objective/cuda/cuda_regression_objective.hpp
+168
View File
@@ -0,0 +1,168 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_REGRESSION_OBJECTIVE_HPP_
#define LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_REGRESSION_OBJECTIVE_HPP_
#ifdef USE_CUDA
#define GET_GRADIENTS_BLOCK_SIZE_REGRESSION (1024)
#include <LightGBM/cuda/cuda_objective_function.hpp>
#include <string>
#include <vector>
#include "../regression_objective.hpp"
namespace LightGBM {
template <typename HOST_OBJECTIVE>
class CUDARegressionObjectiveInterface: public CUDAObjectiveInterface<HOST_OBJECTIVE> {
public:
explicit CUDARegressionObjectiveInterface(const Config& config): CUDAObjectiveInterface<HOST_OBJECTIVE>(config) {}
explicit CUDARegressionObjectiveInterface(const std::vector<std::string>& strs): CUDAObjectiveInterface<HOST_OBJECTIVE>(strs) {}
void Init(const Metadata& metadata, data_size_t num_data) override;
protected:
double LaunchCalcInitScoreKernel(const int class_id) const override;
CUDAVector<double> cuda_block_buffer_;
CUDAVector<label_t> cuda_trans_label_;
};
class CUDARegressionL2loss : public CUDARegressionObjectiveInterface<RegressionL2loss> {
public:
explicit CUDARegressionL2loss(const Config& config);
explicit CUDARegressionL2loss(const std::vector<std::string>& strs);
~CUDARegressionL2loss();
void Init(const Metadata& metadata, data_size_t num_data) override;
protected:
void LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const override;
const double* LaunchConvertOutputCUDAKernel(const data_size_t num_data, const double* input, double* output) const override;
bool NeedConvertOutputCUDA() const override { return sqrt_; }
};
class CUDARegressionL1loss : public CUDARegressionObjectiveInterface<RegressionL1loss> {
public:
explicit CUDARegressionL1loss(const Config& config);
explicit CUDARegressionL1loss(const std::vector<std::string>& strs);
~CUDARegressionL1loss();
void Init(const Metadata& metadata, data_size_t num_data) override;
protected:
CUDAVector<data_size_t> cuda_data_indices_buffer_;
CUDAVector<double> cuda_weights_prefix_sum_;
CUDAVector<double> cuda_weights_prefix_sum_buffer_;
CUDAVector<double> cuda_residual_buffer_;
CUDAVector<label_t> cuda_weight_by_leaf_buffer_;
CUDAVector<label_t> cuda_percentile_result_;
double LaunchCalcInitScoreKernel(const int class_id) const override;
void LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const override;
void LaunchRenewTreeOutputCUDAKernel(
const double* score, const data_size_t* data_indices_in_leaf, const data_size_t* num_data_in_leaf,
const data_size_t* data_start_in_leaf, const int num_leaves, double* leaf_value) const override;
};
class CUDARegressionHuberLoss : public CUDARegressionObjectiveInterface<RegressionHuberLoss> {
public:
explicit CUDARegressionHuberLoss(const Config& config);
explicit CUDARegressionHuberLoss(const std::vector<std::string>& strs);
~CUDARegressionHuberLoss();
private:
void LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const override;
};
// http://research.microsoft.com/en-us/um/people/zhang/INRIA/Publis/Tutorial-Estim/node24.html
class CUDARegressionFairLoss : public CUDARegressionObjectiveInterface<RegressionFairLoss> {
public:
explicit CUDARegressionFairLoss(const Config& config);
explicit CUDARegressionFairLoss(const std::vector<std::string>& strs);
~CUDARegressionFairLoss();
private:
void LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const override;
};
class CUDARegressionPoissonLoss : public CUDARegressionObjectiveInterface<RegressionPoissonLoss> {
public:
explicit CUDARegressionPoissonLoss(const Config& config);
explicit CUDARegressionPoissonLoss(const std::vector<std::string>& strs);
~CUDARegressionPoissonLoss();
void Init(const Metadata& metadata, data_size_t num_data) override;
private:
void LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const override;
const double* LaunchConvertOutputCUDAKernel(const data_size_t num_data, const double* input, double* output) const override;
bool NeedConvertOutputCUDA() const override { return true; }
double LaunchCalcInitScoreKernel(const int class_id) const override;
void LaunchCheckLabelKernel() const;
};
class CUDARegressionQuantileloss : public CUDARegressionObjectiveInterface<RegressionQuantileloss> {
public:
explicit CUDARegressionQuantileloss(const Config& config);
explicit CUDARegressionQuantileloss(const std::vector<std::string>& strs);
~CUDARegressionQuantileloss();
void Init(const Metadata& metadata, data_size_t num_data) override;
protected:
void LaunchGetGradientsKernel(const double* score, score_t* gradients, score_t* hessians) const override;
double LaunchCalcInitScoreKernel(const int class_id) const override;
void LaunchRenewTreeOutputCUDAKernel(
const double* score, const data_size_t* data_indices_in_leaf, const data_size_t* num_data_in_leaf,
const data_size_t* data_start_in_leaf, const int num_leaves, double* leaf_value) const override;
CUDAVector<data_size_t> cuda_data_indices_buffer_;
CUDAVector<double> cuda_weights_prefix_sum_;
CUDAVector<double> cuda_weights_prefix_sum_buffer_;
CUDAVector<double> cuda_residual_buffer_;
CUDAVector<label_t> cuda_weight_by_leaf_buffer_;
CUDAVector<label_t> cuda_percentile_result_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_OBJECTIVE_CUDA_CUDA_REGRESSION_OBJECTIVE_HPP_
src/objective/multiclass_objective.hpp
+280
View File
@@ -0,0 +1,280 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_OBJECTIVE_MULTICLASS_OBJECTIVE_HPP_
#define LIGHTGBM_SRC_OBJECTIVE_MULTICLASS_OBJECTIVE_HPP_
#include <LightGBM/network.h>
#include <LightGBM/objective_function.h>
#include <string>
#include <algorithm>
#include <cmath>
#include <cstring>
#include <memory>
#include <vector>
#include "binary_objective.hpp"
namespace LightGBM {
/*!
* \brief Objective function for multiclass classification, use softmax as objective functions
*/
class MulticlassSoftmax: public ObjectiveFunction {
public:
explicit MulticlassSoftmax(const Config& config) {
num_class_ = config.num_class;
// This factor is to rescale the redundant form of K-classification, to the non-redundant form.
// In the traditional settings of K-classification, there is one redundant class, whose output is set to 0 (like the class 0 in binary classification).
// This is from the Friedman GBDT paper.
factor_ = static_cast<double>(num_class_) / (num_class_ - 1.0f);
}
explicit MulticlassSoftmax(const std::vector<std::string>& strs) {
num_class_ = -1;
for (auto str : strs) {
auto tokens = Common::Split(str.c_str(), ':');
if (tokens.size() == 2) {
if (tokens[0] == std::string("num_class")) {
Common::Atoi(tokens[1].c_str(), &num_class_);
}
}
}
if (num_class_ < 0) {
Log::Fatal("Objective should contain num_class field");
}
factor_ = static_cast<double>(num_class_) / (num_class_ - 1.0f);
}
~MulticlassSoftmax() {
}
void Init(const Metadata& metadata, data_size_t num_data) override {
num_data_ = num_data;
label_ = metadata.label();
weights_ = metadata.weights();
label_int_.resize(num_data_);
class_init_probs_.resize(num_class_, 0.0);
double sum_weight = 0.0;
for (int i = 0; i < num_data_; ++i) {
label_int_[i] = static_cast<int>(label_[i]);
if (label_int_[i] < 0 || label_int_[i] >= num_class_) {
Log::Fatal("Label must be in [0, %d), but found %d in label", num_class_, label_int_[i]);
}
if (weights_ == nullptr) {
class_init_probs_[label_int_[i]] += 1.0;
} else {
class_init_probs_[label_int_[i]] += weights_[i];
sum_weight += weights_[i];
}
}
if (weights_ == nullptr) {
sum_weight = num_data_;
}
if (Network::num_machines() > 1) {
sum_weight = Network::GlobalSyncUpBySum(sum_weight);
for (int i = 0; i < num_class_; ++i) {
class_init_probs_[i] = Network::GlobalSyncUpBySum(class_init_probs_[i]);
}
}
for (int i = 0; i < num_class_; ++i) {
class_init_probs_[i] /= sum_weight;
}
}
void GetGradients(const double* score, score_t* gradients, score_t* hessians) const override {
if (weights_ == nullptr) {
std::vector<double> rec;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) private(rec)
for (data_size_t i = 0; i < num_data_; ++i) {
rec.resize(num_class_);
for (int k = 0; k < num_class_; ++k) {
size_t idx = static_cast<size_t>(num_data_) * k + i;
rec[k] = static_cast<double>(score[idx]);
}
Common::Softmax(&rec);
for (int k = 0; k < num_class_; ++k) {
auto p = rec[k];
size_t idx = static_cast<size_t>(num_data_) * k + i;
if (label_int_[i] == k) {
gradients[idx] = static_cast<score_t>(p - 1.0f);
} else {
gradients[idx] = static_cast<score_t>(p);
}
hessians[idx] = static_cast<score_t>(factor_ * p * (1.0f - p));
}
}
} else {
std::vector<double> rec;
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) private(rec)
for (data_size_t i = 0; i < num_data_; ++i) {
rec.resize(num_class_);
for (int k = 0; k < num_class_; ++k) {
size_t idx = static_cast<size_t>(num_data_) * k + i;
rec[k] = static_cast<double>(score[idx]);
}
Common::Softmax(&rec);
for (int k = 0; k < num_class_; ++k) {
auto p = rec[k];
size_t idx = static_cast<size_t>(num_data_) * k + i;
if (label_int_[i] == k) {
gradients[idx] = static_cast<score_t>((p - 1.0f) * weights_[i]);
} else {
gradients[idx] = static_cast<score_t>(p * weights_[i]);
}
hessians[idx] = static_cast<score_t>((factor_ * p * (1.0f - p))* weights_[i]);
}
}
}
}
void ConvertOutput(const double* input, double* output) const override {
Common::Softmax(input, output, num_class_);
}
const char* GetName() const override {
return "multiclass";
}
std::string ToString() const override {
std::stringstream str_buf;
str_buf << GetName() << " ";
str_buf << "num_class:" << num_class_;
return str_buf.str();
}
bool SkipEmptyClass() const override { return true; }
int NumModelPerIteration() const override { return num_class_; }
int NumPredictOneRow() const override { return num_class_; }
bool NeedAccuratePrediction() const override { return false; }
double BoostFromScore(int class_id) const override {
return std::log(std::max<double>(kEpsilon, class_init_probs_[class_id]));
}
bool ClassNeedTrain(int class_id) const override {
if (std::fabs(class_init_probs_[class_id]) <= kEpsilon
|| std::fabs(class_init_probs_[class_id]) >= 1.0 - kEpsilon) {
return false;
} else {
return true;
}
}
protected:
double factor_;
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Number of classes */
int num_class_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Corresponding integers of label_ */
std::vector<int> label_int_;
/*! \brief Weights for data */
const label_t* weights_;
std::vector<double> class_init_probs_;
};
/*!
* \brief Objective function for multiclass classification, use one-vs-all binary objective function
*/
class MulticlassOVA: public ObjectiveFunction {
public:
explicit MulticlassOVA(const Config& config) {
num_class_ = config.num_class;
for (int i = 0; i < num_class_; ++i) {
binary_loss_.emplace_back(
new BinaryLogloss(config, [i](label_t label) { return static_cast<int>(label) == i; }));
}
sigmoid_ = config.sigmoid;
}
explicit MulticlassOVA(const std::vector<std::string>& strs) {
num_class_ = -1;
sigmoid_ = -1;
for (auto str : strs) {
auto tokens = Common::Split(str.c_str(), ':');
if (tokens.size() == 2) {
if (tokens[0] == std::string("num_class")) {
Common::Atoi(tokens[1].c_str(), &num_class_);
} else if (tokens[0] == std::string("sigmoid")) {
Common::Atof(tokens[1].c_str(), &sigmoid_);
}
}
}
if (num_class_ < 0) {
Log::Fatal("Objective should contain num_class field");
}
if (sigmoid_ <= 0.0) {
Log::Fatal("Sigmoid parameter %f should be greater than zero", sigmoid_);
}
}
~MulticlassOVA() {
}
void Init(const Metadata& metadata, data_size_t num_data) override {
num_data_ = num_data;
for (int i = 0; i < num_class_; ++i) {
binary_loss_[i]->Init(metadata, num_data);
}
}
void GetGradients(const double* score, score_t* gradients, score_t* hessians) const override {
for (int i = 0; i < num_class_; ++i) {
int64_t offset = static_cast<int64_t>(num_data_) * i;
binary_loss_[i]->GetGradients(score + offset, gradients + offset, hessians + offset);
}
}
const char* GetName() const override {
return "multiclassova";
}
void ConvertOutput(const double* input, double* output) const override {
for (int i = 0; i < num_class_; ++i) {
output[i] = 1.0f / (1.0f + std::exp(-sigmoid_ * input[i]));
}
}
std::string ToString() const override {
std::stringstream str_buf;
str_buf << GetName() << " ";
str_buf << "num_class:" << num_class_ << " ";
str_buf << "sigmoid:" << sigmoid_;
return str_buf.str();
}
bool SkipEmptyClass() const override { return true; }
int NumModelPerIteration() const override { return num_class_; }
int NumPredictOneRow() const override { return num_class_; }
bool NeedAccuratePrediction() const override { return false; }
double BoostFromScore(int class_id) const override {
return binary_loss_[class_id]->BoostFromScore(0);
}
bool ClassNeedTrain(int class_id) const override {
return binary_loss_[class_id]->ClassNeedTrain(0);
}
protected:
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Number of classes */
int num_class_;
std::vector<std::unique_ptr<BinaryLogloss>> binary_loss_;
double sigmoid_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_OBJECTIVE_MULTICLASS_OBJECTIVE_HPP_
src/objective/objective_function.cpp
+164
View File
@@ -0,0 +1,164 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#include <LightGBM/objective_function.h>
#include <string>
#include "binary_objective.hpp"
#include "multiclass_objective.hpp"
#include "rank_objective.hpp"
#include "regression_objective.hpp"
#include "xentropy_objective.hpp"
#include <LightGBM/cuda/cuda_objective_function.hpp>
#include "cuda/cuda_binary_objective.hpp"
#include "cuda/cuda_multiclass_objective.hpp"
#include "cuda/cuda_rank_objective.hpp"
#include "cuda/cuda_regression_objective.hpp"
namespace LightGBM {
#ifdef USE_CUDA
ObjectiveFunction* ObjectiveFunction::CreateObjectiveFunctionCUDA(const std::string& type, const Config& config) {
if (type == std::string("regression")) {
return new CUDARegressionL2loss(config);
} else if (type == std::string("regression_l1")) {
return new CUDARegressionL1loss(config);
} else if (type == std::string("quantile")) {
return new CUDARegressionQuantileloss(config);
} else if (type == std::string("huber")) {
return new CUDARegressionHuberLoss(config);
} else if (type == std::string("fair")) {
return new CUDARegressionFairLoss(config);
} else if (type == std::string("poisson")) {
return new CUDARegressionPoissonLoss(config);
} else if (type == std::string("binary")) {
return new CUDABinaryLogloss(config);
} else if (type == std::string("lambdarank")) {
return new CUDALambdarankNDCG(config);
} else if (type == std::string("rank_xendcg")) {
return new CUDARankXENDCG(config);
} else if (type == std::string("multiclass")) {
return new CUDAMulticlassSoftmax(config);
} else if (type == std::string("multiclassova")) {
return new CUDAMulticlassOVA(config);
} else if (type == std::string("cross_entropy")) {
Log::Warning("Objective cross_entropy is not implemented in cuda version. Fall back to boosting on CPU.");
return new CrossEntropy(config);
} else if (type == std::string("cross_entropy_lambda")) {
Log::Warning("Objective cross_entropy_lambda is not implemented in cuda version. Fall back to boosting on CPU.");
return new CrossEntropyLambda(config);
} else if (type == std::string("mape")) {
Log::Warning("Objective mape is not implemented in cuda version. Fall back to boosting on CPU.");
return new RegressionMAPELOSS(config);
} else if (type == std::string("gamma")) {
Log::Warning("Objective gamma is not implemented in cuda version. Fall back to boosting on CPU.");
return new RegressionGammaLoss(config);
} else if (type == std::string("tweedie")) {
Log::Warning("Objective tweedie is not implemented in cuda version. Fall back to boosting on CPU.");
return new RegressionTweedieLoss(config);
} else if (type == std::string("custom")) {
Log::Warning("Using customized objective with cuda. This requires copying gradients from CPU to GPU, which can be slow.");
return nullptr;
}
}
#endif // USE_CUDA
ObjectiveFunction* ObjectiveFunction::CreateObjectiveFunction(const std::string& type, const Config& config) {
#ifdef USE_CUDA
if (config.device_type == std::string("cuda") &&
config.data_sample_strategy != std::string("goss") &&
config.boosting != std::string("rf")) {
return CreateObjectiveFunctionCUDA(type, config);
} else {
#endif // USE_CUDA
if (type == std::string("regression")) {
return new RegressionL2loss(config);
} else if (type == std::string("regression_l1")) {
return new RegressionL1loss(config);
} else if (type == std::string("quantile")) {
return new RegressionQuantileloss(config);
} else if (type == std::string("huber")) {
return new RegressionHuberLoss(config);
} else if (type == std::string("fair")) {
return new RegressionFairLoss(config);
} else if (type == std::string("poisson")) {
return new RegressionPoissonLoss(config);
} else if (type == std::string("binary")) {
return new BinaryLogloss(config);
} else if (type == std::string("lambdarank")) {
return new LambdarankNDCG(config);
} else if (type == std::string("rank_xendcg")) {
return new RankXENDCG(config);
} else if (type == std::string("multiclass")) {
return new MulticlassSoftmax(config);
} else if (type == std::string("multiclassova")) {
return new MulticlassOVA(config);
} else if (type == std::string("cross_entropy")) {
return new CrossEntropy(config);
} else if (type == std::string("cross_entropy_lambda")) {
return new CrossEntropyLambda(config);
} else if (type == std::string("mape")) {
return new RegressionMAPELOSS(config);
} else if (type == std::string("gamma")) {
return new RegressionGammaLoss(config);
} else if (type == std::string("tweedie")) {
return new RegressionTweedieLoss(config);
} else if (type == std::string("custom")) {
return nullptr;
}
#ifdef USE_CUDA
}
#endif // USE_CUDA
Log::Fatal("Unknown objective type name: %s", type.c_str());
return nullptr;
}
ObjectiveFunction* ObjectiveFunction::CreateObjectiveFunction(const std::string& str) {
auto strs = Common::Split(str.c_str(), ' ');
auto type = strs[0];
if (type == std::string("regression")) {
return new RegressionL2loss(strs);
} else if (type == std::string("regression_l1")) {
return new RegressionL1loss(strs);
} else if (type == std::string("quantile")) {
return new RegressionQuantileloss(strs);
} else if (type == std::string("huber")) {
return new RegressionHuberLoss(strs);
} else if (type == std::string("fair")) {
return new RegressionFairLoss(strs);
} else if (type == std::string("poisson")) {
return new RegressionPoissonLoss(strs);
} else if (type == std::string("binary")) {
return new BinaryLogloss(strs);
} else if (type == std::string("lambdarank")) {
return new LambdarankNDCG(strs);
} else if (type == std::string("rank_xendcg")) {
return new RankXENDCG(strs);
} else if (type == std::string("multiclass")) {
return new MulticlassSoftmax(strs);
} else if (type == std::string("multiclassova")) {
return new MulticlassOVA(strs);
} else if (type == std::string("cross_entropy")) {
return new CrossEntropy(strs);
} else if (type == std::string("cross_entropy_lambda")) {
return new CrossEntropyLambda(strs);
} else if (type == std::string("mape")) {
return new RegressionMAPELOSS(strs);
} else if (type == std::string("gamma")) {
return new RegressionGammaLoss(strs);
} else if (type == std::string("tweedie")) {
return new RegressionTweedieLoss(strs);
} else if (type == std::string("custom")) {
return nullptr;
}
Log::Fatal("Unknown objective type name: %s", type.c_str());
return nullptr;
}
} // namespace LightGBM
src/objective/rank_objective.hpp
+466
View File
@@ -0,0 +1,466 @@
/*!
* Copyright (c) 2020-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2020-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_OBJECTIVE_RANK_OBJECTIVE_HPP_
#define LIGHTGBM_SRC_OBJECTIVE_RANK_OBJECTIVE_HPP_
#include <LightGBM/metric.h>
#include <LightGBM/objective_function.h>
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <limits>
#include <string>
#include <vector>
namespace LightGBM {
/*!
* \brief Objective function for Ranking
*/
class RankingObjective : public ObjectiveFunction {
public:
explicit RankingObjective(const Config& config)
: seed_(config.objective_seed) {
learning_rate_ = config.learning_rate;
position_bias_regularization_ = config.lambdarank_position_bias_regularization;
}
explicit RankingObjective(const std::vector<std::string>&) : seed_(0) {}
~RankingObjective() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
num_data_ = num_data;
// get label
label_ = metadata.label();
// get weights
weights_ = metadata.weights();
// get positions
positions_ = metadata.positions();
// get position ids
position_ids_ = metadata.position_ids();
// get number of different position ids
num_position_ids_ = static_cast<data_size_t>(metadata.num_position_ids());
// get boundaries
query_boundaries_ = metadata.query_boundaries();
if (query_boundaries_ == nullptr) {
Log::Fatal("Ranking tasks require query information");
}
num_queries_ = metadata.num_queries();
// initialize position bias vectors
pos_biases_.resize(num_position_ids_, 0.0);
}
void GetGradientsWithSampledQueries(const double* score, const data_size_t num_sampled_queries, const data_size_t* sampled_query_indices,
score_t* gradients, score_t* hessians) const override {
const data_size_t num_queries = (sampled_query_indices == nullptr ? num_queries_ : num_sampled_queries);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(guided)
for (data_size_t i = 0; i < num_queries; ++i) {
const data_size_t query_index = (sampled_query_indices == nullptr ? i : sampled_query_indices[i]);
const data_size_t start = query_boundaries_[query_index];
const data_size_t cnt = query_boundaries_[query_index + 1] - query_boundaries_[query_index];
std::vector<double> score_adjusted;
if (num_position_ids_ > 0) {
for (data_size_t j = 0; j < cnt; ++j) {
score_adjusted.push_back(score[start + j] + pos_biases_[positions_[start + j]]);
}
}
GetGradientsForOneQuery(query_index, cnt, label_ + start, num_position_ids_ > 0 ? score_adjusted.data() : score + start,
gradients + start, hessians + start);
if (weights_ != nullptr) {
for (data_size_t j = 0; j < cnt; ++j) {
gradients[start + j] =
static_cast<score_t>(gradients[start + j] * weights_[start + j]);
hessians[start + j] =
static_cast<score_t>(hessians[start + j] * weights_[start + j]);
}
}
}
if (num_position_ids_ > 0) {
UpdatePositionBiasFactors(gradients, hessians);
}
}
void GetGradients(const double* score, score_t* gradients, score_t* hessians) const override {
GetGradientsWithSampledQueries(score, num_queries_, nullptr, gradients, hessians);
}
virtual void GetGradientsForOneQuery(data_size_t query_id, data_size_t cnt,
const label_t* label,
const double* score, score_t* lambdas,
score_t* hessians) const = 0;
virtual void UpdatePositionBiasFactors(const score_t* /*lambdas*/, const score_t* /*hessians*/) const {}
const char* GetName() const override = 0;
std::string ToString() const override {
std::stringstream str_buf;
str_buf << GetName();
return str_buf.str();
}
bool NeedAccuratePrediction() const override { return false; }
protected:
int seed_;
data_size_t num_queries_;
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Pointer of weights */
const label_t* weights_;
/*! \brief Pointer of positions */
const data_size_t* positions_;
/*! \brief Pointer of position IDs */
const std::string* position_ids_;
/*! \brief Pointer of label */
data_size_t num_position_ids_;
/*! \brief Query boundaries */
const data_size_t* query_boundaries_;
/*! \brief Position bias factors */
mutable std::vector<label_t> pos_biases_;
/*! \brief Learning rate to update position bias factors */
double learning_rate_;
/*! \brief Position bias regularization */
double position_bias_regularization_;
};
/*!
* \brief Objective function for LambdaRank with NDCG
*/
class LambdarankNDCG : public RankingObjective {
public:
explicit LambdarankNDCG(const Config& config)
: RankingObjective(config),
sigmoid_(config.sigmoid),
norm_(config.lambdarank_norm),
truncation_level_(config.lambdarank_truncation_level) {
label_gain_ = config.label_gain;
// initialize DCG calculator
DCGCalculator::DefaultLabelGain(&label_gain_);
DCGCalculator::Init(label_gain_);
sigmoid_table_.clear();
inverse_max_dcgs_.clear();
if (sigmoid_ <= 0.0) {
Log::Fatal("Sigmoid param %f should be greater than zero", sigmoid_);
}
}
explicit LambdarankNDCG(const std::vector<std::string>& strs)
: RankingObjective(strs) {}
~LambdarankNDCG() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
RankingObjective::Init(metadata, num_data);
DCGCalculator::CheckMetadata(metadata, num_queries_);
DCGCalculator::CheckLabel(label_, num_data_);
inverse_max_dcgs_.resize(num_queries_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_queries_; ++i) {
inverse_max_dcgs_[i] = DCGCalculator::CalMaxDCGAtK(
truncation_level_, label_ + query_boundaries_[i],
query_boundaries_[i + 1] - query_boundaries_[i]);
if (inverse_max_dcgs_[i] > 0.0) {
inverse_max_dcgs_[i] = 1.0f / inverse_max_dcgs_[i];
}
}
// construct Sigmoid table to speed up Sigmoid transform
ConstructSigmoidTable();
}
inline void GetGradientsForOneQuery(data_size_t query_id, data_size_t cnt,
const label_t* label, const double* score,
score_t* lambdas,
score_t* hessians) const override {
// get max DCG on current query
const double inverse_max_dcg = inverse_max_dcgs_[query_id];
// initialize with zero
for (data_size_t i = 0; i < cnt; ++i) {
lambdas[i] = 0.0f;
hessians[i] = 0.0f;
}
// get sorted indices for scores
std::vector<data_size_t> sorted_idx(cnt);
for (data_size_t i = 0; i < cnt; ++i) {
sorted_idx[i] = i;
}
std::stable_sort(
sorted_idx.begin(), sorted_idx.end(),
[score](data_size_t a, data_size_t b) { return score[a] > score[b]; });
// get best and worst score
const double best_score = score[sorted_idx[0]];
data_size_t worst_idx = cnt - 1;
if (worst_idx > 0 && score[sorted_idx[worst_idx]] == kMinScore) {
worst_idx -= 1;
}
const double worst_score = score[sorted_idx[worst_idx]];
double sum_lambdas = 0.0;
// start accumulate lambdas by pairs that contain at least one document above truncation level
for (data_size_t i = 0; i < cnt - 1 && i < truncation_level_; ++i) {
if (score[sorted_idx[i]] == kMinScore) {
continue;
}
for (data_size_t j = i + 1; j < cnt; ++j) {
if (score[sorted_idx[j]] == kMinScore) {
continue;
}
// skip pairs with the same labels
if (label[sorted_idx[i]] == label[sorted_idx[j]]) {
continue;
}
data_size_t high_rank, low_rank;
if (label[sorted_idx[i]] > label[sorted_idx[j]]) {
high_rank = i;
low_rank = j;
} else {
high_rank = j;
low_rank = i;
}
const data_size_t high = sorted_idx[high_rank];
const int high_label = static_cast<int>(label[high]);
const double high_score = score[high];
const double high_label_gain = label_gain_[high_label];
const double high_discount = DCGCalculator::GetDiscount(high_rank);
const data_size_t low = sorted_idx[low_rank];
const int low_label = static_cast<int>(label[low]);
const double low_score = score[low];
const double low_label_gain = label_gain_[low_label];
const double low_discount = DCGCalculator::GetDiscount(low_rank);
const double delta_score = high_score - low_score;
// get dcg gap
const double dcg_gap = high_label_gain - low_label_gain;
// get discount of this pair
const double paired_discount = fabs(high_discount - low_discount);
// get delta NDCG
double delta_pair_NDCG = dcg_gap * paired_discount * inverse_max_dcg;
// regular the delta_pair_NDCG by score distance
if (norm_ && best_score != worst_score) {
delta_pair_NDCG /= (0.01f + fabs(delta_score));
}
// calculate lambda for this pair
double p_lambda = GetSigmoid(delta_score);
double p_hessian = p_lambda * (1.0f - p_lambda);
// update
p_lambda *= -sigmoid_ * delta_pair_NDCG;
p_hessian *= sigmoid_ * sigmoid_ * delta_pair_NDCG;
lambdas[low] -= static_cast<score_t>(p_lambda);
hessians[low] += static_cast<score_t>(p_hessian);
lambdas[high] += static_cast<score_t>(p_lambda);
hessians[high] += static_cast<score_t>(p_hessian);
// lambda is negative, so use minus to accumulate
sum_lambdas -= 2 * p_lambda;
}
}
if (norm_ && sum_lambdas > 0) {
double norm_factor = std::log2(1 + sum_lambdas) / sum_lambdas;
for (data_size_t i = 0; i < cnt; ++i) {
lambdas[i] = static_cast<score_t>(lambdas[i] * norm_factor);
hessians[i] = static_cast<score_t>(hessians[i] * norm_factor);
}
}
}
inline double GetSigmoid(double score) const {
if (score <= min_sigmoid_input_) {
// too small, use lower bound
return sigmoid_table_[0];
} else if (score >= max_sigmoid_input_) {
// too large, use upper bound
return sigmoid_table_[_sigmoid_bins - 1];
} else {
return sigmoid_table_[static_cast<size_t>((score - min_sigmoid_input_) *
sigmoid_table_idx_factor_)];
}
}
void ConstructSigmoidTable() {
// get boundary
min_sigmoid_input_ = min_sigmoid_input_ / sigmoid_ / 2;
max_sigmoid_input_ = -min_sigmoid_input_;
sigmoid_table_.resize(_sigmoid_bins);
// get score to bin factor
sigmoid_table_idx_factor_ =
_sigmoid_bins / (max_sigmoid_input_ - min_sigmoid_input_);
// cache
for (size_t i = 0; i < _sigmoid_bins; ++i) {
const double score = i / sigmoid_table_idx_factor_ + min_sigmoid_input_;
sigmoid_table_[i] = 1.0f / (1.0f + std::exp(score * sigmoid_));
}
}
void UpdatePositionBiasFactors(const score_t* lambdas, const score_t* hessians) const override {
/// get number of threads
int num_threads = OMP_NUM_THREADS();
// create per-thread buffers for first and second derivatives of utility w.r.t. position bias factors
std::vector<double> bias_first_derivatives(num_position_ids_ * num_threads, 0.0);
std::vector<double> bias_second_derivatives(num_position_ids_ * num_threads, 0.0);
std::vector<int> instance_counts(num_position_ids_ * num_threads, 0);
#pragma omp parallel for schedule(guided) num_threads(num_threads)
for (data_size_t i = 0; i < num_data_; i++) {
// get thread ID
const int tid = omp_get_thread_num();
size_t offset = static_cast<size_t>(positions_[i] + tid * num_position_ids_);
// accumulate first derivatives of utility w.r.t. position bias factors, for each position
bias_first_derivatives[offset] -= lambdas[i];
// accumulate second derivatives of utility w.r.t. position bias factors, for each position
bias_second_derivatives[offset] -= hessians[i];
instance_counts[offset]++;
}
#pragma omp parallel for schedule(guided) num_threads(num_threads)
for (data_size_t i = 0; i < num_position_ids_; i++) {
double bias_first_derivative = 0.0;
double bias_second_derivative = 0.0;
int instance_count = 0;
// aggregate derivatives from per-thread buffers
for (int tid = 0; tid < num_threads; tid++) {
size_t offset = static_cast<size_t>(i + tid * num_position_ids_);
bias_first_derivative += bias_first_derivatives[offset];
bias_second_derivative += bias_second_derivatives[offset];
instance_count += instance_counts[offset];
}
// L2 regularization on position bias factors
bias_first_derivative -= pos_biases_[i] * position_bias_regularization_ * instance_count;
bias_second_derivative -= position_bias_regularization_ * instance_count;
// do Newton-Raphson step to update position bias factors
pos_biases_[i] += learning_rate_ * bias_first_derivative / (std::abs(bias_second_derivative) + 0.001);
}
LogDebugPositionBiasFactors();
}
const char* GetName() const override { return "lambdarank"; }
protected:
void LogDebugPositionBiasFactors() const {
std::stringstream message_stream;
message_stream << std::setw(15) << "position"
<< std::setw(15) << "bias_factor"
<< std::endl;
Log::Debug(message_stream.str().c_str());
message_stream.str("");
for (int i = 0; i < num_position_ids_; ++i) {
message_stream << std::setw(15) << position_ids_[i]
<< std::setw(15) << pos_biases_[i];
Log::Debug(message_stream.str().c_str());
message_stream.str("");
}
}
/*! \brief Sigmoid param */
double sigmoid_;
/*! \brief Normalize the lambdas or not */
bool norm_;
/*! \brief Truncation position for max DCG */
int truncation_level_;
/*! \brief Cache inverse max DCG, speed up calculation */
std::vector<double> inverse_max_dcgs_;
/*! \brief Cache result for sigmoid transform to speed up */
std::vector<double> sigmoid_table_;
/*! \brief Gains for labels */
std::vector<double> label_gain_;
/*! \brief Number of bins in simoid table */
size_t _sigmoid_bins = 1024 * 1024;
/*! \brief Minimal input of sigmoid table */
double min_sigmoid_input_ = -50;
/*! \brief Maximal input of Sigmoid table */
double max_sigmoid_input_ = 50;
/*! \brief Factor that covert score to bin in Sigmoid table */
double sigmoid_table_idx_factor_;
};
/*!
* \brief Implementation of the learning-to-rank objective function, XE_NDCG
* [arxiv.org/abs/1911.09798].
*/
class RankXENDCG : public RankingObjective {
public:
explicit RankXENDCG(const Config& config) : RankingObjective(config) {}
explicit RankXENDCG(const std::vector<std::string>& strs)
: RankingObjective(strs) {}
~RankXENDCG() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
RankingObjective::Init(metadata, num_data);
for (data_size_t i = 0; i < num_queries_; ++i) {
rands_.emplace_back(seed_ + i);
}
}
inline void GetGradientsForOneQuery(data_size_t query_id, data_size_t cnt,
const label_t* label, const double* score,
score_t* lambdas,
score_t* hessians) const override {
// Skip groups with too few items.
if (cnt <= 1) {
for (data_size_t i = 0; i < cnt; ++i) {
lambdas[i] = 0.0f;
hessians[i] = 0.0f;
}
return;
}
// Turn scores into a probability distribution using Softmax.
std::vector<double> rho(cnt, 0.0);
Common::Softmax(score, rho.data(), cnt);
// An auxiliary buffer of parameters used to form the ground-truth
// distribution and compute the loss.
std::vector<double> params(cnt);
double inv_denominator = 0;
for (data_size_t i = 0; i < cnt; ++i) {
params[i] = Phi(label[i], rands_[query_id].NextFloat());
inv_denominator += params[i];
}
// sum_labels will always be positive number
inv_denominator = 1. / std::max<double>(kEpsilon, inv_denominator);
// Approximate gradients and inverse Hessian.
// First order terms.
double sum_l1 = 0.0;
for (data_size_t i = 0; i < cnt; ++i) {
double term = -params[i] * inv_denominator + rho[i];
lambdas[i] = static_cast<score_t>(term);
// Params will now store terms needed to compute second-order terms.
params[i] = term / (1. - rho[i]);
sum_l1 += params[i];
}
// Second order terms.
double sum_l2 = 0.0;
for (data_size_t i = 0; i < cnt; ++i) {
double term = rho[i] * (sum_l1 - params[i]);
lambdas[i] += static_cast<score_t>(term);
// Params will now store terms needed to compute third-order terms.
params[i] = term / (1. - rho[i]);
sum_l2 += params[i];
}
for (data_size_t i = 0; i < cnt; ++i) {
lambdas[i] += static_cast<score_t>(rho[i] * (sum_l2 - params[i]));
hessians[i] = static_cast<score_t>(rho[i] * (1.0 - rho[i]));
}
}
double Phi(const label_t l, double g) const {
return Common::Pow(2, static_cast<int>(l)) - g;
}
const char* GetName() const override { return "rank_xendcg"; }
protected:
mutable std::vector<Random> rands_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_OBJECTIVE_RANK_OBJECTIVE_HPP_
src/objective/regression_objective.hpp
+764
View File
@@ -0,0 +1,764 @@
/*!
* Copyright (c) 2016-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2016-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_
#define LIGHTGBM_SRC_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_
#include <LightGBM/meta.h>
#include <LightGBM/objective_function.h>
#include <LightGBM/utils/array_args.h>
#include <string>
#include <algorithm>
#include <vector>
namespace LightGBM {
#define PercentileFun(T, data_reader, cnt_data, alpha) \
{ \
if (cnt_data <= 1) { \
return data_reader(0); \
} \
std::vector<T> ref_data(cnt_data); \
for (data_size_t i = 0; i < cnt_data; ++i) { \
ref_data[i] = data_reader(i); \
} \
const double float_pos = static_cast<double>(cnt_data - 1) * (1.0 - alpha); \
const data_size_t pos = static_cast<data_size_t>(float_pos) + 1; \
if (pos < 1) { \
return ref_data[ArrayArgs<T>::ArgMax(ref_data)]; \
} else if (pos >= cnt_data) { \
return ref_data[ArrayArgs<T>::ArgMin(ref_data)]; \
} else { \
const double bias = float_pos - (pos - 1); \
if (pos > cnt_data / 2) { \
ArrayArgs<T>::ArgMaxAtK(&ref_data, 0, cnt_data, pos - 1); \
T v1 = ref_data[pos - 1]; \
T v2 = ref_data[pos + ArrayArgs<T>::ArgMax(ref_data.data() + pos, \
cnt_data - pos)]; \
return static_cast<T>(v1 - (v1 - v2) * bias); \
} else { \
ArrayArgs<T>::ArgMaxAtK(&ref_data, 0, cnt_data, pos); \
T v2 = ref_data[pos]; \
T v1 = ref_data[ArrayArgs<T>::ArgMin(ref_data.data(), pos)]; \
return static_cast<T>(v1 - (v1 - v2) * bias); \
} \
} \
}\
#define WeightedPercentileFun(T, data_reader, weight_reader, cnt_data, alpha) \
{ \
if (cnt_data <= 1) { \
return data_reader(0); \
} \
std::vector<data_size_t> sorted_idx(cnt_data); \
for (data_size_t i = 0; i < cnt_data; ++i) { \
sorted_idx[i] = i; \
} \
std::stable_sort(sorted_idx.begin(), sorted_idx.end(), \
[&](data_size_t a, data_size_t b) { \
return data_reader(a) < data_reader(b); \
}); \
std::vector<double> weighted_cdf(cnt_data); \
weighted_cdf[0] = weight_reader(sorted_idx[0]); \
for (data_size_t i = 1; i < cnt_data; ++i) { \
weighted_cdf[i] = weighted_cdf[i - 1] + weight_reader(sorted_idx[i]); \
} \
double threshold = weighted_cdf[cnt_data - 1] * alpha; \
size_t pos = std::upper_bound(weighted_cdf.begin(), weighted_cdf.end(), \
threshold) - \
weighted_cdf.begin(); \
pos = std::min(pos, static_cast<size_t>(cnt_data - 1)); \
if (pos == 0 || pos == static_cast<size_t>(cnt_data - 1)) { \
return data_reader(sorted_idx[pos]); \
} \
CHECK_GE(threshold, weighted_cdf[pos - 1]); \
CHECK_LT(threshold, weighted_cdf[pos]); \
T v1 = data_reader(sorted_idx[pos - 1]); \
T v2 = data_reader(sorted_idx[pos]); \
if (weighted_cdf[pos] - weighted_cdf[pos - 1] >= 1.0) { \
return static_cast<T>((threshold - weighted_cdf[pos - 1]) / \
(weighted_cdf[pos] - weighted_cdf[pos - 1]) * \
(v2 - v1) + \
v1); \
} else { \
return static_cast<T>(v1); \
} \
}\
/*!
* \brief Objective function for regression
*/
class RegressionL2loss: public ObjectiveFunction {
public:
explicit RegressionL2loss(const Config& config)
: deterministic_(config.deterministic) {
sqrt_ = config.reg_sqrt;
}
explicit RegressionL2loss(const std::vector<std::string>& strs)
: deterministic_(false) {
sqrt_ = false;
for (auto str : strs) {
if (str == std::string("sqrt")) {
sqrt_ = true;
}
}
}
~RegressionL2loss() {
}
void Init(const Metadata& metadata, data_size_t num_data) override {
num_data_ = num_data;
label_ = metadata.label();
if (sqrt_) {
trans_label_.resize(num_data_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data; ++i) {
trans_label_[i] = Common::Sign(label_[i]) * std::sqrt(std::fabs(label_[i]));
}
label_ = trans_label_.data();
}
weights_ = metadata.weights();
}
void GetGradients(const double* score, score_t* gradients,
score_t* hessians) const override {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
gradients[i] = static_cast<score_t>(score[i] - label_[i]);
hessians[i] = 1.0f;
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
gradients[i] = static_cast<score_t>(static_cast<score_t>((score[i] - label_[i])) * weights_[i]);
hessians[i] = static_cast<score_t>(weights_[i]);
}
}
}
const char* GetName() const override {
return "regression";
}
void ConvertOutput(const double* input, double* output) const override {
if (sqrt_) {
output[0] = Common::Sign(input[0]) * input[0] * input[0];
} else {
output[0] = input[0];
}
}
std::string ToString() const override {
std::stringstream str_buf;
str_buf << GetName();
if (sqrt_) {
str_buf << " sqrt";
}
return str_buf.str();
}
bool IsConstantHessian() const override {
if (weights_ == nullptr) {
return true;
} else {
return false;
}
}
double BoostFromScore(int) const override {
double suml = 0.0f;
double sumw = 0.0f;
if (weights_ != nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:suml, sumw) if (!deterministic_)
for (data_size_t i = 0; i < num_data_; ++i) {
suml += static_cast<double>(label_[i]) * weights_[i];
sumw += weights_[i];
}
} else {
sumw = static_cast<double>(num_data_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:suml) if (!deterministic_)
for (data_size_t i = 0; i < num_data_; ++i) {
suml += label_[i];
}
}
return suml / sumw;
}
protected:
bool sqrt_;
/*! \brief Number of data */
data_size_t num_data_;
/*! \brief Pointer of label */
const label_t* label_;
/*! \brief Pointer of weights */
const label_t* weights_;
std::vector<label_t> trans_label_;
const bool deterministic_;
};
/*!
* \brief L1 regression loss
*/
class RegressionL1loss: public RegressionL2loss {
public:
explicit RegressionL1loss(const Config& config): RegressionL2loss(config) {
}
explicit RegressionL1loss(const std::vector<std::string>& strs): RegressionL2loss(strs) {
}
~RegressionL1loss() {}
void GetGradients(const double* score, score_t* gradients,
score_t* hessians) const override {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double diff = score[i] - label_[i];
gradients[i] = static_cast<score_t>(Common::Sign(diff));
hessians[i] = 1.0f;
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double diff = score[i] - label_[i];
gradients[i] = static_cast<score_t>(Common::Sign(diff) * weights_[i]);
hessians[i] = weights_[i];
}
}
}
double BoostFromScore(int) const override {
const double alpha = 0.5;
if (weights_ != nullptr) {
#define data_reader(i) (label_[i])
#define weight_reader(i) (weights_[i])
WeightedPercentileFun(label_t, data_reader, weight_reader, num_data_, alpha);
#undef data_reader
#undef weight_reader
} else {
#define data_reader(i) (label_[i])
PercentileFun(label_t, data_reader, num_data_, alpha);
#undef data_reader
}
}
bool IsRenewTreeOutput() const override { return true; }
double RenewTreeOutput(double, std::function<double(const label_t*, int)> residual_getter,
const data_size_t* index_mapper,
const data_size_t* bagging_mapper,
data_size_t num_data_in_leaf) const override {
const double alpha = 0.5;
if (weights_ == nullptr) {
if (bagging_mapper == nullptr) {
#define data_reader(i) (residual_getter(label_, index_mapper[i]))
PercentileFun(double, data_reader, num_data_in_leaf, alpha);
#undef data_reader
} else {
#define data_reader(i) (residual_getter(label_, bagging_mapper[index_mapper[i]]))
PercentileFun(double, data_reader, num_data_in_leaf, alpha);
#undef data_reader
}
} else {
if (bagging_mapper == nullptr) {
#define data_reader(i) (residual_getter(label_, index_mapper[i]))
#define weight_reader(i) (weights_[index_mapper[i]])
WeightedPercentileFun(double, data_reader, weight_reader, num_data_in_leaf, alpha);
#undef data_reader
#undef weight_reader
} else {
#define data_reader(i) (residual_getter(label_, bagging_mapper[index_mapper[i]]))
#define weight_reader(i) (weights_[bagging_mapper[index_mapper[i]]])
WeightedPercentileFun(double, data_reader, weight_reader, num_data_in_leaf, alpha);
#undef data_reader
#undef weight_reader
}
}
}
const char* GetName() const override {
return "regression_l1";
}
};
/*!
* \brief Huber regression loss
*/
class RegressionHuberLoss: public RegressionL2loss {
public:
explicit RegressionHuberLoss(const Config& config): RegressionL2loss(config) {
alpha_ = static_cast<double>(config.alpha);
if (sqrt_) {
Log::Warning("Cannot use sqrt transform in %s Regression, will auto disable it", GetName());
sqrt_ = false;
}
}
explicit RegressionHuberLoss(const std::vector<std::string>& strs): RegressionL2loss(strs) {
if (sqrt_) {
Log::Warning("Cannot use sqrt transform in %s Regression, will auto disable it", GetName());
sqrt_ = false;
}
}
~RegressionHuberLoss() {
}
void GetGradients(const double* score, score_t* gradients,
score_t* hessians) const override {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double diff = score[i] - label_[i];
if (std::abs(diff) <= alpha_) {
gradients[i] = static_cast<score_t>(diff);
} else {
gradients[i] = static_cast<score_t>(Common::Sign(diff) * alpha_);
}
hessians[i] = 1.0f;
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double diff = score[i] - label_[i];
if (std::abs(diff) <= alpha_) {
gradients[i] = static_cast<score_t>(diff * weights_[i]);
} else {
gradients[i] = static_cast<score_t>(Common::Sign(diff) * static_cast<score_t>(weights_[i]) * alpha_);
}
hessians[i] = static_cast<score_t>(weights_[i]);
}
}
}
const char* GetName() const override {
return "huber";
}
protected:
/*! \brief delta for Huber loss */
double alpha_;
};
// http://research.microsoft.com/en-us/um/people/zhang/INRIA/Publis/Tutorial-Estim/node24.html
class RegressionFairLoss: public RegressionL2loss {
public:
explicit RegressionFairLoss(const Config& config): RegressionL2loss(config) {
c_ = static_cast<double>(config.fair_c);
}
explicit RegressionFairLoss(const std::vector<std::string>& strs): RegressionL2loss(strs) {
}
~RegressionFairLoss() {}
void GetGradients(const double* score, score_t* gradients,
score_t* hessians) const override {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double x = score[i] - label_[i];
gradients[i] = static_cast<score_t>(c_ * x / (std::fabs(x) + c_));
hessians[i] = static_cast<score_t>(c_ * c_ / ((std::fabs(x) + c_) * (std::fabs(x) + c_)));
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double x = score[i] - label_[i];
gradients[i] = static_cast<score_t>(c_ * x / (std::fabs(x) + c_) * weights_[i]);
hessians[i] = static_cast<score_t>(c_ * c_ / ((std::fabs(x) + c_) * (std::fabs(x) + c_)) * weights_[i]);
}
}
}
const char* GetName() const override {
return "fair";
}
bool IsConstantHessian() const override {
return false;
}
protected:
/*! \brief c for Fair loss */
double c_;
};
/*!
* \brief Objective function for Poisson regression
*/
class RegressionPoissonLoss: public RegressionL2loss {
public:
explicit RegressionPoissonLoss(const Config& config): RegressionL2loss(config) {
max_delta_step_ = static_cast<double>(config.poisson_max_delta_step);
if (sqrt_) {
Log::Warning("Cannot use sqrt transform in %s Regression, will auto disable it", GetName());
sqrt_ = false;
}
}
explicit RegressionPoissonLoss(const std::vector<std::string>& strs): RegressionL2loss(strs) {
}
~RegressionPoissonLoss() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
if (sqrt_) {
Log::Warning("Cannot use sqrt transform in %s Regression, will auto disable it", GetName());
sqrt_ = false;
}
RegressionL2loss::Init(metadata, num_data);
// Safety check of labels
label_t miny;
double sumy;
Common::ObtainMinMaxSum(label_, num_data_, &miny, static_cast<label_t*>(nullptr), &sumy);
if (miny < 0.0f) {
Log::Fatal("[%s]: at least one target label is negative", GetName());
}
if (sumy == 0.0f) {
Log::Fatal("[%s]: sum of labels is zero", GetName());
}
}
/* Parametrize with unbounded internal score "f"; then
* loss = exp(f) - label * f
* grad = exp(f) - label
* hess = exp(f)
*
* And the output is exp(f); so the associated metric get s=exp(f)
* so that its loss = s - label * log(s); a little awkward maybe.
*
*/
void GetGradients(const double* score, score_t* gradients,
score_t* hessians) const override {
double exp_max_delta_step_ = std::exp(max_delta_step_);
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
double exp_score = std::exp(score[i]);
gradients[i] = static_cast<score_t>(exp_score - label_[i]);
hessians[i] = static_cast<score_t>(exp_score * exp_max_delta_step_);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
double exp_score = std::exp(score[i]);
gradients[i] = static_cast<score_t>((exp_score - label_[i]) * weights_[i]);
hessians[i] = static_cast<score_t>(exp_score * exp_max_delta_step_ * weights_[i]);
}
}
}
void ConvertOutput(const double* input, double* output) const override {
output[0] = std::exp(input[0]);
}
const char* GetName() const override {
return "poisson";
}
double BoostFromScore(int) const override {
return Common::SafeLog(RegressionL2loss::BoostFromScore(0));
}
bool IsConstantHessian() const override {
return false;
}
protected:
/*! \brief used to safeguard optimization */
double max_delta_step_;
};
class RegressionQuantileloss : public RegressionL2loss {
public:
explicit RegressionQuantileloss(const Config& config): RegressionL2loss(config) {
alpha_ = static_cast<score_t>(config.alpha);
CHECK(alpha_ > 0 && alpha_ < 1);
}
explicit RegressionQuantileloss(const std::vector<std::string>& strs): RegressionL2loss(strs) {
}
~RegressionQuantileloss() {}
void GetGradients(const double* score, score_t* gradients,
score_t* hessians) const override {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
score_t delta = static_cast<score_t>(score[i] - label_[i]);
if (delta >= 0) {
gradients[i] = (1.0f - alpha_);
} else {
gradients[i] = -alpha_;
}
hessians[i] = 1.0f;
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
score_t delta = static_cast<score_t>(score[i] - label_[i]);
if (delta >= 0) {
gradients[i] = static_cast<score_t>((1.0f - alpha_) * weights_[i]);
} else {
gradients[i] = static_cast<score_t>(-alpha_ * weights_[i]);
}
hessians[i] = static_cast<score_t>(weights_[i]);
}
}
}
const char* GetName() const override {
return "quantile";
}
double BoostFromScore(int) const override {
if (weights_ != nullptr) {
#define data_reader(i) (label_[i])
#define weight_reader(i) (weights_[i])
WeightedPercentileFun(label_t, data_reader, weight_reader, num_data_, alpha_);
#undef data_reader
#undef weight_reader
} else {
#define data_reader(i) (label_[i])
PercentileFun(label_t, data_reader, num_data_, alpha_);
#undef data_reader
}
}
bool IsRenewTreeOutput() const override { return true; }
double RenewTreeOutput(double, std::function<double(const label_t*, int)> residual_getter,
const data_size_t* index_mapper,
const data_size_t* bagging_mapper,
data_size_t num_data_in_leaf) const override {
if (weights_ == nullptr) {
if (bagging_mapper == nullptr) {
#define data_reader(i) (residual_getter(label_, index_mapper[i]))
PercentileFun(double, data_reader, num_data_in_leaf, alpha_);
#undef data_reader
} else {
#define data_reader(i) (residual_getter(label_, bagging_mapper[index_mapper[i]]))
PercentileFun(double, data_reader, num_data_in_leaf, alpha_);
#undef data_reader
}
} else {
if (bagging_mapper == nullptr) {
#define data_reader(i) (residual_getter(label_, index_mapper[i]))
#define weight_reader(i) (weights_[index_mapper[i]])
WeightedPercentileFun(double, data_reader, weight_reader, num_data_in_leaf, alpha_);
#undef data_reader
#undef weight_reader
} else {
#define data_reader(i) (residual_getter(label_, bagging_mapper[index_mapper[i]]))
#define weight_reader(i) (weights_[bagging_mapper[index_mapper[i]]])
WeightedPercentileFun(double, data_reader, weight_reader, num_data_in_leaf, alpha_);
#undef data_reader
#undef weight_reader
}
}
}
protected:
score_t alpha_;
};
/*!
* \brief MAPE Regression Loss
*/
class RegressionMAPELOSS : public RegressionL1loss {
public:
explicit RegressionMAPELOSS(const Config& config) : RegressionL1loss(config) {
}
explicit RegressionMAPELOSS(const std::vector<std::string>& strs) : RegressionL1loss(strs) {
}
~RegressionMAPELOSS() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
RegressionL2loss::Init(metadata, num_data);
for (data_size_t i = 0; i < num_data_; ++i) {
if (std::fabs(label_[i]) < 1) {
Log::Warning(
"Some label values are < 1 in absolute value. MAPE is unstable with such values, "
"so LightGBM rounds them to 1.0 when calculating MAPE.");
break;
}
}
label_weight_.resize(num_data);
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
label_weight_[i] = 1.0f / std::max(1.0f, std::fabs(label_[i]));
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
label_weight_[i] = 1.0f / std::max(1.0f, std::fabs(label_[i])) * weights_[i];
}
}
}
void GetGradients(const double* score, score_t* gradients,
score_t* hessians) const override {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double diff = score[i] - label_[i];
gradients[i] = static_cast<score_t>(Common::Sign(diff) * label_weight_[i]);
hessians[i] = 1.0f;
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double diff = score[i] - label_[i];
gradients[i] = static_cast<score_t>(Common::Sign(diff) * label_weight_[i]);
hessians[i] = weights_[i];
}
}
}
double BoostFromScore(int) const override {
const double alpha = 0.5;
#define data_reader(i) (label_[i])
#define weight_reader(i) (label_weight_[i])
WeightedPercentileFun(label_t, data_reader, weight_reader, num_data_, alpha);
#undef data_reader
#undef weight_reader
}
bool IsRenewTreeOutput() const override { return true; }
double RenewTreeOutput(double, std::function<double(const label_t*, int)> residual_getter,
const data_size_t* index_mapper,
const data_size_t* bagging_mapper,
data_size_t num_data_in_leaf) const override {
const double alpha = 0.5;
if (bagging_mapper == nullptr) {
#define data_reader(i) (residual_getter(label_, index_mapper[i]))
#define weight_reader(i) (label_weight_[index_mapper[i]])
WeightedPercentileFun(double, data_reader, weight_reader, num_data_in_leaf, alpha);
#undef data_reader
#undef weight_reader
} else {
#define data_reader(i) (residual_getter(label_, bagging_mapper[index_mapper[i]]))
#define weight_reader(i) (label_weight_[bagging_mapper[index_mapper[i]]])
WeightedPercentileFun(double, data_reader, weight_reader, num_data_in_leaf, alpha);
#undef data_reader
#undef weight_reader
}
}
const char* GetName() const override {
return "mape";
}
bool IsConstantHessian() const override {
return true;
}
private:
std::vector<label_t> label_weight_;
};
/*!
* \brief Objective function for Gamma regression
*/
class RegressionGammaLoss : public RegressionPoissonLoss {
public:
explicit RegressionGammaLoss(const Config& config) : RegressionPoissonLoss(config) {
}
explicit RegressionGammaLoss(const std::vector<std::string>& strs) : RegressionPoissonLoss(strs) {
}
~RegressionGammaLoss() {}
void GetGradients(const double* score, score_t* gradients,
score_t* hessians) const override {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
double exp_score = std::exp(-score[i]);
gradients[i] = static_cast<score_t>(1.0 - label_[i] * exp_score);
hessians[i] = static_cast<score_t>(label_[i] * exp_score);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
double exp_score = std::exp(-score[i]);
gradients[i] = static_cast<score_t>((1.0 - label_[i] * exp_score) * weights_[i]);
hessians[i] = static_cast<score_t>(label_[i] * exp_score * weights_[i]);
}
}
}
const char* GetName() const override {
return "gamma";
}
};
/*!
* \brief Objective function for Tweedie regression
*/
class RegressionTweedieLoss: public RegressionPoissonLoss {
public:
explicit RegressionTweedieLoss(const Config& config) : RegressionPoissonLoss(config) {
rho_ = config.tweedie_variance_power;
}
explicit RegressionTweedieLoss(const std::vector<std::string>& strs) : RegressionPoissonLoss(strs) {
}
~RegressionTweedieLoss() {}
void GetGradients(const double* score, score_t* gradients,
score_t* hessians) const override {
if (weights_ == nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
double exp_1_score = std::exp((1 - rho_) * score[i]);
double exp_2_score = std::exp((2 - rho_) * score[i]);
gradients[i] = static_cast<score_t>(-label_[i] * exp_1_score + exp_2_score);
hessians[i] = static_cast<score_t>(-label_[i] * (1 - rho_) * exp_1_score +
(2 - rho_) * exp_2_score);
}
} else {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
double exp_1_score = std::exp((1 - rho_) * score[i]);
double exp_2_score = std::exp((2 - rho_) * score[i]);
gradients[i] = static_cast<score_t>((-label_[i] * exp_1_score + exp_2_score) * weights_[i]);
hessians[i] = static_cast<score_t>((-label_[i] * (1 - rho_) * exp_1_score +
(2 - rho_) * exp_2_score) * weights_[i]);
}
}
}
const char* GetName() const override {
return "tweedie";
}
private:
double rho_;
};
#undef PercentileFun
#undef WeightedPercentileFun
} // namespace LightGBM
#endif // LIGHTGBM_SRC_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_
src/objective/xentropy_objective.hpp
+317
View File
@@ -0,0 +1,317 @@
/*!
* Copyright (c) 2017-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2017-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for license information.
*/
#ifndef LIGHTGBM_SRC_OBJECTIVE_XENTROPY_OBJECTIVE_HPP_
#define LIGHTGBM_SRC_OBJECTIVE_XENTROPY_OBJECTIVE_HPP_
#include <LightGBM/meta.h>
#include <LightGBM/objective_function.h>
#include <LightGBM/utils/common.h>
#include <string>
#include <algorithm>
#include <cmath>
#include <cstring>
#include <vector>
/*
* Implements gradients and Hessians for the following point losses.
* Target y is anything in interval [0, 1].
*
* (1) CrossEntropy; "xentropy";
*
* loss(y, p, w) = { -(1-y)*log(1-p)-y*log(p) }*w,
* with probability p = 1/(1+exp(-f)), where f is being boosted
*
* ConvertToOutput: f -> p
*
* (2) CrossEntropyLambda; "xentlambda"
*
* loss(y, p, w) = -(1-y)*log(1-p)-y*log(p),
* with p = 1-exp(-lambda*w), lambda = log(1+exp(f)), f being boosted, and w > 0
*
* ConvertToOutput: f -> lambda
*
* (1) and (2) are the same if w=1; but outputs still differ.
*
*/
namespace LightGBM {
/*!
* \brief Objective function for cross-entropy (with optional linear weights)
*/
class CrossEntropy: public ObjectiveFunction {
public:
explicit CrossEntropy(const Config& config)
: deterministic_(config.deterministic) {}
explicit CrossEntropy(const std::vector<std::string>&)
: deterministic_(false) {
}
~CrossEntropy() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
num_data_ = num_data;
label_ = metadata.label();
weights_ = metadata.weights();
CHECK_NOTNULL(label_);
Common::CheckElementsIntervalClosed<label_t>(label_, 0.0f, 1.0f, num_data_, GetName());
Log::Info("[%s:%s]: (objective) labels passed interval [0, 1] check", GetName(), __func__);
if (weights_ != nullptr) {
label_t minw;
double sumw;
Common::ObtainMinMaxSum(weights_, num_data_, &minw, static_cast<label_t*>(nullptr), &sumw);
if (minw < 0.0f) {
Log::Fatal("[%s]: at least one weight is negative", GetName());
}
if (sumw == 0.0f) {
Log::Fatal("[%s]: sum of weights is zero", GetName());
}
}
}
void GetGradients(const double* score, score_t* gradients, score_t* hessians) const override {
// z = expit(score) = 1 / (1 + exp(-score))
// gradient = z - label = expit(score) - label
// Numerically more stable, see http://fa.bianp.net/blog/2019/evaluate_logistic/
// if score < 0:
// exp_tmp = exp(score)
// return ((1 - label) * exp_tmp - label) / (1 + exp_tmp)
// else:
// exp_tmp = exp(-score)
// return ((1 - label) - label * exp_tmp) / (1 + exp_tmp)
// Note that optimal speed would be achieved, at the cost of precision, by
// return expit(score) - y_true
// i.e. no "if else" and an own inline implementation of expit.
// The case distinction score < 0 in the stable implementation does not
// provide significant better precision apart from protecting overflow of exp(..).
// The branch (if else), however, can incur runtime costs of up to 30%.
// Instead, we help branch prediction by almost always ending in the first if clause
// and making the second branch (else) a bit simpler. This has the exact same
// precision but is faster than the stable implementation.
// As branching criteria, we use the same cutoff as in log1pexp, see link above.
// Note that the maximal value to get gradient = -1 with label = 1 is -37.439198610162731
// (based on mpmath), and scipy.special.logit(np.finfo(float).eps) ~ -36.04365.
if (weights_ == nullptr) {
// compute pointwise gradients and Hessians with implied unit weights
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
if (score[i] > -37.0) {
const double exp_tmp = std::exp(-score[i]);
gradients[i] = static_cast<score_t>(((1.0f - label_[i]) - label_[i] * exp_tmp) / (1.0f + exp_tmp));
hessians[i] = static_cast<score_t>(exp_tmp / ((1 + exp_tmp) * (1 + exp_tmp)));
} else {
const double exp_tmp = std::exp(score[i]);
gradients[i] = static_cast<score_t>(exp_tmp - label_[i]);
hessians[i] = static_cast<score_t>(exp_tmp);
}
}
} else {
// compute pointwise gradients and Hessians with given weights
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
if (score[i] > -37.0) {
const double exp_tmp = std::exp(-score[i]);
gradients[i] = static_cast<score_t>(((1.0f - label_[i]) - label_[i] * exp_tmp) / (1.0f + exp_tmp) * weights_[i]);
hessians[i] = static_cast<score_t>(exp_tmp / ((1 + exp_tmp) * (1 + exp_tmp)) * weights_[i]);
} else {
const double exp_tmp = std::exp(score[i]);
gradients[i] = static_cast<score_t>((exp_tmp - label_[i]) * weights_[i]);
hessians[i] = static_cast<score_t>(exp_tmp * weights_[i]);
}
}
}
}
const char* GetName() const override {
return "cross_entropy";
}
// convert score to a probability
void ConvertOutput(const double* input, double* output) const override {
output[0] = 1.0f / (1.0f + std::exp(-input[0]));
}
std::string ToString() const override {
std::stringstream str_buf;
str_buf << GetName();
return str_buf.str();
}
// implement custom average to boost from (if enabled among options)
double BoostFromScore(int) const override {
double suml = 0.0f;
double sumw = 0.0f;
if (weights_ != nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:suml, sumw) if (!deterministic_)
for (data_size_t i = 0; i < num_data_; ++i) {
suml += static_cast<double>(label_[i]) * weights_[i];
sumw += weights_[i];
}
} else {
sumw = static_cast<double>(num_data_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:suml) if (!deterministic_)
for (data_size_t i = 0; i < num_data_; ++i) {
suml += label_[i];
}
}
double pavg = suml / sumw;
pavg = std::min(pavg, 1.0 - kEpsilon);
pavg = std::max<double>(pavg, kEpsilon);
double initscore = std::log(pavg / (1.0f - pavg));
Log::Info("[%s:%s]: pavg = %f -> initscore = %f", GetName(), __func__, pavg, initscore);
return initscore;
}
private:
/*! \brief Number of data points */
data_size_t num_data_;
/*! \brief Pointer for label */
const label_t* label_;
/*! \brief Weights for data */
const label_t* weights_;
const bool deterministic_;
};
/*!
* \brief Objective function for alternative parameterization of cross-entropy (see top of file for explanation)
*/
class CrossEntropyLambda: public ObjectiveFunction {
public:
explicit CrossEntropyLambda(const Config& config)
: deterministic_(config.deterministic) {
min_weight_ = max_weight_ = 0.0f;
}
explicit CrossEntropyLambda(const std::vector<std::string>&)
: deterministic_(false) {}
~CrossEntropyLambda() {}
void Init(const Metadata& metadata, data_size_t num_data) override {
num_data_ = num_data;
label_ = metadata.label();
weights_ = metadata.weights();
CHECK_NOTNULL(label_);
Common::CheckElementsIntervalClosed<label_t>(label_, 0.0f, 1.0f, num_data_, GetName());
Log::Info("[%s:%s]: (objective) labels passed interval [0, 1] check", GetName(), __func__);
if (weights_ != nullptr) {
Common::ObtainMinMaxSum(weights_, num_data_, &min_weight_, &max_weight_, static_cast<label_t*>(nullptr));
if (min_weight_ <= 0.0f) {
Log::Fatal("[%s]: at least one weight is non-positive", GetName());
}
// Issue an info statement about this ratio
double weight_ratio = max_weight_ / min_weight_;
Log::Info("[%s:%s]: min, max weights = %f, %f; ratio = %f",
GetName(), __func__,
min_weight_, max_weight_,
weight_ratio);
} else {
// all weights are implied to be unity; no need to do anything
}
}
void GetGradients(const double* score, score_t* gradients, score_t* hessians) const override {
if (weights_ == nullptr) {
// compute pointwise gradients and Hessians with implied unit weights; exactly equivalent to CrossEntropy with unit weights
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double z = 1.0f / (1.0f + std::exp(-score[i]));
gradients[i] = static_cast<score_t>(z - label_[i]);
hessians[i] = static_cast<score_t>(z * (1.0f - z));
}
} else {
// compute pointwise gradients and Hessians with given weights
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static)
for (data_size_t i = 0; i < num_data_; ++i) {
const double w = weights_[i];
const double y = label_[i];
const double epf = std::exp(score[i]);
const double hhat = std::log1p(epf);
const double z = 1.0f - std::exp(-w*hhat);
const double enf = 1.0f / epf; // = std::exp(-score[i]);
gradients[i] = static_cast<score_t>((1.0f - y / z) * w / (1.0f + enf));
const double c = 1.0f / (1.0f - z);
double d = 1.0f + epf;
const double a = w * epf / (d * d);
d = c - 1.0f;
const double b = (c / (d * d) ) * (1.0f + w * epf - c);
hessians[i] = static_cast<score_t>(a * (1.0f + y * b));
}
}
}
const char* GetName() const override {
return "cross_entropy_lambda";
}
//
// ATTENTION: the function output is the "normalized exponential parameter" lambda > 0, not the probability
//
// If this code would read: output[0] = 1.0f / (1.0f + std::exp(-input[0]));
// The output would still not be the probability unless the weights are unity.
//
// Let z = 1 / (1 + exp(-f)), then prob(z) = 1-(1-z)^w, where w is the weight for the specific point.
//
void ConvertOutput(const double* input, double* output) const override {
output[0] = std::log1p(std::exp(input[0]));
}
std::string ToString() const override {
std::stringstream str_buf;
str_buf << GetName();
return str_buf.str();
}
double BoostFromScore(int) const override {
double suml = 0.0f;
double sumw = 0.0f;
if (weights_ != nullptr) {
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:suml, sumw) if (!deterministic_)
for (data_size_t i = 0; i < num_data_; ++i) {
suml += static_cast<double>(label_[i]) * weights_[i];
sumw += weights_[i];
}
} else {
sumw = static_cast<double>(num_data_);
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static) reduction(+:suml) if (!deterministic_)
for (data_size_t i = 0; i < num_data_; ++i) {
suml += label_[i];
}
}
double havg = suml / sumw;
double initscore = std::log(std::expm1(havg));
Log::Info("[%s:%s]: havg = %f -> initscore = %f", GetName(), __func__, havg, initscore);
return initscore;
}
private:
/*! \brief Number of data points */
data_size_t num_data_;
/*! \brief Pointer for label */
const label_t* label_;
/*! \brief Weights for data */
const label_t* weights_;
/*! \brief Minimum weight found during init */
label_t min_weight_;
/*! \brief Maximum weight found during init */
label_t max_weight_;
const bool deterministic_;
};
} // end namespace LightGBM
#endif // LIGHTGBM_SRC_OBJECTIVE_XENTROPY_OBJECTIVE_HPP_
src/treelearner/col_sampler.hpp
+208
View File
@@ -0,0 +1,208 @@
/*!
* Copyright (c) 2020-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2020-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_TREELEARNER_COL_SAMPLER_HPP_
#define LIGHTGBM_SRC_TREELEARNER_COL_SAMPLER_HPP_
#include <LightGBM/dataset.h>
#include <LightGBM/meta.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/openmp_wrapper.h>
#include <LightGBM/utils/random.h>
#include <algorithm>
#include <unordered_set>
#include <vector>
namespace LightGBM {
class ColSampler {
public:
explicit ColSampler(const Config* config)
: fraction_bytree_(config->feature_fraction),
fraction_bynode_(config->feature_fraction_bynode),
seed_(config->feature_fraction_seed),
random_(config->feature_fraction_seed) {
for (auto constraint : config->interaction_constraints_vector) {
std::unordered_set<int> constraint_set(constraint.begin(), constraint.end());
interaction_constraints_.push_back(constraint_set);
}
}
static int GetCnt(size_t total_cnt, double fraction) {
const int min = std::min(1, static_cast<int>(total_cnt));
int used_feature_cnt = static_cast<int>(Common::RoundInt(total_cnt * fraction));
return std::max(used_feature_cnt, min);
}
void SetTrainingData(const Dataset* train_data) {
train_data_ = train_data;
is_feature_used_.resize(train_data_->num_features(), 1);
valid_feature_indices_ = train_data->ValidFeatureIndices();
if (fraction_bytree_ >= 1.0f) {
need_reset_bytree_ = false;
used_cnt_bytree_ = static_cast<int>(valid_feature_indices_.size());
} else {
need_reset_bytree_ = true;
used_cnt_bytree_ =
GetCnt(valid_feature_indices_.size(), fraction_bytree_);
}
ResetByTree();
}
void SetConfig(const Config* config) {
fraction_bytree_ = config->feature_fraction;
fraction_bynode_ = config->feature_fraction_bynode;
is_feature_used_.resize(train_data_->num_features(), 1);
// seed is changed
if (seed_ != config->feature_fraction_seed) {
seed_ = config->feature_fraction_seed;
random_ = Random(seed_);
}
if (fraction_bytree_ >= 1.0f) {
need_reset_bytree_ = false;
used_cnt_bytree_ = static_cast<int>(valid_feature_indices_.size());
} else {
need_reset_bytree_ = true;
used_cnt_bytree_ =
GetCnt(valid_feature_indices_.size(), fraction_bytree_);
}
ResetByTree();
}
void ResetByTree() {
if (need_reset_bytree_) {
std::memset(is_feature_used_.data(), 0,
sizeof(int8_t) * is_feature_used_.size());
used_feature_indices_ = random_.Sample(
static_cast<int>(valid_feature_indices_.size()), used_cnt_bytree_);
int omp_loop_size = static_cast<int>(used_feature_indices_.size());
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (omp_loop_size >= 1024)
for (int i = 0; i < omp_loop_size; ++i) {
int used_feature = valid_feature_indices_[used_feature_indices_[i]];
int inner_feature_index = train_data_->InnerFeatureIndex(used_feature);
is_feature_used_[inner_feature_index] = 1;
}
}
}
std::vector<int8_t> GetByNode(const Tree* tree, int leaf) {
// get interaction constraints for current branch
std::unordered_set<int> allowed_features;
if (!interaction_constraints_.empty()) {
std::vector<int> branch_features = tree->branch_features(leaf);
allowed_features.insert(branch_features.begin(), branch_features.end());
for (auto constraint : interaction_constraints_) {
int num_feat_found = 0;
if (branch_features.size() == 0) {
allowed_features.insert(constraint.begin(), constraint.end());
}
for (int feat : branch_features) {
if (constraint.count(feat) == 0) {
break;
}
++num_feat_found;
if (num_feat_found == static_cast<int>(branch_features.size())) {
allowed_features.insert(constraint.begin(), constraint.end());
break;
}
}
}
}
std::vector<int8_t> ret(train_data_->num_features(), 0);
if (fraction_bynode_ >= 1.0f) {
if (interaction_constraints_.empty()) {
return std::vector<int8_t>(train_data_->num_features(), 1);
} else {
for (int feat : allowed_features) {
int inner_feat = train_data_->InnerFeatureIndex(feat);
if (inner_feat >= 0) {
ret[inner_feat] = 1;
}
}
return ret;
}
}
if (need_reset_bytree_) {
auto used_feature_cnt = GetCnt(used_feature_indices_.size(), fraction_bynode_);
std::vector<int>* allowed_used_feature_indices;
std::vector<int> filtered_feature_indices;
if (interaction_constraints_.empty()) {
allowed_used_feature_indices = &used_feature_indices_;
} else {
for (int feat_ind : used_feature_indices_) {
if (allowed_features.count(valid_feature_indices_[feat_ind]) == 1) {
filtered_feature_indices.push_back(feat_ind);
}
}
used_feature_cnt = std::min(used_feature_cnt, static_cast<int>(filtered_feature_indices.size()));
allowed_used_feature_indices = &filtered_feature_indices;
}
auto sampled_indices = random_.Sample(
static_cast<int>((*allowed_used_feature_indices).size()), used_feature_cnt);
int omp_loop_size = static_cast<int>(sampled_indices.size());
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (omp_loop_size >= 1024)
for (int i = 0; i < omp_loop_size; ++i) {
int used_feature =
valid_feature_indices_[(*allowed_used_feature_indices)[sampled_indices[i]]];
int inner_feature_index = train_data_->InnerFeatureIndex(used_feature);
ret[inner_feature_index] = 1;
}
} else {
auto used_feature_cnt =
GetCnt(valid_feature_indices_.size(), fraction_bynode_);
std::vector<int>* allowed_valid_feature_indices;
std::vector<int> filtered_feature_indices;
if (interaction_constraints_.empty()) {
allowed_valid_feature_indices = &valid_feature_indices_;
} else {
for (int feat : valid_feature_indices_) {
if (allowed_features.count(feat) == 1) {
filtered_feature_indices.push_back(feat);
}
}
allowed_valid_feature_indices = &filtered_feature_indices;
used_feature_cnt = std::min(used_feature_cnt, static_cast<int>(filtered_feature_indices.size()));
}
auto sampled_indices = random_.Sample(
static_cast<int>((*allowed_valid_feature_indices).size()), used_feature_cnt);
int omp_loop_size = static_cast<int>(sampled_indices.size());
#pragma omp parallel for num_threads(OMP_NUM_THREADS()) schedule(static, 512) if (omp_loop_size >= 1024)
for (int i = 0; i < omp_loop_size; ++i) {
int used_feature = (*allowed_valid_feature_indices)[sampled_indices[i]];
int inner_feature_index = train_data_->InnerFeatureIndex(used_feature);
ret[inner_feature_index] = 1;
}
}
return ret;
}
const std::vector<int8_t>& is_feature_used_bytree() const {
return is_feature_used_;
}
void SetIsFeatureUsedByTree(int fid, bool val) {
is_feature_used_[fid] = val;
}
private:
const Dataset* train_data_;
double fraction_bytree_;
double fraction_bynode_;
bool need_reset_bytree_;
int used_cnt_bytree_;
int seed_;
Random random_;
std::vector<int8_t> is_feature_used_;
std::vector<int> used_feature_indices_;
std::vector<int> valid_feature_indices_;
/*! \brief interaction constraints index in original (raw data) features */
std::vector<std::unordered_set<int>> interaction_constraints_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_TREELEARNER_COL_SAMPLER_HPP_
src/treelearner/cost_effective_gradient_boosting.hpp
+175
View File
@@ -0,0 +1,175 @@
/*!
* Copyright (c) 2019-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2019-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_TREELEARNER_COST_EFFECTIVE_GRADIENT_BOOSTING_HPP_
#define LIGHTGBM_SRC_TREELEARNER_COST_EFFECTIVE_GRADIENT_BOOSTING_HPP_
#include <LightGBM/config.h>
#include <LightGBM/dataset.h>
#include <LightGBM/utils/common.h>
#include <LightGBM/utils/log.h>
#include <LightGBM/utils/threading.h>
#include <vector>
#include "data_partition.hpp"
#include "serial_tree_learner.h"
#include "split_info.hpp"
namespace LightGBM {
class CostEfficientGradientBoosting {
public:
explicit CostEfficientGradientBoosting(const SerialTreeLearner* tree_learner)
: init_(false), tree_learner_(tree_learner) {}
static bool IsEnable(const Config* config) {
if (config->cegb_tradeoff >= 1.0f && config->cegb_penalty_split <= 0.0f &&
config->cegb_penalty_feature_coupled.empty() &&
config->cegb_penalty_feature_lazy.empty()) {
return false;
} else {
return true;
}
}
void Init() {
auto train_data = tree_learner_->train_data_;
if (!init_) {
splits_per_leaf_.resize(
static_cast<size_t>(tree_learner_->config_->num_leaves) *
train_data->num_features());
is_feature_used_in_split_.clear();
is_feature_used_in_split_.resize(train_data->num_features());
}
if (!tree_learner_->config_->cegb_penalty_feature_coupled.empty() &&
tree_learner_->config_->cegb_penalty_feature_coupled.size() !=
static_cast<size_t>(train_data->num_total_features())) {
Log::Fatal(
"cegb_penalty_feature_coupled should be the same size as feature "
"number.");
}
if (!tree_learner_->config_->cegb_penalty_feature_lazy.empty()) {
if (tree_learner_->config_->cegb_penalty_feature_lazy.size() !=
static_cast<size_t>(train_data->num_total_features())) {
Log::Fatal(
"cegb_penalty_feature_lazy should be the same size as feature "
"number.");
}
if (!init_) {
feature_used_in_data_ = Common::EmptyBitset(train_data->num_features() *
tree_learner_->num_data_);
}
}
init_ = true;
}
void BeforeTrain() {
// clear the splits in splits_per_leaf_
Threading::For<size_t>(0, splits_per_leaf_.size(), 1024,
[this] (int /*thread_index*/, size_t start, size_t end) {
for (size_t i = start; i < end; ++i) {
splits_per_leaf_[i].Reset();
}
});
}
double DeltaGain(int feature_index, int real_fidx, int leaf_index,
int num_data_in_leaf, SplitInfo split_info) {
auto config = tree_learner_->config_;
double delta =
config->cegb_tradeoff * config->cegb_penalty_split * num_data_in_leaf;
if (!config->cegb_penalty_feature_coupled.empty() &&
!is_feature_used_in_split_[feature_index]) {
delta += config->cegb_tradeoff *
config->cegb_penalty_feature_coupled[real_fidx];
}
if (!config->cegb_penalty_feature_lazy.empty()) {
delta += config->cegb_tradeoff *
CalculateOndemandCosts(feature_index, real_fidx, leaf_index);
}
splits_per_leaf_[static_cast<size_t>(leaf_index) *
tree_learner_->train_data_->num_features() +
feature_index] = split_info;
return delta;
}
void UpdateLeafBestSplits(Tree* tree, int best_leaf,
const SplitInfo* best_split_info,
std::vector<SplitInfo>* best_split_per_leaf) {
auto config = tree_learner_->config_;
auto train_data = tree_learner_->train_data_;
const int inner_feature_index =
train_data->InnerFeatureIndex(best_split_info->feature);
auto& ref_best_split_per_leaf = *best_split_per_leaf;
if (!config->cegb_penalty_feature_coupled.empty() &&
!is_feature_used_in_split_[inner_feature_index]) {
is_feature_used_in_split_[inner_feature_index] = true;
for (int i = 0; i < tree->num_leaves(); ++i) {
if (i == best_leaf) continue;
auto split = &splits_per_leaf_[static_cast<size_t>(i) *
train_data->num_features() +
inner_feature_index];
split->gain +=
config->cegb_tradeoff *
config->cegb_penalty_feature_coupled[best_split_info->feature];
// Avoid to update the leaf that cannot split
if (ref_best_split_per_leaf[i].gain > kMinScore &&
*split > ref_best_split_per_leaf[i]) {
ref_best_split_per_leaf[i] = *split;
}
}
}
if (!config->cegb_penalty_feature_lazy.empty()) {
data_size_t cnt_leaf_data = 0;
auto tmp_idx = tree_learner_->data_partition_->GetIndexOnLeaf(
best_leaf, &cnt_leaf_data);
for (data_size_t i_input = 0; i_input < cnt_leaf_data; ++i_input) {
int real_idx = tmp_idx[i_input];
Common::InsertBitset(
&feature_used_in_data_,
train_data->num_data() * inner_feature_index + real_idx);
}
}
}
private:
double CalculateOndemandCosts(int feature_index, int real_fidx,
int leaf_index) const {
if (tree_learner_->config_->cegb_penalty_feature_lazy.empty()) {
return 0.0f;
}
auto train_data = tree_learner_->train_data_;
double penalty =
tree_learner_->config_->cegb_penalty_feature_lazy[real_fidx];
double total = 0.0f;
data_size_t cnt_leaf_data = 0;
auto tmp_idx = tree_learner_->data_partition_->GetIndexOnLeaf(
leaf_index, &cnt_leaf_data);
for (data_size_t i_input = 0; i_input < cnt_leaf_data; ++i_input) {
int real_idx = tmp_idx[i_input];
if (Common::FindInBitset(
feature_used_in_data_.data(),
train_data->num_data() * train_data->num_features(),
train_data->num_data() * feature_index + real_idx)) {
continue;
}
total += penalty;
}
return total;
}
bool init_;
const SerialTreeLearner* tree_learner_;
std::vector<SplitInfo> splits_per_leaf_;
std::vector<bool> is_feature_used_in_split_;
std::vector<uint32_t> feature_used_in_data_;
};
} // namespace LightGBM
#endif // LIGHTGBM_SRC_TREELEARNER_COST_EFFECTIVE_GRADIENT_BOOSTING_HPP_
src/treelearner/cuda/cuda_best_split_finder.cpp
+380
View File
@@ -0,0 +1,380 @@
/*!
* Copyright (c) 2021 Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include <algorithm>
#include <vector>
#include "cuda_best_split_finder.hpp"
#include "cuda_leaf_splits.hpp"
namespace LightGBM {
CUDABestSplitFinder::CUDABestSplitFinder(
const hist_t* cuda_hist,
const Dataset* train_data,
const std::vector<uint32_t>& feature_hist_offsets,
const bool select_features_by_node,
const Config* config):
num_features_(train_data->num_features()),
num_leaves_(config->num_leaves),
feature_hist_offsets_(feature_hist_offsets),
lambda_l1_(config->lambda_l1),
lambda_l2_(config->lambda_l2),
min_data_in_leaf_(config->min_data_in_leaf),
min_sum_hessian_in_leaf_(config->min_sum_hessian_in_leaf),
min_gain_to_split_(config->min_gain_to_split),
cat_smooth_(config->cat_smooth),
cat_l2_(config->cat_l2),
max_cat_threshold_(config->max_cat_threshold),
min_data_per_group_(config->min_data_per_group),
max_cat_to_onehot_(config->max_cat_to_onehot),
extra_trees_(config->extra_trees),
extra_seed_(config->extra_seed),
use_smoothing_(config->path_smooth > 0),
path_smooth_(config->path_smooth),
num_total_bin_(feature_hist_offsets.empty() ? 0 : static_cast<int>(feature_hist_offsets.back())),
select_features_by_node_(select_features_by_node),
cuda_hist_(cuda_hist) {
InitFeatureMetaInfo(train_data);
if (has_categorical_feature_ && config->use_quantized_grad) {
Log::Fatal("Quantized training on GPU with categorical features is not supported yet.");
}
}
CUDABestSplitFinder::~CUDABestSplitFinder() {
gpuAssert(cudaStreamDestroy(cuda_streams_[0]), __FILE__, __LINE__);
gpuAssert(cudaStreamDestroy(cuda_streams_[1]), __FILE__, __LINE__);
cuda_streams_.clear();
cuda_streams_.shrink_to_fit();
}
void CUDABestSplitFinder::InitFeatureMetaInfo(const Dataset* train_data) {
feature_missing_type_.resize(num_features_);
feature_mfb_offsets_.resize(num_features_);
feature_default_bins_.resize(num_features_);
feature_num_bins_.resize(num_features_);
max_num_bin_in_feature_ = 0;
has_categorical_feature_ = false;
max_num_categorical_bin_ = 0;
is_categorical_.resize(train_data->num_features(), 0);
for (int inner_feature_index = 0; inner_feature_index < num_features_; ++inner_feature_index) {
const BinMapper* bin_mapper = train_data->FeatureBinMapper(inner_feature_index);
if (bin_mapper->bin_type() == BinType::CategoricalBin) {
has_categorical_feature_ = true;
is_categorical_[inner_feature_index] = 1;
if (bin_mapper->num_bin() > max_num_categorical_bin_) {
max_num_categorical_bin_ = bin_mapper->num_bin();
}
}
const MissingType missing_type = bin_mapper->missing_type();
feature_missing_type_[inner_feature_index] = missing_type;
feature_mfb_offsets_[inner_feature_index] = static_cast<int8_t>(bin_mapper->GetMostFreqBin() == 0);
feature_default_bins_[inner_feature_index] = bin_mapper->GetDefaultBin();
feature_num_bins_[inner_feature_index] = static_cast<uint32_t>(bin_mapper->num_bin());
const int num_bin_hist = bin_mapper->num_bin() - feature_mfb_offsets_[inner_feature_index];
if (num_bin_hist > max_num_bin_in_feature_) {
max_num_bin_in_feature_ = num_bin_hist;
}
}
if (max_num_bin_in_feature_ > NUM_THREADS_PER_BLOCK_BEST_SPLIT_FINDER) {
use_global_memory_ = true;
} else {
use_global_memory_ = false;
}
}
void CUDABestSplitFinder::Init() {
InitCUDAFeatureMetaInfo();
cuda_streams_.resize(2);
CUDASUCCESS_OR_FATAL(cudaStreamCreate(&cuda_streams_[0]));
CUDASUCCESS_OR_FATAL(cudaStreamCreate(&cuda_streams_[1]));
cuda_best_split_info_buffer_.Resize(8);
if (use_global_memory_) {
cuda_feature_hist_grad_buffer_.Resize(static_cast<size_t>(num_total_bin_));
cuda_feature_hist_hess_buffer_.Resize(static_cast<size_t>(num_total_bin_));
if (has_categorical_feature_) {
cuda_feature_hist_stat_buffer_.Resize(static_cast<size_t>(num_total_bin_));
cuda_feature_hist_index_buffer_.Resize(static_cast<size_t>(num_total_bin_));
}
}
if (select_features_by_node_) {
is_feature_used_by_smaller_node_.Resize(num_features_);
is_feature_used_by_larger_node_.Resize(num_features_);
}
}
void CUDABestSplitFinder::InitCUDAFeatureMetaInfo() {
cuda_is_feature_used_bytree_.Resize(static_cast<size_t>(num_features_));
// initialize split find task information (a split find task is one pass through the histogram of a feature)
num_tasks_ = 0;
for (int inner_feature_index = 0; inner_feature_index < num_features_; ++inner_feature_index) {
const uint32_t num_bin = feature_num_bins_[inner_feature_index];
const MissingType missing_type = feature_missing_type_[inner_feature_index];
if (num_bin > 2 && missing_type != MissingType::None && !is_categorical_[inner_feature_index]) {
num_tasks_ += 2;
} else {
++num_tasks_;
}
}
split_find_tasks_.resize(num_tasks_);
split_find_tasks_.shrink_to_fit();
int cur_task_index = 0;
for (int inner_feature_index = 0; inner_feature_index < num_features_; ++inner_feature_index) {
const uint32_t num_bin = feature_num_bins_[inner_feature_index];
const MissingType missing_type = feature_missing_type_[inner_feature_index];
if (num_bin > 2 && missing_type != MissingType::None && !is_categorical_[inner_feature_index]) {
if (missing_type == MissingType::Zero) {
SplitFindTask* new_task = &split_find_tasks_[cur_task_index];
new_task->reverse = false;
new_task->skip_default_bin = true;
new_task->na_as_missing = false;
new_task->inner_feature_index = inner_feature_index;
new_task->assume_out_default_left = false;
new_task->is_categorical = false;
uint32_t num_bin = feature_num_bins_[inner_feature_index];
new_task->is_one_hot = false;
new_task->hist_offset = feature_hist_offsets_[inner_feature_index];
new_task->mfb_offset = feature_mfb_offsets_[inner_feature_index];
new_task->default_bin = feature_default_bins_[inner_feature_index];
new_task->num_bin = num_bin;
++cur_task_index;
new_task = &split_find_tasks_[cur_task_index];
new_task->reverse = true;
new_task->skip_default_bin = true;
new_task->na_as_missing = false;
new_task->inner_feature_index = inner_feature_index;
new_task->assume_out_default_left = true;
new_task->is_categorical = false;
num_bin = feature_num_bins_[inner_feature_index];
new_task->is_one_hot = false;
new_task->hist_offset = feature_hist_offsets_[inner_feature_index];
new_task->default_bin = feature_default_bins_[inner_feature_index];
new_task->mfb_offset = feature_mfb_offsets_[inner_feature_index];
new_task->num_bin = num_bin;
++cur_task_index;
} else {
SplitFindTask* new_task = &split_find_tasks_[cur_task_index];
new_task->reverse = false;
new_task->skip_default_bin = false;
new_task->na_as_missing = true;
new_task->inner_feature_index = inner_feature_index;
new_task->assume_out_default_left = false;
new_task->is_categorical = false;
uint32_t num_bin = feature_num_bins_[inner_feature_index];
new_task->is_one_hot = false;
new_task->hist_offset = feature_hist_offsets_[inner_feature_index];
new_task->mfb_offset = feature_mfb_offsets_[inner_feature_index];
new_task->default_bin = feature_default_bins_[inner_feature_index];
new_task->num_bin = num_bin;
++cur_task_index;
new_task = &split_find_tasks_[cur_task_index];
new_task->reverse = true;
new_task->skip_default_bin = false;
new_task->na_as_missing = true;
new_task->inner_feature_index = inner_feature_index;
new_task->assume_out_default_left = true;
new_task->is_categorical = false;
num_bin = feature_num_bins_[inner_feature_index];
new_task->is_one_hot = false;
new_task->hist_offset = feature_hist_offsets_[inner_feature_index];
new_task->mfb_offset = feature_mfb_offsets_[inner_feature_index];
new_task->default_bin = feature_default_bins_[inner_feature_index];
new_task->num_bin = num_bin;
++cur_task_index;
}
} else {
SplitFindTask& new_task = split_find_tasks_[cur_task_index];
const uint32_t num_bin = feature_num_bins_[inner_feature_index];
if (is_categorical_[inner_feature_index]) {
new_task.reverse = false;
new_task.is_categorical = true;
new_task.is_one_hot = (static_cast<int>(num_bin) <= max_cat_to_onehot_);
} else {
new_task.reverse = true;
new_task.is_categorical = false;
new_task.is_one_hot = false;
}
new_task.skip_default_bin = false;
new_task.na_as_missing = false;
new_task.inner_feature_index = inner_feature_index;
if (missing_type != MissingType::NaN && !is_categorical_[inner_feature_index]) {
new_task.assume_out_default_left = true;
} else {
new_task.assume_out_default_left = false;
}
new_task.hist_offset = feature_hist_offsets_[inner_feature_index];
new_task.mfb_offset = feature_mfb_offsets_[inner_feature_index];
new_task.default_bin = feature_default_bins_[inner_feature_index];
new_task.num_bin = num_bin;
++cur_task_index;
}
}
CHECK_EQ(cur_task_index, static_cast<int>(split_find_tasks_.size()));
if (extra_trees_) {
cuda_randoms_.Resize(num_tasks_ * 2);
LaunchInitCUDARandomKernel();
}
const int num_task_blocks = (num_tasks_ + NUM_TASKS_PER_SYNC_BLOCK - 1) / NUM_TASKS_PER_SYNC_BLOCK;
const size_t cuda_best_leaf_split_info_buffer_size = static_cast<size_t>(num_task_blocks) * static_cast<size_t>(num_leaves_);
cuda_leaf_best_split_info_.Resize(cuda_best_leaf_split_info_buffer_size);
cuda_split_find_tasks_.Resize(num_tasks_);
CopyFromHostToCUDADevice<SplitFindTask>(cuda_split_find_tasks_.RawData(),
split_find_tasks_.data(),
split_find_tasks_.size(),
__FILE__,
__LINE__);
const size_t output_buffer_size = 2 * static_cast<size_t>(num_tasks_);
cuda_best_split_info_.Resize(output_buffer_size);
max_num_categories_in_split_ = std::min(max_cat_threshold_, max_num_categorical_bin_ / 2);
cuda_cat_threshold_feature_.Resize(max_num_categories_in_split_ * output_buffer_size);
cuda_cat_threshold_real_feature_.Resize(max_num_categories_in_split_ * output_buffer_size);
cuda_cat_threshold_leaf_.Resize(max_num_categories_in_split_ * cuda_best_leaf_split_info_buffer_size);
cuda_cat_threshold_real_leaf_.Resize(max_num_categories_in_split_ * cuda_best_leaf_split_info_buffer_size);
AllocateCatVectors(cuda_leaf_best_split_info_.RawData(), cuda_cat_threshold_leaf_.RawData(), cuda_cat_threshold_real_leaf_.RawData(), cuda_best_leaf_split_info_buffer_size);
AllocateCatVectors(cuda_best_split_info_.RawData(), cuda_cat_threshold_feature_.RawData(), cuda_cat_threshold_real_feature_.RawData(), output_buffer_size);
}
void CUDABestSplitFinder::ResetTrainingData(
const hist_t* cuda_hist,
const Dataset* train_data,
const std::vector<uint32_t>& feature_hist_offsets) {
cuda_hist_ = cuda_hist;
num_features_ = train_data->num_features();
feature_hist_offsets_ = feature_hist_offsets;
InitFeatureMetaInfo(train_data);
cuda_is_feature_used_bytree_.Clear();
cuda_best_split_info_.Clear();
InitCUDAFeatureMetaInfo();
}
void CUDABestSplitFinder::ResetConfig(const Config* config, const hist_t* cuda_hist) {
num_leaves_ = config->num_leaves;
lambda_l1_ = config->lambda_l1;
lambda_l2_ = config->lambda_l2;
min_data_in_leaf_ = config->min_data_in_leaf;
min_sum_hessian_in_leaf_ = config->min_sum_hessian_in_leaf;
min_gain_to_split_ = config->min_gain_to_split;
cat_smooth_ = config->cat_smooth;
cat_l2_ = config->cat_l2;
max_cat_threshold_ = config->max_cat_threshold;
min_data_per_group_ = config->min_data_per_group;
max_cat_to_onehot_ = config->max_cat_to_onehot;
extra_trees_ = config->extra_trees;
extra_seed_ = config->extra_seed;
use_smoothing_ = (config->path_smooth > 0.0f);
path_smooth_ = config->path_smooth;
cuda_hist_ = cuda_hist;
const int num_task_blocks = (num_tasks_ + NUM_TASKS_PER_SYNC_BLOCK - 1) / NUM_TASKS_PER_SYNC_BLOCK;
size_t cuda_best_leaf_split_info_buffer_size = static_cast<size_t>(num_task_blocks) * static_cast<size_t>(num_leaves_);
cuda_leaf_best_split_info_.Resize(cuda_best_leaf_split_info_buffer_size);
max_num_categories_in_split_ = std::min(max_cat_threshold_, max_num_categorical_bin_ / 2);
size_t total_cat_threshold_size = max_num_categories_in_split_ * cuda_best_leaf_split_info_buffer_size;
cuda_cat_threshold_leaf_.Resize(total_cat_threshold_size);
cuda_cat_threshold_real_leaf_.Resize(total_cat_threshold_size);
AllocateCatVectors(cuda_leaf_best_split_info_.RawData(), cuda_cat_threshold_leaf_.RawData(), cuda_cat_threshold_real_leaf_.RawData(), cuda_best_leaf_split_info_buffer_size);
cuda_best_leaf_split_info_buffer_size = 2 * static_cast<size_t>(num_tasks_);
total_cat_threshold_size = max_num_categories_in_split_ * cuda_best_leaf_split_info_buffer_size;
cuda_cat_threshold_feature_.Resize(total_cat_threshold_size);
cuda_cat_threshold_real_feature_.Resize(total_cat_threshold_size);
AllocateCatVectors(cuda_best_split_info_.RawData(), cuda_cat_threshold_feature_.RawData(), cuda_cat_threshold_real_feature_.RawData(), cuda_best_leaf_split_info_buffer_size);
}
void CUDABestSplitFinder::BeforeTrain(const std::vector<int8_t>& is_feature_used_bytree) {
CopyFromHostToCUDADevice<int8_t>(cuda_is_feature_used_bytree_.RawData(),
is_feature_used_bytree.data(),
is_feature_used_bytree.size(), __FILE__, __LINE__);
}
void CUDABestSplitFinder::FindBestSplitsForLeaf(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const CUDALeafSplitsStruct* larger_leaf_splits,
const int smaller_leaf_index,
const int larger_leaf_index,
const data_size_t num_data_in_smaller_leaf,
const data_size_t num_data_in_larger_leaf,
const double sum_hessians_in_smaller_leaf,
const double sum_hessians_in_larger_leaf,
const score_t* grad_scale,
const score_t* hess_scale,
const uint8_t smaller_num_bits_in_histogram_bins,
const uint8_t larger_num_bits_in_histogram_bins) {
const bool is_smaller_leaf_valid = (num_data_in_smaller_leaf > min_data_in_leaf_ &&
sum_hessians_in_smaller_leaf > min_sum_hessian_in_leaf_);
const bool is_larger_leaf_valid = (num_data_in_larger_leaf > min_data_in_leaf_ &&
sum_hessians_in_larger_leaf > min_sum_hessian_in_leaf_ && larger_leaf_index >= 0);
if (grad_scale != nullptr && hess_scale != nullptr) {
LaunchFindBestSplitsDiscretizedForLeafKernel(smaller_leaf_splits, larger_leaf_splits,
smaller_leaf_index, larger_leaf_index, is_smaller_leaf_valid, is_larger_leaf_valid,
grad_scale, hess_scale, smaller_num_bits_in_histogram_bins, larger_num_bits_in_histogram_bins, num_data_in_smaller_leaf, num_data_in_larger_leaf);
} else {
LaunchFindBestSplitsForLeafKernel(smaller_leaf_splits, larger_leaf_splits,
smaller_leaf_index, larger_leaf_index, is_smaller_leaf_valid, is_larger_leaf_valid, num_data_in_smaller_leaf, num_data_in_larger_leaf);
}
global_timer.Start("CUDABestSplitFinder::LaunchSyncBestSplitForLeafKernel");
LaunchSyncBestSplitForLeafKernel(smaller_leaf_index, larger_leaf_index, is_smaller_leaf_valid, is_larger_leaf_valid);
SynchronizeCUDADevice(__FILE__, __LINE__);
global_timer.Stop("CUDABestSplitFinder::LaunchSyncBestSplitForLeafKernel");
}
const CUDASplitInfo* CUDABestSplitFinder::FindBestFromAllSplits(
const int cur_num_leaves,
const int smaller_leaf_index,
const int larger_leaf_index,
int* smaller_leaf_best_split_feature,
uint32_t* smaller_leaf_best_split_threshold,
uint8_t* smaller_leaf_best_split_default_left,
int* larger_leaf_best_split_feature,
uint32_t* larger_leaf_best_split_threshold,
uint8_t* larger_leaf_best_split_default_left,
int* best_leaf_index,
int* num_cat_threshold) {
LaunchFindBestFromAllSplitsKernel(
cur_num_leaves,
smaller_leaf_index,
larger_leaf_index,
smaller_leaf_best_split_feature,
smaller_leaf_best_split_threshold,
smaller_leaf_best_split_default_left,
larger_leaf_best_split_feature,
larger_leaf_best_split_threshold,
larger_leaf_best_split_default_left,
best_leaf_index,
num_cat_threshold);
SynchronizeCUDADevice(__FILE__, __LINE__);
return cuda_leaf_best_split_info_.RawData() + (*best_leaf_index);
}
void CUDABestSplitFinder::AllocateCatVectors(CUDASplitInfo* cuda_split_infos, uint32_t* cat_threshold_vec, int* cat_threshold_real_vec, size_t len) {
LaunchAllocateCatVectorsKernel(cuda_split_infos, cat_threshold_vec, cat_threshold_real_vec, len);
}
void CUDABestSplitFinder::SetUsedFeatureByNode(const std::vector<int8_t>& is_feature_used_by_smaller_node,
const std::vector<int8_t>& is_feature_used_by_larger_node) {
if (select_features_by_node_) {
CopyFromHostToCUDADevice<int8_t>(is_feature_used_by_smaller_node_.RawData(),
is_feature_used_by_smaller_node.data(), is_feature_used_by_smaller_node.size(), __FILE__, __LINE__);
CopyFromHostToCUDADevice<int8_t>(is_feature_used_by_larger_node_.RawData(),
is_feature_used_by_larger_node.data(), is_feature_used_by_larger_node.size(), __FILE__, __LINE__);
}
}
} // namespace LightGBM
#endif // USE_CUDA
src/treelearner/cuda/cuda_best_split_finder.cu
+2268
View File
File diff suppressed because it is too large Load Diff
src/treelearner/cuda/cuda_best_split_finder.hpp
+249
View File
@@ -0,0 +1,249 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_BEST_SPLIT_FINDER_HPP_
#define LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_BEST_SPLIT_FINDER_HPP_
#ifdef USE_CUDA
#include <LightGBM/bin.h>
#include <LightGBM/dataset.h>
#include <vector>
#include <LightGBM/cuda/cuda_random.hpp>
#include <LightGBM/cuda/cuda_split_info.hpp>
#include "cuda_leaf_splits.hpp"
#define NUM_THREADS_PER_BLOCK_BEST_SPLIT_FINDER (256)
#define NUM_THREADS_FIND_BEST_LEAF (256)
#define NUM_TASKS_PER_SYNC_BLOCK (1024)
namespace LightGBM {
struct SplitFindTask {
int inner_feature_index;
bool reverse;
bool skip_default_bin;
bool na_as_missing;
bool assume_out_default_left;
bool is_categorical;
bool is_one_hot;
uint32_t hist_offset;
uint8_t mfb_offset;
uint32_t num_bin;
uint32_t default_bin;
int rand_threshold;
};
class CUDABestSplitFinder {
public:
CUDABestSplitFinder(
const hist_t* cuda_hist,
const Dataset* train_data,
const std::vector<uint32_t>& feature_hist_offsets,
const bool select_features_by_node,
const Config* config);
~CUDABestSplitFinder();
void InitFeatureMetaInfo(const Dataset* train_data);
void Init();
void InitCUDAFeatureMetaInfo();
void BeforeTrain(const std::vector<int8_t>& is_feature_used_bytree);
void FindBestSplitsForLeaf(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const CUDALeafSplitsStruct* larger_leaf_splits,
const int smaller_leaf_index,
const int larger_leaf_index,
const data_size_t num_data_in_smaller_leaf,
const data_size_t num_data_in_larger_leaf,
const double sum_hessians_in_smaller_leaf,
const double sum_hessians_in_larger_leaf,
const score_t* grad_scale,
const score_t* hess_scale,
const uint8_t smaller_num_bits_in_histogram_bins,
const uint8_t larger_num_bits_in_histogram_bins);
const CUDASplitInfo* FindBestFromAllSplits(
const int cur_num_leaves,
const int smaller_leaf_index,
const int larger_leaf_index,
int* smaller_leaf_best_split_feature,
uint32_t* smaller_leaf_best_split_threshold,
uint8_t* smaller_leaf_best_split_default_left,
int* larger_leaf_best_split_feature,
uint32_t* larger_leaf_best_split_threshold,
uint8_t* larger_leaf_best_split_default_left,
int* best_leaf_index,
int* num_cat_threshold);
void ResetTrainingData(
const hist_t* cuda_hist,
const Dataset* train_data,
const std::vector<uint32_t>& feature_hist_offsets);
void ResetConfig(const Config* config, const hist_t* cuda_hist);
void SetUsedFeatureByNode(const std::vector<int8_t>& is_feature_used_by_smaller_node,
const std::vector<int8_t>& is_feature_used_by_larger_node);
private:
#define LaunchFindBestSplitsForLeafKernel_PARAMS \
const CUDALeafSplitsStruct* smaller_leaf_splits, \
const CUDALeafSplitsStruct* larger_leaf_splits, \
const int smaller_leaf_index, \
const int larger_leaf_index, \
const bool is_smaller_leaf_valid, \
const bool is_larger_leaf_valid, \
const data_size_t global_num_data_in_smaller_leaf, \
const data_size_t global_num_data_in_larger_leaf
void LaunchFindBestSplitsForLeafKernel(LaunchFindBestSplitsForLeafKernel_PARAMS);
template <bool USE_RAND>
void LaunchFindBestSplitsForLeafKernelInner0(LaunchFindBestSplitsForLeafKernel_PARAMS);
template <bool USE_RAND, bool USE_L1>
void LaunchFindBestSplitsForLeafKernelInner1(LaunchFindBestSplitsForLeafKernel_PARAMS);
template <bool USE_RAND, bool USE_L1, bool USE_SMOOTHING>
void LaunchFindBestSplitsForLeafKernelInner2(LaunchFindBestSplitsForLeafKernel_PARAMS);
#undef LaunchFindBestSplitsForLeafKernel_PARAMS
#define LaunchFindBestSplitsDiscretizedForLeafKernel_PARAMS \
const CUDALeafSplitsStruct* smaller_leaf_splits, \
const CUDALeafSplitsStruct* larger_leaf_splits, \
const int smaller_leaf_index, \
const int larger_leaf_index, \
const bool is_smaller_leaf_valid, \
const bool is_larger_leaf_valid, \
const score_t* grad_scale, \
const score_t* hess_scale, \
const uint8_t smaller_num_bits_in_histogram_bins, \
const uint8_t larger_num_bits_in_histogram_bins, \
const data_size_t global_num_data_in_smaller_leaf, \
const data_size_t global_num_data_in_larger_leaf
void LaunchFindBestSplitsDiscretizedForLeafKernel(LaunchFindBestSplitsDiscretizedForLeafKernel_PARAMS);
template <bool USE_RAND>
void LaunchFindBestSplitsDiscretizedForLeafKernelInner0(LaunchFindBestSplitsDiscretizedForLeafKernel_PARAMS);
template <bool USE_RAND, bool USE_L1>
void LaunchFindBestSplitsDiscretizedForLeafKernelInner1(LaunchFindBestSplitsDiscretizedForLeafKernel_PARAMS);
template <bool USE_RAND, bool USE_L1, bool USE_SMOOTHING>
void LaunchFindBestSplitsDiscretizedForLeafKernelInner2(LaunchFindBestSplitsDiscretizedForLeafKernel_PARAMS);
#undef LaunchFindBestSplitsDiscretizedForLeafKernel_PARAMS
void LaunchSyncBestSplitForLeafKernel(
const int host_smaller_leaf_index,
const int host_larger_leaf_index,
const bool is_smaller_leaf_valid,
const bool is_larger_leaf_valid);
void LaunchFindBestFromAllSplitsKernel(
const int cur_num_leaves,
const int smaller_leaf_index,
const int larger_leaf_index,
int* smaller_leaf_best_split_feature,
uint32_t* smaller_leaf_best_split_threshold,
uint8_t* smaller_leaf_best_split_default_left,
int* larger_leaf_best_split_feature,
uint32_t* larger_leaf_best_split_threshold,
uint8_t* larger_leaf_best_split_default_left,
int* best_leaf_index,
data_size_t* num_cat_threshold);
void AllocateCatVectors(CUDASplitInfo* cuda_split_infos, uint32_t* cat_threshold_vec, int* cat_threshold_real_vec, size_t len);
void LaunchAllocateCatVectorsKernel(CUDASplitInfo* cuda_split_infos, uint32_t* cat_threshold_vec, int* cat_threshold_real_vec, size_t len);
void LaunchInitCUDARandomKernel();
// Host memory
int num_features_;
int num_leaves_;
int max_num_bin_in_feature_;
std::vector<uint32_t> feature_hist_offsets_;
std::vector<uint8_t> feature_mfb_offsets_;
std::vector<uint32_t> feature_default_bins_;
std::vector<uint32_t> feature_num_bins_;
std::vector<MissingType> feature_missing_type_;
double lambda_l1_;
double lambda_l2_;
data_size_t min_data_in_leaf_;
double min_sum_hessian_in_leaf_;
double min_gain_to_split_;
double cat_smooth_;
double cat_l2_;
int max_cat_threshold_;
int min_data_per_group_;
int max_cat_to_onehot_;
bool extra_trees_;
int extra_seed_;
bool use_smoothing_;
double path_smooth_;
std::vector<cudaStream_t> cuda_streams_;
// for best split find tasks
std::vector<SplitFindTask> split_find_tasks_;
int num_tasks_;
// use global memory
bool use_global_memory_;
// number of total bins in the dataset
const int num_total_bin_;
// has categorical feature
bool has_categorical_feature_;
// maximum number of bins of categorical features
int max_num_categorical_bin_;
// marks whether a feature is categorical
std::vector<int8_t> is_categorical_;
// whether need to select features by node
bool select_features_by_node_;
// CUDA memory, held by this object
// for per leaf best split information
CUDAVector<CUDASplitInfo> cuda_leaf_best_split_info_;
// for best split information when finding best split
CUDAVector<CUDASplitInfo> cuda_best_split_info_;
// best split information buffer, to be copied to host
CUDAVector<int> cuda_best_split_info_buffer_;
// find best split task information
CUDAVector<SplitFindTask> cuda_split_find_tasks_;
CUDAVector<int8_t> cuda_is_feature_used_bytree_;
// used when finding best split with global memory
CUDAVector<hist_t> cuda_feature_hist_grad_buffer_;
CUDAVector<hist_t> cuda_feature_hist_hess_buffer_;
CUDAVector<hist_t> cuda_feature_hist_stat_buffer_;
CUDAVector<data_size_t> cuda_feature_hist_index_buffer_;
CUDAVector<uint32_t> cuda_cat_threshold_leaf_;
CUDAVector<int> cuda_cat_threshold_real_leaf_;
CUDAVector<uint32_t> cuda_cat_threshold_feature_;
CUDAVector<int> cuda_cat_threshold_real_feature_;
int max_num_categories_in_split_;
// used for extremely randomized trees
CUDAVector<CUDARandom> cuda_randoms_;
// features used by node
CUDAVector<int8_t> is_feature_used_by_smaller_node_;
CUDAVector<int8_t> is_feature_used_by_larger_node_;
// CUDA memory, held by other object
const hist_t* cuda_hist_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_BEST_SPLIT_FINDER_HPP_
src/treelearner/cuda/cuda_data_partition.cpp
+352
View File
@@ -0,0 +1,352 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include <algorithm>
#include <memory>
#include <vector>
#include "cuda_data_partition.hpp"
namespace LightGBM {
CUDADataPartition::CUDADataPartition(
const Dataset* train_data,
const int num_total_bin,
const int num_leaves,
const int num_threads,
const bool use_quantized_grad,
hist_t* cuda_hist):
num_data_(train_data->num_data()),
num_features_(train_data->num_features()),
num_total_bin_(num_total_bin),
num_leaves_(num_leaves),
num_threads_(num_threads),
use_quantized_grad_(use_quantized_grad),
cuda_hist_(cuda_hist) {
CalcBlockDim(num_data_);
max_num_split_indices_blocks_ = grid_dim_;
cur_num_leaves_ = 1;
cuda_column_data_ = train_data->cuda_column_data();
is_categorical_feature_.resize(train_data->num_features(), false);
is_single_feature_in_column_.resize(train_data->num_features(), false);
for (int feature_index = 0; feature_index < train_data->num_features(); ++feature_index) {
if (train_data->FeatureBinMapper(feature_index)->bin_type() == BinType::CategoricalBin) {
is_categorical_feature_[feature_index] = true;
}
const int feature_group_index = train_data->Feature2Group(feature_index);
if (!train_data->IsMultiGroup(feature_group_index)) {
if ((feature_index == 0 || train_data->Feature2Group(feature_index - 1) != feature_group_index) &&
(feature_index == train_data->num_features() - 1 || train_data->Feature2Group(feature_index + 1) != feature_group_index)) {
is_single_feature_in_column_[feature_index] = true;
}
} else {
is_single_feature_in_column_[feature_index] = true;
}
}
}
CUDADataPartition::~CUDADataPartition() {
CUDASUCCESS_OR_FATAL(cudaStreamDestroy(cuda_streams_[0]));
CUDASUCCESS_OR_FATAL(cudaStreamDestroy(cuda_streams_[1]));
CUDASUCCESS_OR_FATAL(cudaStreamDestroy(cuda_streams_[2]));
CUDASUCCESS_OR_FATAL(cudaStreamDestroy(cuda_streams_[3]));
cuda_streams_.clear();
cuda_streams_.shrink_to_fit();
}
void CUDADataPartition::Init() {
// allocate CUDA memory
cuda_data_indices_.Resize(static_cast<size_t>(num_data_));
cuda_leaf_data_start_.Resize(static_cast<size_t>(num_leaves_));
cuda_leaf_data_end_.Resize(static_cast<size_t>(num_leaves_));
cuda_leaf_num_data_.Resize(static_cast<size_t>(num_leaves_));
// leave some space for alignment
cuda_block_to_left_offset_.Resize(static_cast<size_t>(num_data_));
cuda_data_index_to_leaf_index_.Resize(static_cast<size_t>(num_data_));
cuda_block_data_to_left_offset_.Resize(static_cast<size_t>(max_num_split_indices_blocks_) + 1);
cuda_block_data_to_right_offset_.Resize(static_cast<size_t>(max_num_split_indices_blocks_) + 1);
SetCUDAMemory<data_size_t>(cuda_block_data_to_left_offset_.RawData(), 0, static_cast<size_t>(max_num_split_indices_blocks_) + 1, __FILE__, __LINE__);
SetCUDAMemory<data_size_t>(cuda_block_data_to_right_offset_.RawData(), 0, static_cast<size_t>(max_num_split_indices_blocks_) + 1, __FILE__, __LINE__);
cuda_out_data_indices_in_leaf_.Resize(static_cast<size_t>(num_data_));
cuda_hist_pool_.Resize(static_cast<size_t>(num_leaves_));
CopyFromHostToCUDADevice<hist_t*>(cuda_hist_pool_.RawData(), &cuda_hist_, 1, __FILE__, __LINE__);
cuda_split_info_buffer_.Resize(18);
cuda_leaf_output_.Resize(static_cast<size_t>(num_leaves_));
cuda_streams_.resize(4);
gpuAssert(cudaStreamCreate(&cuda_streams_[0]), __FILE__, __LINE__);
gpuAssert(cudaStreamCreate(&cuda_streams_[1]), __FILE__, __LINE__);
gpuAssert(cudaStreamCreate(&cuda_streams_[2]), __FILE__, __LINE__);
gpuAssert(cudaStreamCreate(&cuda_streams_[3]), __FILE__, __LINE__);
cuda_num_data_.InitFromHostVector(std::vector<data_size_t>{num_data_});
use_bagging_ = false;
used_indices_ = nullptr;
}
void CUDADataPartition::BeforeTrain() {
if (!use_bagging_) {
LaunchFillDataIndicesBeforeTrain();
}
SetCUDAMemory<data_size_t>(cuda_leaf_num_data_.RawData(), 0, static_cast<size_t>(num_leaves_), __FILE__, __LINE__);
SetCUDAMemory<data_size_t>(cuda_leaf_data_start_.RawData(), 0, static_cast<size_t>(num_leaves_), __FILE__, __LINE__);
SetCUDAMemory<data_size_t>(cuda_leaf_data_end_.RawData(), 0, static_cast<size_t>(num_leaves_), __FILE__, __LINE__);
SynchronizeCUDADevice(__FILE__, __LINE__);
if (!use_bagging_) {
CopyFromCUDADeviceToCUDADevice<data_size_t>(cuda_leaf_num_data_.RawData(), cuda_num_data_.RawData(), 1, __FILE__, __LINE__);
CopyFromCUDADeviceToCUDADevice<data_size_t>(cuda_leaf_data_end_.RawData(), cuda_num_data_.RawData(), 1, __FILE__, __LINE__);
} else {
CopyFromHostToCUDADevice<data_size_t>(cuda_leaf_num_data_.RawData(), &num_used_indices_, 1, __FILE__, __LINE__);
CopyFromHostToCUDADevice<data_size_t>(cuda_leaf_data_end_.RawData(), &num_used_indices_, 1, __FILE__, __LINE__);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
CopyFromHostToCUDADevice<hist_t*>(cuda_hist_pool_.RawData(), &cuda_hist_, 1, __FILE__, __LINE__);
}
void CUDADataPartition::Split(
// input best split info
const CUDASplitInfo* best_split_info,
const int left_leaf_index,
const int right_leaf_index,
const int leaf_best_split_feature,
const uint32_t leaf_best_split_threshold,
const uint32_t* categorical_bitset,
const int categorical_bitset_len,
const uint8_t leaf_best_split_default_left,
const data_size_t num_data_in_leaf,
const data_size_t leaf_data_start,
// for leaf information update
CUDALeafSplitsStruct* smaller_leaf_splits,
CUDALeafSplitsStruct* larger_leaf_splits,
// gather information for CPU, used for launching kernels
data_size_t* left_leaf_num_data,
data_size_t* right_leaf_num_data,
data_size_t* left_leaf_start,
data_size_t* right_leaf_start,
double* left_leaf_sum_of_hessians,
double* right_leaf_sum_of_hessians,
double* left_leaf_sum_of_gradients,
double* right_leaf_sum_of_gradients,
data_size_t* global_left_leaf_num_data,
data_size_t* global_right_leaf_num_data) {
CalcBlockDim(num_data_in_leaf);
global_timer.Start("GenDataToLeftBitVector");
GenDataToLeftBitVector(num_data_in_leaf,
leaf_best_split_feature,
leaf_best_split_threshold,
categorical_bitset,
categorical_bitset_len,
leaf_best_split_default_left,
leaf_data_start,
left_leaf_index,
right_leaf_index);
global_timer.Stop("GenDataToLeftBitVector");
global_timer.Start("SplitInner");
SplitInner(num_data_in_leaf,
best_split_info,
left_leaf_index,
right_leaf_index,
smaller_leaf_splits,
larger_leaf_splits,
left_leaf_num_data,
right_leaf_num_data,
left_leaf_start,
right_leaf_start,
left_leaf_sum_of_hessians,
right_leaf_sum_of_hessians,
left_leaf_sum_of_gradients,
right_leaf_sum_of_gradients,
global_left_leaf_num_data,
global_right_leaf_num_data);
global_timer.Stop("SplitInner");
}
void CUDADataPartition::GenDataToLeftBitVector(
const data_size_t num_data_in_leaf,
const int split_feature_index,
const uint32_t split_threshold,
const uint32_t* categorical_bitset,
const int categorical_bitset_len,
const uint8_t split_default_left,
const data_size_t leaf_data_start,
const int left_leaf_index,
const int right_leaf_index) {
if (is_categorical_feature_[split_feature_index]) {
LaunchGenDataToLeftBitVectorCategoricalKernel(
num_data_in_leaf,
split_feature_index,
categorical_bitset,
categorical_bitset_len,
split_default_left,
leaf_data_start,
left_leaf_index,
right_leaf_index);
} else {
LaunchGenDataToLeftBitVectorKernel(
num_data_in_leaf,
split_feature_index,
split_threshold,
split_default_left,
leaf_data_start,
left_leaf_index,
right_leaf_index);
}
}
void CUDADataPartition::SplitInner(
const data_size_t num_data_in_leaf,
const CUDASplitInfo* best_split_info,
const int left_leaf_index,
const int right_leaf_index,
// for leaf splits information update
CUDALeafSplitsStruct* smaller_leaf_splits,
CUDALeafSplitsStruct* larger_leaf_splits,
data_size_t* left_leaf_num_data,
data_size_t* right_leaf_num_data,
data_size_t* left_leaf_start,
data_size_t* right_leaf_start,
double* left_leaf_sum_of_hessians,
double* right_leaf_sum_of_hessians,
double* left_leaf_sum_of_gradients,
double* right_leaf_sum_of_gradients,
data_size_t* global_left_leaf_num_data,
data_size_t* global_right_leaf_num_data) {
LaunchSplitInnerKernel(
num_data_in_leaf,
best_split_info,
left_leaf_index,
right_leaf_index,
smaller_leaf_splits,
larger_leaf_splits,
left_leaf_num_data,
right_leaf_num_data,
left_leaf_start,
right_leaf_start,
left_leaf_sum_of_hessians,
right_leaf_sum_of_hessians,
left_leaf_sum_of_gradients,
right_leaf_sum_of_gradients,
global_left_leaf_num_data,
global_right_leaf_num_data);
++cur_num_leaves_;
}
void CUDADataPartition::UpdateTrainScore(const Tree* tree, double* scores) {
const CUDATree* cuda_tree = nullptr;
std::unique_ptr<CUDATree> cuda_tree_ptr;
if (tree->is_cuda_tree()) {
cuda_tree = reinterpret_cast<const CUDATree*>(tree);
} else {
cuda_tree_ptr.reset(new CUDATree(tree));
cuda_tree = cuda_tree_ptr.get();
}
if (use_bagging_) {
// we need restore the order of indices in cuda_data_indices_
CopyFromCUDADeviceToCUDADevice<data_size_t>(cuda_data_indices_.RawData(), used_indices_, static_cast<size_t>(num_used_indices_), __FILE__, __LINE__);
}
LaunchAddPredictionToScoreKernel(cuda_tree->cuda_leaf_value(), scores);
}
void CUDADataPartition::CalcBlockDim(const data_size_t num_data_in_leaf) {
const int min_num_blocks = num_data_in_leaf <= 100 ? 1 : 80;
const int num_blocks = std::max(min_num_blocks, (num_data_in_leaf + SPLIT_INDICES_BLOCK_SIZE_DATA_PARTITION - 1) / SPLIT_INDICES_BLOCK_SIZE_DATA_PARTITION);
int split_indices_block_size_data_partition = (num_data_in_leaf + num_blocks - 1) / num_blocks - 1;
CHECK_GT(split_indices_block_size_data_partition, 0);
int split_indices_block_size_data_partition_aligned = 1;
while (split_indices_block_size_data_partition > 0) {
split_indices_block_size_data_partition_aligned <<= 1;
split_indices_block_size_data_partition >>= 1;
}
const int num_blocks_final = (num_data_in_leaf + split_indices_block_size_data_partition_aligned - 1) / split_indices_block_size_data_partition_aligned;
grid_dim_ = num_blocks_final;
block_dim_ = split_indices_block_size_data_partition_aligned;
}
void CUDADataPartition::SetUsedDataIndices(const data_size_t* used_indices, const data_size_t num_used_indices) {
use_bagging_ = true;
num_used_indices_ = num_used_indices;
used_indices_ = used_indices;
CopyFromCUDADeviceToCUDADevice<data_size_t>(cuda_data_indices_.RawData(), used_indices, static_cast<size_t>(num_used_indices), __FILE__, __LINE__);
LaunchFillDataIndexToLeafIndex();
}
void CUDADataPartition::ResetTrainingData(const Dataset* train_data, const int num_total_bin, hist_t* cuda_hist) {
const data_size_t old_num_data = num_data_;
num_data_ = train_data->num_data();
num_features_ = train_data->num_features();
num_total_bin_ = num_total_bin;
cuda_column_data_ = train_data->cuda_column_data();
cuda_hist_ = cuda_hist;
CopyFromHostToCUDADevice<hist_t*>(cuda_hist_pool_.RawData(), &cuda_hist_, 1, __FILE__, __LINE__);
CopyFromHostToCUDADevice<int>(cuda_num_data_.RawData(), &num_data_, 1, __FILE__, __LINE__);
if (num_data_ > old_num_data) {
CalcBlockDim(num_data_);
const int old_max_num_split_indices_blocks = max_num_split_indices_blocks_;
max_num_split_indices_blocks_ = grid_dim_;
if (max_num_split_indices_blocks_ > old_max_num_split_indices_blocks) {
cuda_block_data_to_left_offset_.Resize(static_cast<size_t>(max_num_split_indices_blocks_) + 1);
cuda_block_data_to_right_offset_.Resize(static_cast<size_t>(max_num_split_indices_blocks_) + 1);
SetCUDAMemory<data_size_t>(cuda_block_data_to_left_offset_.RawData(), 0, static_cast<size_t>(max_num_split_indices_blocks_) + 1, __FILE__, __LINE__);
SetCUDAMemory<data_size_t>(cuda_block_data_to_right_offset_.RawData(), 0, static_cast<size_t>(max_num_split_indices_blocks_) + 1, __FILE__, __LINE__);
}
cuda_data_indices_.Resize(static_cast<size_t>(num_data_));
cuda_block_to_left_offset_.Resize(static_cast<size_t>(num_data_));
cuda_data_index_to_leaf_index_.Resize(static_cast<size_t>(num_data_));
cuda_out_data_indices_in_leaf_.Resize(static_cast<size_t>(num_data_));
}
used_indices_ = nullptr;
use_bagging_ = false;
num_used_indices_ = 0;
cur_num_leaves_ = 1;
}
void CUDADataPartition::ResetConfig(const Config* config, hist_t* cuda_hist) {
num_threads_ = OMP_NUM_THREADS();
num_leaves_ = config->num_leaves;
cuda_hist_ = cuda_hist;
cuda_leaf_data_start_.Resize(static_cast<size_t>(num_leaves_));
cuda_leaf_data_end_.Resize(static_cast<size_t>(num_leaves_));
cuda_leaf_num_data_.Resize(static_cast<size_t>(num_leaves_));
cuda_hist_pool_.Resize(static_cast<size_t>(num_leaves_));
cuda_leaf_output_.Resize(static_cast<size_t>(num_leaves_));
}
void CUDADataPartition::SetBaggingSubset(const Dataset* subset) {
num_used_indices_ = subset->num_data();
used_indices_ = nullptr;
use_bagging_ = true;
cuda_column_data_ = subset->cuda_column_data();
}
void CUDADataPartition::ResetByLeafPred(const std::vector<int>& leaf_pred, int num_leaves) {
if (leaf_pred.size() != static_cast<size_t>(num_data_)) {
cuda_data_index_to_leaf_index_.Clear();
cuda_data_index_to_leaf_index_.InitFromHostVector(leaf_pred);
num_data_ = static_cast<data_size_t>(leaf_pred.size());
} else {
CopyFromHostToCUDADevice<int>(cuda_data_index_to_leaf_index_.RawData(), leaf_pred.data(), leaf_pred.size(), __FILE__, __LINE__);
}
num_leaves_ = num_leaves;
cur_num_leaves_ = num_leaves;
}
void CUDADataPartition::ReduceLeafGradStat(
const score_t* gradients, const score_t* hessians,
CUDATree* tree, double* leaf_grad_stat_buffer, double* leaf_hess_state_buffer) const {
LaunchReduceLeafGradStat(gradients, hessians, tree, leaf_grad_stat_buffer, leaf_hess_state_buffer);
}
} // namespace LightGBM
#endif // USE_CUDA
src/treelearner/cuda/cuda_data_partition.cu
+1169
View File
File diff suppressed because it is too large Load Diff
src/treelearner/cuda/cuda_data_partition.hpp
+406
View File
@@ -0,0 +1,406 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_DATA_PARTITION_HPP_
#define LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_DATA_PARTITION_HPP_
#ifdef USE_CUDA
#include <LightGBM/bin.h>
#include <LightGBM/meta.h>
#include <LightGBM/tree.h>
#include <vector>
#include <LightGBM/cuda/cuda_column_data.hpp>
#include <LightGBM/cuda/cuda_split_info.hpp>
#include <LightGBM/cuda/cuda_tree.hpp>
#include "cuda_leaf_splits.hpp"
#define FILL_INDICES_BLOCK_SIZE_DATA_PARTITION (1024)
#define SPLIT_INDICES_BLOCK_SIZE_DATA_PARTITION (1024)
#define AGGREGATE_BLOCK_SIZE_DATA_PARTITION (1024)
namespace LightGBM {
class CUDADataPartition: public NCCLInfo {
public:
CUDADataPartition(
const Dataset* train_data,
const int num_total_bin,
const int num_leaves,
const int num_threads,
const bool use_quantized_grad,
hist_t* cuda_hist);
~CUDADataPartition();
void Init();
void BeforeTrain();
void Split(
// input best split info
const CUDASplitInfo* best_split_info,
const int left_leaf_index,
const int right_leaf_index,
const int leaf_best_split_feature,
const uint32_t leaf_best_split_threshold,
const uint32_t* categorical_bitset,
const int categorical_bitset_len,
const uint8_t leaf_best_split_default_left,
const data_size_t num_data_in_leaf,
const data_size_t leaf_data_start,
// for leaf information update
CUDALeafSplitsStruct* smaller_leaf_splits,
CUDALeafSplitsStruct* larger_leaf_splits,
// gather information for CPU, used for launching kernels
data_size_t* left_leaf_num_data,
data_size_t* right_leaf_num_data,
data_size_t* left_leaf_start,
data_size_t* right_leaf_start,
double* left_leaf_sum_of_hessians,
double* right_leaf_sum_of_hessians,
double* left_leaf_sum_of_gradients,
double* right_leaf_sum_of_gradients,
data_size_t* global_left_leaf_num_data,
data_size_t* global_right_leaf_num_data);
void UpdateTrainScore(const Tree* tree, double* cuda_scores);
void SetUsedDataIndices(const data_size_t* used_indices, const data_size_t num_used_indices);
void SetBaggingSubset(const Dataset* subset);
void ResetTrainingData(const Dataset* train_data, const int num_total_bin, hist_t* cuda_hist);
void ResetConfig(const Config* config, hist_t* cuda_hist);
void ResetByLeafPred(const std::vector<int>& leaf_pred, int num_leaves);
void ReduceLeafGradStat(
const score_t* gradients, const score_t* hessians,
CUDATree* tree, double* leaf_grad_stat_buffer, double* leaf_hess_state_buffer) const;
data_size_t root_num_data() const {
if (use_bagging_) {
return num_used_indices_;
} else {
return num_data_;
}
}
const data_size_t* cuda_data_indices() const { return cuda_data_indices_.RawData(); }
const data_size_t* cuda_leaf_num_data() const { return cuda_leaf_num_data_.RawData(); }
const data_size_t* cuda_leaf_data_start() const { return cuda_leaf_data_start_.RawData(); }
const int* cuda_data_index_to_leaf_index() const { return cuda_data_index_to_leaf_index_.RawData(); }
bool use_bagging() const { return use_bagging_; }
private:
void CalcBlockDim(const data_size_t num_data_in_leaf);
void GenDataToLeftBitVector(
const data_size_t num_data_in_leaf,
const int split_feature_index,
const uint32_t split_threshold,
const uint32_t* categorical_bitset,
const int categorical_bitset_len,
const uint8_t split_default_left,
const data_size_t leaf_data_start,
const int left_leaf_index,
const int right_leaf_index);
void SplitInner(
// input best split info
const data_size_t num_data_in_leaf,
const CUDASplitInfo* best_split_info,
const int left_leaf_index,
const int right_leaf_index,
// for leaf splits information update
CUDALeafSplitsStruct* smaller_leaf_splits,
CUDALeafSplitsStruct* larger_leaf_splits,
// gather information for CPU, used for launching kernels
data_size_t* left_leaf_num_data,
data_size_t* right_leaf_num_data,
data_size_t* left_leaf_start,
data_size_t* right_leaf_start,
double* left_leaf_sum_of_hessians,
double* right_leaf_sum_of_hessians,
double* left_leaf_sum_of_gradients,
double* right_leaf_sum_of_gradients,
data_size_t* global_left_leaf_num_data,
data_size_t* global_right_leaf_num_data);
// kernel launch functions
void LaunchFillDataIndicesBeforeTrain();
void LaunchSplitInnerKernel(
// input best split info
const data_size_t num_data_in_leaf,
const CUDASplitInfo* best_split_info,
const int left_leaf_index,
const int right_leaf_index,
// for leaf splits information update
CUDALeafSplitsStruct* smaller_leaf_splits,
CUDALeafSplitsStruct* larger_leaf_splits,
// gather information for CPU, used for launching kernels
data_size_t* left_leaf_num_data,
data_size_t* right_leaf_num_data,
data_size_t* left_leaf_start,
data_size_t* right_leaf_start,
double* left_leaf_sum_of_hessians,
double* right_leaf_sum_of_hessians,
double* left_leaf_sum_of_gradients,
double* right_leaf_sum_of_gradients,
data_size_t* global_left_leaf_num_data,
data_size_t* global_right_leaf_num_data);
void LaunchGenDataToLeftBitVectorKernel(
const data_size_t num_data_in_leaf,
const int split_feature_index,
const uint32_t split_threshold,
const uint8_t split_default_left,
const data_size_t leaf_data_start,
const int left_leaf_index,
const int right_leaf_index);
void LaunchGenDataToLeftBitVectorCategoricalKernel(
const data_size_t num_data_in_leaf,
const int split_feature_index,
const uint32_t* bitset,
const int bitset_len,
const uint8_t split_default_left,
const data_size_t leaf_data_start,
const int left_leaf_index,
const int right_leaf_index);
#define GenDataToLeftBitVectorKernel_PARAMS \
const BIN_TYPE* column_data, \
const data_size_t num_data_in_leaf, \
const data_size_t* data_indices_in_leaf, \
const uint32_t th, \
const uint32_t t_zero_bin, \
const uint32_t max_bin, \
const uint32_t min_bin, \
const uint8_t split_default_to_left, \
const uint8_t split_missing_default_to_left
template <typename BIN_TYPE>
void LaunchGenDataToLeftBitVectorKernelInner(
GenDataToLeftBitVectorKernel_PARAMS,
const bool missing_is_zero,
const bool missing_is_na,
const bool mfb_is_zero,
const bool mfb_is_na,
const bool max_bin_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, typename BIN_TYPE>
void LaunchGenDataToLeftBitVectorKernelInner0(
GenDataToLeftBitVectorKernel_PARAMS,
const bool missing_is_na,
const bool mfb_is_zero,
const bool mfb_is_na,
const bool max_bin_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, bool MISSING_IS_NA, typename BIN_TYPE>
void LaunchGenDataToLeftBitVectorKernelInner1(
GenDataToLeftBitVectorKernel_PARAMS,
const bool mfb_is_zero,
const bool mfb_is_na,
const bool max_bin_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, bool MISSING_IS_NA, bool MFB_IS_ZERO, typename BIN_TYPE>
void LaunchGenDataToLeftBitVectorKernelInner2(
GenDataToLeftBitVectorKernel_PARAMS,
const bool mfb_is_na,
const bool max_bin_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, bool MISSING_IS_NA, bool MFB_IS_ZERO, bool MFB_IS_NA, typename BIN_TYPE>
void LaunchGenDataToLeftBitVectorKernelInner3(
GenDataToLeftBitVectorKernel_PARAMS,
const bool max_bin_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, bool MISSING_IS_NA, bool MFB_IS_ZERO, bool MFB_IS_NA, bool MAX_TO_LEFT, typename BIN_TYPE>
void LaunchGenDataToLeftBitVectorKernelInner4(
GenDataToLeftBitVectorKernel_PARAMS,
const bool is_single_feature_in_column);
#undef GenDataToLeftBitVectorKernel_PARAMS
#define UpdateDataIndexToLeafIndexKernel_PARAMS \
const BIN_TYPE* column_data, \
const data_size_t num_data_in_leaf, \
const data_size_t* data_indices_in_leaf, \
const uint32_t th, \
const uint32_t t_zero_bin, \
const uint32_t max_bin_ref, \
const uint32_t min_bin_ref, \
const int left_leaf_index, \
const int right_leaf_index, \
const int default_leaf_index, \
const int missing_default_leaf_index
template <typename BIN_TYPE>
void LaunchUpdateDataIndexToLeafIndexKernel(
UpdateDataIndexToLeafIndexKernel_PARAMS,
const bool missing_is_zero,
const bool missing_is_na,
const bool mfb_is_zero,
const bool mfb_is_na,
const bool max_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, typename BIN_TYPE>
void LaunchUpdateDataIndexToLeafIndexKernel_Inner0(
UpdateDataIndexToLeafIndexKernel_PARAMS,
const bool missing_is_na,
const bool mfb_is_zero,
const bool mfb_is_na,
const bool max_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, bool MISSING_IS_NA, typename BIN_TYPE>
void LaunchUpdateDataIndexToLeafIndexKernel_Inner1(
UpdateDataIndexToLeafIndexKernel_PARAMS,
const bool mfb_is_zero,
const bool mfb_is_na,
const bool max_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, bool MISSING_IS_NA, bool MFB_IS_ZERO, typename BIN_TYPE>
void LaunchUpdateDataIndexToLeafIndexKernel_Inner2(
UpdateDataIndexToLeafIndexKernel_PARAMS,
const bool mfb_is_na,
const bool max_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, bool MISSING_IS_NA, bool MFB_IS_ZERO, bool MFB_IS_NA, typename BIN_TYPE>
void LaunchUpdateDataIndexToLeafIndexKernel_Inner3(
UpdateDataIndexToLeafIndexKernel_PARAMS,
const bool max_to_left,
const bool is_single_feature_in_column);
template <bool MIN_IS_MAX, bool MISSING_IS_ZERO, bool MISSING_IS_NA, bool MFB_IS_ZERO, bool MFB_IS_NA, bool MAX_TO_LEFT, typename BIN_TYPE>
void LaunchUpdateDataIndexToLeafIndexKernel_Inner4(
UpdateDataIndexToLeafIndexKernel_PARAMS,
const bool is_single_feature_in_column);
#undef UpdateDataIndexToLeafIndexKernel_PARAMS
void LaunchAddPredictionToScoreKernel(const double* leaf_value, double* cuda_scores);
void LaunchFillDataIndexToLeafIndex();
void LaunchReduceLeafGradStat(
const score_t* gradients, const score_t* hessians,
CUDATree* tree, double* leaf_grad_stat_buffer, double* leaf_hess_state_buffer) const;
// Host memory
// dataset information
/*! \brief number of training data */
data_size_t num_data_;
/*! \brief number of features in training data */
int num_features_;
/*! \brief number of total bins in training data */
int num_total_bin_;
/*! \brief bin data stored by column */
const CUDAColumnData* cuda_column_data_;
/*! \brief grid dimension when splitting one leaf */
int grid_dim_;
/*! \brief block dimension when splitting one leaf */
int block_dim_;
/*! \brief data indices used in this iteration */
const data_size_t* used_indices_;
/*! \brief marks whether a feature is a categorical feature */
std::vector<bool> is_categorical_feature_;
/*! \brief marks whether a feature is the only feature in its group */
std::vector<bool> is_single_feature_in_column_;
// config information
/*! \brief maximum number of leaves in a tree */
int num_leaves_;
/*! \brief number of threads */
int num_threads_;
/*! \brief whether to use quantized gradients */
bool use_quantized_grad_;
// per iteration information
/*! \brief whether bagging is used in this iteration */
bool use_bagging_;
/*! \brief number of used data indices in this iteration */
data_size_t num_used_indices_;
// tree structure information
/*! \brief current number of leaves in tree */
int cur_num_leaves_;
// split algorithm related
/*! \brief maximum number of blocks to aggregate after finding bit vector by blocks */
int max_num_split_indices_blocks_;
// CUDA streams
/*! \brief cuda streams used for asynchronizing kernel computing and memory copy */
std::vector<cudaStream_t> cuda_streams_;
// CUDA memory, held by this object
// tree structure information
/*! \brief data indices by leaf */
CUDAVector<data_size_t> cuda_data_indices_;
/*! \brief start position of each leaf in cuda_data_indices_ */
CUDAVector<data_size_t> cuda_leaf_data_start_;
/*! \brief end position of each leaf in cuda_data_indices_ */
CUDAVector<data_size_t> cuda_leaf_data_end_;
/*! \brief number of data in each leaf */
CUDAVector<data_size_t> cuda_leaf_num_data_;
/*! \brief records the histogram of each leaf */
CUDAVector<hist_t*> cuda_hist_pool_;
/*! \brief records the value of each leaf */
CUDAVector<double> cuda_leaf_output_;
// split data algorithm related
CUDAVector<uint16_t> cuda_block_to_left_offset_;
/*! \brief maps data index to leaf index, for adding scores to training data set */
CUDAVector<int> cuda_data_index_to_leaf_index_;
/*! \brief prefix sum of number of data going to left in all blocks */
CUDAVector<data_size_t> cuda_block_data_to_left_offset_;
/*! \brief prefix sum of number of data going to right in all blocks */
CUDAVector<data_size_t> cuda_block_data_to_right_offset_;
/*! \brief buffer for splitting data indices, will be copied back to cuda_data_indices_ after split */
CUDAVector<data_size_t> cuda_out_data_indices_in_leaf_;
// split tree structure algorithm related
/*! \brief buffer to store split information, prepared to be copied to cpu */
CUDAVector<int> cuda_split_info_buffer_;
// dataset information
/*! \brief number of data in training set, for initialization of cuda_leaf_num_data_ and cuda_leaf_data_end_ */
CUDAVector<data_size_t> cuda_num_data_;
// CUDA memory, held by other object
// dataset information
/*! \brief beginning of histograms, for initialization of cuda_hist_pool_ */
hist_t* cuda_hist_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_DATA_PARTITION_HPP_
src/treelearner/cuda/cuda_gradient_discretizer.cu
+185
View File
@@ -0,0 +1,185 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include <algorithm>
#include <LightGBM/cuda/cuda_algorithms.hpp>
#include "cuda_gradient_discretizer.hpp"
namespace LightGBM {
__global__ void ReduceMinMaxKernel(
const data_size_t num_data,
const score_t* input_gradients,
const score_t* input_hessians,
score_t* grad_min_block_buffer,
score_t* grad_max_block_buffer,
score_t* hess_min_block_buffer,
score_t* hess_max_block_buffer) {
__shared__ score_t shared_mem_buffer[WARPSIZE];
const data_size_t index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
score_t grad_max_val = kMinScore;
score_t grad_min_val = kMaxScore;
score_t hess_max_val = kMinScore;
score_t hess_min_val = kMaxScore;
if (index < num_data) {
grad_max_val = input_gradients[index];
grad_min_val = input_gradients[index];
hess_max_val = input_hessians[index];
hess_min_val = input_hessians[index];
}
grad_min_val = ShuffleReduceMin<score_t>(grad_min_val, shared_mem_buffer, blockDim.x);
__syncthreads();
grad_max_val = ShuffleReduceMax<score_t>(grad_max_val, shared_mem_buffer, blockDim.x);
__syncthreads();
hess_min_val = ShuffleReduceMin<score_t>(hess_min_val, shared_mem_buffer, blockDim.x);
__syncthreads();
hess_max_val = ShuffleReduceMax<score_t>(hess_max_val, shared_mem_buffer, blockDim.x);
if (threadIdx.x == 0) {
grad_min_block_buffer[blockIdx.x] = grad_min_val;
grad_max_block_buffer[blockIdx.x] = grad_max_val;
hess_min_block_buffer[blockIdx.x] = hess_min_val;
hess_max_block_buffer[blockIdx.x] = hess_max_val;
}
}
__global__ void ReduceBlockMinMaxKernel(
const int num_blocks,
const int grad_discretize_bins,
score_t* grad_min_block_buffer,
score_t* grad_max_block_buffer,
score_t* hess_min_block_buffer,
score_t* hess_max_block_buffer) {
__shared__ score_t shared_mem_buffer[WARPSIZE];
score_t grad_max_val = kMinScore;
score_t grad_min_val = kMaxScore;
score_t hess_max_val = kMinScore;
score_t hess_min_val = kMaxScore;
for (int block_index = static_cast<int>(threadIdx.x); block_index < num_blocks; block_index += static_cast<int>(blockDim.x)) {
grad_min_val = min(grad_min_val, grad_min_block_buffer[block_index]);
grad_max_val = max(grad_max_val, grad_max_block_buffer[block_index]);
hess_min_val = min(hess_min_val, hess_min_block_buffer[block_index]);
hess_max_val = max(hess_max_val, hess_max_block_buffer[block_index]);
}
grad_min_val = ShuffleReduceMin<score_t>(grad_min_val, shared_mem_buffer, blockDim.x);
__syncthreads();
grad_max_val = ShuffleReduceMax<score_t>(grad_max_val, shared_mem_buffer, blockDim.x);
__syncthreads();
hess_max_val = ShuffleReduceMax<score_t>(hess_max_val, shared_mem_buffer, blockDim.x);
__syncthreads();
hess_max_val = ShuffleReduceMax<score_t>(hess_max_val, shared_mem_buffer, blockDim.x);
if (threadIdx.x == 0) {
const score_t grad_abs_max = max(fabs(grad_min_val), fabs(grad_max_val));
const score_t hess_abs_max = max(fabs(hess_min_val), fabs(hess_max_val));
grad_min_block_buffer[0] = 1.0f / (grad_abs_max / (grad_discretize_bins / 2));
grad_max_block_buffer[0] = (grad_abs_max / (grad_discretize_bins / 2));
hess_min_block_buffer[0] = 1.0f / (hess_abs_max / (grad_discretize_bins));
hess_max_block_buffer[0] = (hess_abs_max / (grad_discretize_bins));
}
}
template <bool STOCHASTIC_ROUNDING>
__global__ void DiscretizeGradientsKernel(
const data_size_t num_data,
const score_t* input_gradients,
const score_t* input_hessians,
const score_t* grad_scale_ptr,
const score_t* hess_scale_ptr,
const int iter,
const int* random_values_use_start,
const score_t* gradient_random_values,
const score_t* hessian_random_values,
const int grad_discretize_bins,
int8_t* output_gradients_and_hessians) {
const int start = random_values_use_start[iter];
const data_size_t index = static_cast<data_size_t>(threadIdx.x + blockIdx.x * blockDim.x);
const score_t grad_scale = *grad_scale_ptr;
const score_t hess_scale = *hess_scale_ptr;
int16_t* output_gradients_and_hessians_ptr = reinterpret_cast<int16_t*>(output_gradients_and_hessians);
if (index < num_data) {
if (STOCHASTIC_ROUNDING) {
const data_size_t index_offset = (index + start) % num_data;
const score_t gradient = input_gradients[index];
const score_t hessian = input_hessians[index];
const score_t gradient_random_value = gradient_random_values[index_offset];
const score_t hessian_random_value = hessian_random_values[index_offset];
output_gradients_and_hessians_ptr[2 * index + 1] = gradient > 0.0f ?
static_cast<int16_t>(gradient * grad_scale + gradient_random_value) :
static_cast<int16_t>(gradient * grad_scale - gradient_random_value);
output_gradients_and_hessians_ptr[2 * index] = static_cast<int16_t>(hessian * hess_scale + hessian_random_value);
} else {
const score_t gradient = input_gradients[index];
const score_t hessian = input_hessians[index];
output_gradients_and_hessians_ptr[2 * index + 1] = gradient > 0.0f ?
static_cast<int16_t>(gradient * grad_scale + 0.5) :
static_cast<int16_t>(gradient * grad_scale - 0.5);
output_gradients_and_hessians_ptr[2 * index] = static_cast<int16_t>(hessian * hess_scale + 0.5);
}
}
}
void CUDAGradientDiscretizer::DiscretizeGradients(
const data_size_t num_data,
const score_t* input_gradients,
const score_t* input_hessians) {
ReduceMinMaxKernel<<<num_reduce_blocks_, CUDA_GRADIENT_DISCRETIZER_BLOCK_SIZE>>>(
num_data, input_gradients, input_hessians,
grad_min_block_buffer_.RawData(),
grad_max_block_buffer_.RawData(),
hess_min_block_buffer_.RawData(),
hess_max_block_buffer_.RawData());
SynchronizeCUDADevice(__FILE__, __LINE__);
ReduceBlockMinMaxKernel<<<1, CUDA_GRADIENT_DISCRETIZER_BLOCK_SIZE>>>(
num_reduce_blocks_,
num_grad_quant_bins_,
grad_min_block_buffer_.RawData(),
grad_max_block_buffer_.RawData(),
hess_min_block_buffer_.RawData(),
hess_max_block_buffer_.RawData());
SynchronizeCUDADevice(__FILE__, __LINE__);
if (nccl_communicator_ != nullptr) {
SynchronizeCUDADevice(__FILE__, __LINE__);
cudaStream_t cuda_stream = CUDAStreamCreate();
NCCLGroupStart();
NCCLAllReduce<score_t>(grad_min_block_buffer_.RawDataReadOnly(), grad_min_block_buffer_.RawData(), 1, ncclFloat32, ncclMin, nccl_communicator_, cuda_stream);
NCCLAllReduce<score_t>(hess_min_block_buffer_.RawDataReadOnly(), hess_min_block_buffer_.RawData(), 1, ncclFloat32, ncclMin, nccl_communicator_, cuda_stream);
NCCLAllReduce<score_t>(grad_max_block_buffer_.RawDataReadOnly(), grad_max_block_buffer_.RawData(), 1, ncclFloat32, ncclMax, nccl_communicator_, cuda_stream);
NCCLAllReduce<score_t>(hess_max_block_buffer_.RawDataReadOnly(), hess_max_block_buffer_.RawData(), 1, ncclFloat32, ncclMax, nccl_communicator_, cuda_stream);
NCCLGroupEnd();
SynchronizeCUDAStream(cuda_stream, __FILE__, __LINE__);
CUDAStreamDestroy(cuda_stream);
}
#define DiscretizeGradientsKernel_ARGS \
num_data, \
input_gradients, \
input_hessians, \
grad_min_block_buffer_.RawData(), \
hess_min_block_buffer_.RawData(), \
iter_, \
random_values_use_start_.RawData(), \
gradient_random_values_.RawData(), \
hessian_random_values_.RawData(), \
num_grad_quant_bins_, \
discretized_gradients_and_hessians_.RawData()
if (stochastic_rounding_) {
DiscretizeGradientsKernel<true><<<num_reduce_blocks_, CUDA_GRADIENT_DISCRETIZER_BLOCK_SIZE>>>(DiscretizeGradientsKernel_ARGS);
} else {
DiscretizeGradientsKernel<false><<<num_reduce_blocks_, CUDA_GRADIENT_DISCRETIZER_BLOCK_SIZE>>>(DiscretizeGradientsKernel_ARGS);
}
SynchronizeCUDADevice(__FILE__, __LINE__);
++iter_;
}
} // namespace LightGBM
#endif // USE_CUDA
src/treelearner/cuda/cuda_gradient_discretizer.hpp
+121
View File
@@ -0,0 +1,121 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_GRADIENT_DISCRETIZER_HPP_
#define LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_GRADIENT_DISCRETIZER_HPP_
#ifdef USE_CUDA
#include <LightGBM/bin.h>
#include <LightGBM/meta.h>
#include <LightGBM/cuda/cuda_utils.hu>
#include <LightGBM/utils/threading.h>
#include <algorithm>
#include <random>
#include <vector>
#include "cuda_leaf_splits.hpp"
#include "../gradient_discretizer.hpp"
namespace LightGBM {
#define CUDA_GRADIENT_DISCRETIZER_BLOCK_SIZE (1024)
class CUDAGradientDiscretizer: public GradientDiscretizer, public NCCLInfo {
public:
CUDAGradientDiscretizer(int num_grad_quant_bins, int num_trees, int random_seed, bool is_constant_hessian, bool stochastic_roudning):
GradientDiscretizer(num_grad_quant_bins, num_trees, random_seed, is_constant_hessian, stochastic_roudning) {
}
~CUDAGradientDiscretizer() {}
void DiscretizeGradients(
const data_size_t num_data,
const score_t* input_gradients,
const score_t* input_hessians) override;
const int8_t* discretized_gradients_and_hessians() const override { return discretized_gradients_and_hessians_.RawData(); }
double grad_scale() const override {
Log::Fatal("grad_scale() of CUDAGradientDiscretizer should not be called.");
return 0.0;
}
double hess_scale() const override {
Log::Fatal("hess_scale() of CUDAGradientDiscretizer should not be called.");
return 0.0;
}
const score_t* grad_scale_ptr() const { return grad_max_block_buffer_.RawData(); }
const score_t* hess_scale_ptr() const { return hess_max_block_buffer_.RawData(); }
void Init(const data_size_t num_data, const int num_leaves,
const int num_features, const Dataset* train_data) override {
GradientDiscretizer::Init(num_data, num_leaves, num_features, train_data);
discretized_gradients_and_hessians_.Resize(num_data * 2);
num_reduce_blocks_ = (num_data + CUDA_GRADIENT_DISCRETIZER_BLOCK_SIZE - 1) / CUDA_GRADIENT_DISCRETIZER_BLOCK_SIZE;
grad_min_block_buffer_.Resize(num_reduce_blocks_);
grad_max_block_buffer_.Resize(num_reduce_blocks_);
hess_min_block_buffer_.Resize(num_reduce_blocks_);
hess_max_block_buffer_.Resize(num_reduce_blocks_);
random_values_use_start_.Resize(num_trees_);
gradient_random_values_.Resize(num_data);
hessian_random_values_.Resize(num_data);
std::vector<score_t> gradient_random_values(num_data, 0.0f);
std::vector<score_t> hessian_random_values(num_data, 0.0f);
std::vector<int> random_values_use_start(num_trees_, 0);
const int num_threads = OMP_NUM_THREADS();
std::mt19937 random_values_use_start_eng = std::mt19937(random_seed_);
std::uniform_int_distribution<data_size_t> random_values_use_start_dist = std::uniform_int_distribution<data_size_t>(0, num_data);
for (int tree_index = 0; tree_index < num_trees_; ++tree_index) {
random_values_use_start[tree_index] = random_values_use_start_dist(random_values_use_start_eng);
}
int num_blocks = 0;
data_size_t block_size = 0;
Threading::BlockInfo<data_size_t>(num_data, 512, &num_blocks, &block_size);
#pragma omp parallel for schedule(static, 1) num_threads(num_threads)
for (int thread_id = 0; thread_id < num_blocks; ++thread_id) {
const data_size_t start = thread_id * block_size;
const data_size_t end = std::min(start + block_size, num_data);
std::mt19937 gradient_random_values_eng(random_seed_ + thread_id);
std::uniform_real_distribution<double> gradient_random_values_dist(0.0f, 1.0f);
std::mt19937 hessian_random_values_eng(random_seed_ + thread_id + num_threads);
std::uniform_real_distribution<double> hessian_random_values_dist(0.0f, 1.0f);
for (data_size_t i = start; i < end; ++i) {
gradient_random_values[i] = gradient_random_values_dist(gradient_random_values_eng);
hessian_random_values[i] = hessian_random_values_dist(hessian_random_values_eng);
}
}
CopyFromHostToCUDADevice<score_t>(gradient_random_values_.RawData(), gradient_random_values.data(), gradient_random_values.size(), __FILE__, __LINE__);
CopyFromHostToCUDADevice<score_t>(hessian_random_values_.RawData(), hessian_random_values.data(), hessian_random_values.size(), __FILE__, __LINE__);
CopyFromHostToCUDADevice<int>(random_values_use_start_.RawData(), random_values_use_start.data(), random_values_use_start.size(), __FILE__, __LINE__);
iter_ = 0;
}
protected:
mutable CUDAVector<int8_t> discretized_gradients_and_hessians_;
mutable CUDAVector<score_t> grad_min_block_buffer_;
mutable CUDAVector<score_t> grad_max_block_buffer_;
mutable CUDAVector<score_t> hess_min_block_buffer_;
mutable CUDAVector<score_t> hess_max_block_buffer_;
CUDAVector<int> random_values_use_start_;
CUDAVector<score_t> gradient_random_values_;
CUDAVector<score_t> hessian_random_values_;
int num_reduce_blocks_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_GRADIENT_DISCRETIZER_HPP_
src/treelearner/cuda/cuda_histogram_constructor.cpp
+192
View File
@@ -0,0 +1,192 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include "cuda_histogram_constructor.hpp"
#include <algorithm>
#include <vector>
namespace LightGBM {
CUDAHistogramConstructor::CUDAHistogramConstructor(
const Dataset* train_data,
const int num_leaves,
const int num_threads,
const std::vector<uint32_t>& feature_hist_offsets,
const int min_data_in_leaf,
const double min_sum_hessian_in_leaf,
const int gpu_device_id,
const bool gpu_use_dp,
const bool use_quantized_grad,
const int num_grad_quant_bins):
num_data_(train_data->num_data()),
num_features_(train_data->num_features()),
num_leaves_(num_leaves),
num_threads_(num_threads),
min_data_in_leaf_(min_data_in_leaf),
min_sum_hessian_in_leaf_(min_sum_hessian_in_leaf),
gpu_device_id_(gpu_device_id),
gpu_use_dp_(gpu_use_dp),
use_quantized_grad_(use_quantized_grad),
num_grad_quant_bins_(num_grad_quant_bins) {
InitFeatureMetaInfo(train_data, feature_hist_offsets);
cuda_row_data_.reset(nullptr);
}
CUDAHistogramConstructor::~CUDAHistogramConstructor() {
gpuAssert(cudaStreamDestroy(cuda_stream_), __FILE__, __LINE__);
}
void CUDAHistogramConstructor::InitFeatureMetaInfo(const Dataset* train_data, const std::vector<uint32_t>& feature_hist_offsets) {
need_fix_histogram_features_.clear();
need_fix_histogram_features_num_bin_aligend_.clear();
feature_num_bins_.clear();
feature_most_freq_bins_.clear();
for (int feature_index = 0; feature_index < train_data->num_features(); ++feature_index) {
const BinMapper* bin_mapper = train_data->FeatureBinMapper(feature_index);
const uint32_t most_freq_bin = bin_mapper->GetMostFreqBin();
if (most_freq_bin != 0) {
need_fix_histogram_features_.emplace_back(feature_index);
uint32_t num_bin_ref = static_cast<uint32_t>(bin_mapper->num_bin()) - 1;
uint32_t num_bin_aligned = 1;
while (num_bin_ref > 0) {
num_bin_aligned <<= 1;
num_bin_ref >>= 1;
}
need_fix_histogram_features_num_bin_aligend_.emplace_back(num_bin_aligned);
}
feature_num_bins_.emplace_back(static_cast<uint32_t>(bin_mapper->num_bin()));
feature_most_freq_bins_.emplace_back(most_freq_bin);
}
feature_hist_offsets_.clear();
for (size_t i = 0; i < feature_hist_offsets.size(); ++i) {
feature_hist_offsets_.emplace_back(feature_hist_offsets[i]);
}
if (feature_hist_offsets.empty()) {
num_total_bin_ = 0;
} else {
num_total_bin_ = static_cast<int>(feature_hist_offsets.back());
}
}
void CUDAHistogramConstructor::BeforeTrain(const score_t* gradients, const score_t* hessians) {
cuda_gradients_ = gradients;
cuda_hessians_ = hessians;
cuda_hist_.SetValue(0);
}
void CUDAHistogramConstructor::Init(const Dataset* train_data, TrainingShareStates* share_state) {
cuda_hist_.Resize(static_cast<size_t>(num_total_bin_ * 2 * num_leaves_));
cuda_hist_.SetValue(0);
cuda_feature_num_bins_.InitFromHostVector(feature_num_bins_);
cuda_feature_hist_offsets_.InitFromHostVector(feature_hist_offsets_);
cuda_feature_most_freq_bins_.InitFromHostVector(feature_most_freq_bins_);
cuda_row_data_.reset(new CUDARowData(train_data, share_state, gpu_device_id_, gpu_use_dp_));
cuda_row_data_->Init(train_data, share_state);
CUDASUCCESS_OR_FATAL(cudaStreamCreate(&cuda_stream_));
cuda_need_fix_histogram_features_.InitFromHostVector(need_fix_histogram_features_);
cuda_need_fix_histogram_features_num_bin_aligned_.InitFromHostVector(need_fix_histogram_features_num_bin_aligend_);
if (cuda_row_data_->NumLargeBinPartition() > 0) {
int grid_dim_x = 0, grid_dim_y = 0, block_dim_x = 0, block_dim_y = 0;
CalcConstructHistogramKernelDim(&grid_dim_x, &grid_dim_y, &block_dim_x, &block_dim_y, num_data_);
const size_t buffer_size = static_cast<size_t>(grid_dim_y) * static_cast<size_t>(num_total_bin_);
if (!use_quantized_grad_) {
if (gpu_use_dp_) {
// need to double the size of histogram buffer in global memory when using double precision in histogram construction
cuda_hist_buffer_.Resize(buffer_size * 4);
} else {
cuda_hist_buffer_.Resize(buffer_size * 2);
}
} else {
// use only half the size of histogram buffer in global memory when quantized training since each gradient and hessian takes only 2 bytes
cuda_hist_buffer_.Resize(buffer_size);
}
}
hist_buffer_for_num_bit_change_.Resize(num_total_bin_ * 2);
}
void CUDAHistogramConstructor::ConstructHistogramForLeaf(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const CUDALeafSplitsStruct* /*cuda_larger_leaf_splits*/,
const data_size_t global_num_data_in_smaller_leaf,
const data_size_t global_num_data_in_larger_leaf,
const data_size_t num_data_in_smaller_leaf,
const data_size_t /*num_data_in_larger_leaf*/,
const double sum_hessians_in_smaller_leaf,
const double sum_hessians_in_larger_leaf,
const uint8_t num_bits_in_histogram_bins) {
if ((global_num_data_in_smaller_leaf <= min_data_in_leaf_ || sum_hessians_in_smaller_leaf <= min_sum_hessian_in_leaf_) &&
(global_num_data_in_larger_leaf <= min_data_in_leaf_ || sum_hessians_in_larger_leaf <= min_sum_hessian_in_leaf_)) {
return;
}
LaunchConstructHistogramKernel(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDAHistogramConstructor::SubtractHistogramForLeaf(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const CUDALeafSplitsStruct* cuda_larger_leaf_splits,
const bool use_quantized_grad,
const uint8_t parent_num_bits_in_histogram_bins,
const uint8_t smaller_num_bits_in_histogram_bins,
const uint8_t larger_num_bits_in_histogram_bins) {
global_timer.Start("CUDAHistogramConstructor::ConstructHistogramForLeaf::LaunchSubtractHistogramKernel");
LaunchSubtractHistogramKernel(cuda_smaller_leaf_splits, cuda_larger_leaf_splits, use_quantized_grad,
parent_num_bits_in_histogram_bins, smaller_num_bits_in_histogram_bins, larger_num_bits_in_histogram_bins);
global_timer.Stop("CUDAHistogramConstructor::ConstructHistogramForLeaf::LaunchSubtractHistogramKernel");
}
void CUDAHistogramConstructor::CalcConstructHistogramKernelDim(
int* grid_dim_x,
int* grid_dim_y,
int* block_dim_x,
int* block_dim_y,
const data_size_t num_data_in_smaller_leaf) {
*block_dim_x = cuda_row_data_->max_num_column_per_partition();
*block_dim_y = NUM_THREADS_PER_BLOCK / cuda_row_data_->max_num_column_per_partition();
*grid_dim_x = cuda_row_data_->num_feature_partitions();
*grid_dim_y = std::max(min_grid_dim_y_,
((num_data_in_smaller_leaf + NUM_DATA_PER_THREAD - 1) / NUM_DATA_PER_THREAD + (*block_dim_y) - 1) / (*block_dim_y));
}
void CUDAHistogramConstructor::ResetTrainingData(const Dataset* train_data, TrainingShareStates* share_states) {
num_data_ = train_data->num_data();
num_features_ = train_data->num_features();
InitFeatureMetaInfo(train_data, share_states->feature_hist_offsets());
cuda_hist_.Resize(static_cast<size_t>(num_total_bin_ * 2 * num_leaves_));
cuda_hist_.SetValue(0);
cuda_feature_num_bins_.InitFromHostVector(feature_num_bins_);
cuda_feature_hist_offsets_.InitFromHostVector(feature_hist_offsets_);
cuda_feature_most_freq_bins_.InitFromHostVector(feature_most_freq_bins_);
cuda_row_data_.reset(new CUDARowData(train_data, share_states, gpu_device_id_, gpu_use_dp_));
cuda_row_data_->Init(train_data, share_states);
cuda_need_fix_histogram_features_.InitFromHostVector(need_fix_histogram_features_);
cuda_need_fix_histogram_features_num_bin_aligned_.InitFromHostVector(need_fix_histogram_features_num_bin_aligend_);
}
void CUDAHistogramConstructor::ResetConfig(const Config* config) {
num_threads_ = OMP_NUM_THREADS();
num_leaves_ = config->num_leaves;
min_data_in_leaf_ = config->min_data_in_leaf;
min_sum_hessian_in_leaf_ = config->min_sum_hessian_in_leaf;
cuda_hist_.Resize(static_cast<size_t>(num_total_bin_ * 2 * num_leaves_));
cuda_hist_.SetValue(0);
}
} // namespace LightGBM
#endif // USE_CUDA
src/treelearner/cuda/cuda_histogram_constructor.cu
+963
View File
@@ -0,0 +1,963 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
* Modifications Copyright(C) 2023 Advanced Micro Devices, Inc. All rights reserved.
*/
#ifdef USE_CUDA
#include "cuda_histogram_constructor.hpp"
#include <LightGBM/cuda/cuda_algorithms.hpp>
#include <LightGBM/cuda/cuda_rocm_interop.h>
#include <algorithm>
namespace LightGBM {
template <typename BIN_TYPE, typename HIST_TYPE, size_t SHARED_HIST_SIZE>
__global__ void CUDAConstructHistogramDenseKernel(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const score_t* cuda_gradients,
const score_t* cuda_hessians,
const BIN_TYPE* data,
const uint32_t* column_hist_offsets,
const uint32_t* column_hist_offsets_full,
const int* feature_partition_column_index_offsets,
const data_size_t num_data) {
const int dim_y = static_cast<int>(gridDim.y * blockDim.y);
const data_size_t num_data_in_smaller_leaf = smaller_leaf_splits->num_data_in_leaf;
const data_size_t num_data_per_thread = (num_data_in_smaller_leaf + dim_y - 1) / dim_y;
const data_size_t* data_indices_ref = smaller_leaf_splits->data_indices_in_leaf;
__shared__ HIST_TYPE shared_hist[SHARED_HIST_SIZE];
const unsigned int num_threads_per_block = blockDim.x * blockDim.y;
const int partition_column_start = feature_partition_column_index_offsets[blockIdx.x];
const int partition_column_end = feature_partition_column_index_offsets[blockIdx.x + 1];
const BIN_TYPE* data_ptr = data + static_cast<size_t>(partition_column_start) * num_data;
const int num_columns_in_partition = partition_column_end - partition_column_start;
const uint32_t partition_hist_start = column_hist_offsets_full[blockIdx.x];
const uint32_t partition_hist_end = column_hist_offsets_full[blockIdx.x + 1];
const uint32_t num_items_in_partition = (partition_hist_end - partition_hist_start) << 1;
const unsigned int thread_idx = threadIdx.x + threadIdx.y * blockDim.x;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
shared_hist[i] = 0.0f;
}
__syncthreads();
const unsigned int blockIdx_y = blockIdx.y;
const data_size_t block_start = (static_cast<size_t>(blockIdx_y) * blockDim.y) * num_data_per_thread;
const data_size_t* data_indices_ref_this_block = data_indices_ref + block_start;
data_size_t block_num_data = max(0, min(num_data_in_smaller_leaf - block_start, num_data_per_thread * static_cast<data_size_t>(blockDim.y)));
const int column_index = static_cast<int>(threadIdx.x) + partition_column_start;
if (threadIdx.x < static_cast<unsigned int>(num_columns_in_partition)) {
HIST_TYPE* shared_hist_ptr = shared_hist + (column_hist_offsets[column_index] << 1);
for (data_size_t inner_data_index = static_cast<data_size_t>(threadIdx.y); inner_data_index < block_num_data; inner_data_index += blockDim.y) {
const data_size_t data_index = data_indices_ref_this_block[inner_data_index];
const score_t grad = cuda_gradients[data_index];
const score_t hess = cuda_hessians[data_index];
const uint32_t bin = static_cast<uint32_t>(data_ptr[static_cast<size_t>(data_index) * num_columns_in_partition + threadIdx.x]);
const uint32_t pos = bin << 1;
HIST_TYPE* pos_ptr = shared_hist_ptr + pos;
atomicAdd_block(pos_ptr, grad);
atomicAdd_block(pos_ptr + 1, hess);
}
}
__syncthreads();
hist_t* feature_histogram_ptr = smaller_leaf_splits->hist_in_leaf + (partition_hist_start << 1);
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
atomicAdd_system(feature_histogram_ptr + i, shared_hist[i]);
}
}
template <typename BIN_TYPE, typename DATA_PTR_TYPE, typename HIST_TYPE, size_t SHARED_HIST_SIZE>
__global__ void CUDAConstructHistogramSparseKernel(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const score_t* cuda_gradients,
const score_t* cuda_hessians,
const BIN_TYPE* data,
const DATA_PTR_TYPE* row_ptr,
const DATA_PTR_TYPE* partition_ptr,
const uint32_t* column_hist_offsets_full,
const data_size_t num_data) {
const int dim_y = static_cast<int>(gridDim.y * blockDim.y);
const data_size_t num_data_in_smaller_leaf = smaller_leaf_splits->num_data_in_leaf;
const data_size_t num_data_per_thread = (num_data_in_smaller_leaf + dim_y - 1) / dim_y;
const data_size_t* data_indices_ref = smaller_leaf_splits->data_indices_in_leaf;
__shared__ HIST_TYPE shared_hist[SHARED_HIST_SIZE];
const unsigned int num_threads_per_block = blockDim.x * blockDim.y;
const DATA_PTR_TYPE* block_row_ptr = row_ptr + static_cast<size_t>(blockIdx.x) * (num_data + 1);
const BIN_TYPE* data_ptr = data + partition_ptr[blockIdx.x];
const uint32_t partition_hist_start = column_hist_offsets_full[blockIdx.x];
const uint32_t partition_hist_end = column_hist_offsets_full[blockIdx.x + 1];
const uint32_t num_items_in_partition = (partition_hist_end - partition_hist_start) << 1;
const unsigned int thread_idx = threadIdx.x + threadIdx.y * blockDim.x;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
shared_hist[i] = 0.0f;
}
__syncthreads();
const unsigned int threadIdx_y = threadIdx.y;
const unsigned int blockIdx_y = blockIdx.y;
const data_size_t block_start = (blockIdx_y * blockDim.y) * num_data_per_thread;
const data_size_t* data_indices_ref_this_block = data_indices_ref + block_start;
data_size_t block_num_data = max(0, min(num_data_in_smaller_leaf - block_start, num_data_per_thread * static_cast<data_size_t>(blockDim.y)));
const data_size_t num_iteration_total = (block_num_data + blockDim.y - 1) / blockDim.y;
const data_size_t remainder = block_num_data % blockDim.y;
const data_size_t num_iteration_this = remainder == 0 ? num_iteration_total : num_iteration_total - static_cast<data_size_t>(threadIdx_y >= remainder);
data_size_t inner_data_index = static_cast<data_size_t>(threadIdx_y);
for (data_size_t i = 0; i < num_iteration_this; ++i) {
const data_size_t data_index = data_indices_ref_this_block[inner_data_index];
const DATA_PTR_TYPE row_start = block_row_ptr[data_index];
const DATA_PTR_TYPE row_end = block_row_ptr[data_index + 1];
const DATA_PTR_TYPE row_size = row_end - row_start;
if (threadIdx.x < row_size) {
const score_t grad = cuda_gradients[data_index];
const score_t hess = cuda_hessians[data_index];
const uint32_t bin = static_cast<uint32_t>(data_ptr[row_start + threadIdx.x]);
const uint32_t pos = bin << 1;
HIST_TYPE* pos_ptr = shared_hist + pos;
atomicAdd_block(pos_ptr, grad);
atomicAdd_block(pos_ptr + 1, hess);
}
inner_data_index += blockDim.y;
}
__syncthreads();
hist_t* feature_histogram_ptr = smaller_leaf_splits->hist_in_leaf + (partition_hist_start << 1);
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
atomicAdd_system(feature_histogram_ptr + i, shared_hist[i]);
}
}
template <typename BIN_TYPE, typename HIST_TYPE>
__global__ void CUDAConstructHistogramDenseKernel_GlobalMemory(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const score_t* cuda_gradients,
const score_t* cuda_hessians,
const BIN_TYPE* data,
const uint32_t* column_hist_offsets,
const uint32_t* column_hist_offsets_full,
const int* feature_partition_column_index_offsets,
const data_size_t num_data,
HIST_TYPE* global_hist_buffer) {
const int dim_y = static_cast<int>(gridDim.y * blockDim.y);
const data_size_t num_data_in_smaller_leaf = smaller_leaf_splits->num_data_in_leaf;
const data_size_t num_data_per_thread = (num_data_in_smaller_leaf + dim_y - 1) / dim_y;
const data_size_t* data_indices_ref = smaller_leaf_splits->data_indices_in_leaf;
const unsigned int num_threads_per_block = blockDim.x * blockDim.y;
const int partition_column_start = feature_partition_column_index_offsets[blockIdx.x];
const int partition_column_end = feature_partition_column_index_offsets[blockIdx.x + 1];
const BIN_TYPE* data_ptr = data + static_cast<size_t>(partition_column_start) * num_data;
const int num_columns_in_partition = partition_column_end - partition_column_start;
const uint32_t partition_hist_start = column_hist_offsets_full[blockIdx.x];
const uint32_t partition_hist_end = column_hist_offsets_full[blockIdx.x + 1];
const uint32_t num_items_in_partition = (partition_hist_end - partition_hist_start) << 1;
const unsigned int thread_idx = threadIdx.x + threadIdx.y * blockDim.x;
const int num_total_bin = column_hist_offsets_full[gridDim.x];
HIST_TYPE* shared_hist = global_hist_buffer + (blockIdx.y * num_total_bin + partition_hist_start) * 2;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
shared_hist[i] = 0.0f;
}
__syncthreads();
const unsigned int threadIdx_y = threadIdx.y;
const unsigned int blockIdx_y = blockIdx.y;
const data_size_t block_start = (static_cast<size_t>(blockIdx_y) * blockDim.y) * num_data_per_thread;
const data_size_t* data_indices_ref_this_block = data_indices_ref + block_start;
data_size_t block_num_data = max(0, min(num_data_in_smaller_leaf - block_start, num_data_per_thread * static_cast<data_size_t>(blockDim.y)));
const data_size_t num_iteration_total = (block_num_data + blockDim.y - 1) / blockDim.y;
const data_size_t remainder = block_num_data % blockDim.y;
const data_size_t num_iteration_this = remainder == 0 ? num_iteration_total : num_iteration_total - static_cast<data_size_t>(threadIdx_y >= remainder);
data_size_t inner_data_index = static_cast<data_size_t>(threadIdx_y);
const int column_index = static_cast<int>(threadIdx.x) + partition_column_start;
if (threadIdx.x < static_cast<unsigned int>(num_columns_in_partition)) {
HIST_TYPE* shared_hist_ptr = shared_hist + (column_hist_offsets[column_index] << 1);
for (data_size_t i = 0; i < num_iteration_this; ++i) {
const data_size_t data_index = data_indices_ref_this_block[inner_data_index];
const score_t grad = cuda_gradients[data_index];
const score_t hess = cuda_hessians[data_index];
const uint32_t bin = static_cast<uint32_t>(data_ptr[static_cast<size_t>(data_index) * num_columns_in_partition + threadIdx.x]);
const uint32_t pos = bin << 1;
HIST_TYPE* pos_ptr = shared_hist_ptr + pos;
atomicAdd_block(pos_ptr, grad);
atomicAdd_block(pos_ptr + 1, hess);
inner_data_index += blockDim.y;
}
}
__syncthreads();
hist_t* feature_histogram_ptr = smaller_leaf_splits->hist_in_leaf + (partition_hist_start << 1);
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
atomicAdd_system(feature_histogram_ptr + i, shared_hist[i]);
}
}
template <typename BIN_TYPE, typename HIST_TYPE, typename DATA_PTR_TYPE>
__global__ void CUDAConstructHistogramSparseKernel_GlobalMemory(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const score_t* cuda_gradients,
const score_t* cuda_hessians,
const BIN_TYPE* data,
const DATA_PTR_TYPE* row_ptr,
const DATA_PTR_TYPE* partition_ptr,
const uint32_t* column_hist_offsets_full,
const data_size_t num_data,
HIST_TYPE* global_hist_buffer) {
const int dim_y = static_cast<int>(gridDim.y * blockDim.y);
const data_size_t num_data_in_smaller_leaf = smaller_leaf_splits->num_data_in_leaf;
const data_size_t num_data_per_thread = (num_data_in_smaller_leaf + dim_y - 1) / dim_y;
const data_size_t* data_indices_ref = smaller_leaf_splits->data_indices_in_leaf;
const unsigned int num_threads_per_block = blockDim.x * blockDim.y;
const DATA_PTR_TYPE* block_row_ptr = row_ptr + static_cast<size_t>(blockIdx.x) * (num_data + 1);
const BIN_TYPE* data_ptr = data + partition_ptr[blockIdx.x];
const uint32_t partition_hist_start = column_hist_offsets_full[blockIdx.x];
const uint32_t partition_hist_end = column_hist_offsets_full[blockIdx.x + 1];
const uint32_t num_items_in_partition = (partition_hist_end - partition_hist_start) << 1;
const unsigned int thread_idx = threadIdx.x + threadIdx.y * blockDim.x;
const int num_total_bin = column_hist_offsets_full[gridDim.x];
HIST_TYPE* shared_hist = global_hist_buffer + (blockIdx.y * num_total_bin + partition_hist_start) * 2;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
shared_hist[i] = 0.0f;
}
__syncthreads();
const unsigned int threadIdx_y = threadIdx.y;
const unsigned int blockIdx_y = blockIdx.y;
const data_size_t block_start = (blockIdx_y * blockDim.y) * num_data_per_thread;
const data_size_t* data_indices_ref_this_block = data_indices_ref + block_start;
data_size_t block_num_data = max(0, min(num_data_in_smaller_leaf - block_start, num_data_per_thread * static_cast<data_size_t>(blockDim.y)));
const data_size_t num_iteration_total = (block_num_data + blockDim.y - 1) / blockDim.y;
const data_size_t remainder = block_num_data % blockDim.y;
const data_size_t num_iteration_this = remainder == 0 ? num_iteration_total : num_iteration_total - static_cast<data_size_t>(threadIdx_y >= remainder);
data_size_t inner_data_index = static_cast<data_size_t>(threadIdx_y);
for (data_size_t i = 0; i < num_iteration_this; ++i) {
const data_size_t data_index = data_indices_ref_this_block[inner_data_index];
const DATA_PTR_TYPE row_start = block_row_ptr[data_index];
const DATA_PTR_TYPE row_end = block_row_ptr[data_index + 1];
const DATA_PTR_TYPE row_size = row_end - row_start;
if (threadIdx.x < row_size) {
const score_t grad = cuda_gradients[data_index];
const score_t hess = cuda_hessians[data_index];
const uint32_t bin = static_cast<uint32_t>(data_ptr[row_start + threadIdx.x]);
const uint32_t pos = bin << 1;
HIST_TYPE* pos_ptr = shared_hist + pos;
atomicAdd_block(pos_ptr, grad);
atomicAdd_block(pos_ptr + 1, hess);
}
inner_data_index += blockDim.y;
}
__syncthreads();
hist_t* feature_histogram_ptr = smaller_leaf_splits->hist_in_leaf + (partition_hist_start << 1);
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
atomicAdd_system(feature_histogram_ptr + i, shared_hist[i]);
}
}
template <typename BIN_TYPE, int SHARED_HIST_SIZE, bool USE_16BIT_HIST>
__global__ void CUDAConstructDiscretizedHistogramDenseKernel(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const int32_t* cuda_gradients_and_hessians,
const BIN_TYPE* data,
const uint32_t* column_hist_offsets,
const uint32_t* column_hist_offsets_full,
const int* feature_partition_column_index_offsets,
const data_size_t num_data) {
const int dim_y = static_cast<int>(gridDim.y * blockDim.y);
const data_size_t num_data_in_smaller_leaf = smaller_leaf_splits->num_data_in_leaf;
const data_size_t num_data_per_thread = (num_data_in_smaller_leaf + dim_y - 1) / dim_y;
const data_size_t* data_indices_ref = smaller_leaf_splits->data_indices_in_leaf;
__shared__ int16_t shared_hist[SHARED_HIST_SIZE];
int32_t* shared_hist_packed = reinterpret_cast<int32_t*>(shared_hist);
const unsigned int num_threads_per_block = blockDim.x * blockDim.y;
const int partition_column_start = feature_partition_column_index_offsets[blockIdx.x];
const int partition_column_end = feature_partition_column_index_offsets[blockIdx.x + 1];
const BIN_TYPE* data_ptr = data + partition_column_start * num_data;
const int num_columns_in_partition = partition_column_end - partition_column_start;
const uint32_t partition_hist_start = column_hist_offsets_full[blockIdx.x];
const uint32_t partition_hist_end = column_hist_offsets_full[blockIdx.x + 1];
const uint32_t num_items_in_partition = (partition_hist_end - partition_hist_start);
const unsigned int thread_idx = threadIdx.x + threadIdx.y * blockDim.x;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
shared_hist_packed[i] = 0;
}
__syncthreads();
const unsigned int threadIdx_y = threadIdx.y;
const unsigned int blockIdx_y = blockIdx.y;
const data_size_t block_start = (blockIdx_y * blockDim.y) * num_data_per_thread;
const data_size_t* data_indices_ref_this_block = data_indices_ref + block_start;
data_size_t block_num_data = max(0, min(num_data_in_smaller_leaf - block_start, num_data_per_thread * static_cast<data_size_t>(blockDim.y)));
const data_size_t num_iteration_total = (block_num_data + blockDim.y - 1) / blockDim.y;
const data_size_t remainder = block_num_data % blockDim.y;
const data_size_t num_iteration_this = remainder == 0 ? num_iteration_total : num_iteration_total - static_cast<data_size_t>(threadIdx_y >= remainder);
data_size_t inner_data_index = static_cast<data_size_t>(threadIdx_y);
const int column_index = static_cast<int>(threadIdx.x) + partition_column_start;
if (threadIdx.x < static_cast<unsigned int>(num_columns_in_partition)) {
int32_t* shared_hist_ptr = shared_hist_packed + (column_hist_offsets[column_index]);
for (data_size_t i = 0; i < num_iteration_this; ++i) {
const data_size_t data_index = data_indices_ref_this_block[inner_data_index];
const int32_t grad_and_hess = cuda_gradients_and_hessians[data_index];
const uint32_t bin = static_cast<uint32_t>(data_ptr[data_index * num_columns_in_partition + threadIdx.x]);
int32_t* pos_ptr = shared_hist_ptr + bin;
atomicAdd_block(pos_ptr, grad_and_hess);
inner_data_index += blockDim.y;
}
}
__syncthreads();
if (USE_16BIT_HIST) {
int32_t* feature_histogram_ptr = reinterpret_cast<int32_t*>(smaller_leaf_splits->hist_in_leaf) + partition_hist_start;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
const int32_t packed_grad_hess = shared_hist_packed[i];
atomicAdd_system(feature_histogram_ptr + i, packed_grad_hess);
}
} else {
atomic_add_long_t* feature_histogram_ptr = reinterpret_cast<atomic_add_long_t*>(smaller_leaf_splits->hist_in_leaf) + partition_hist_start;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
const int32_t packed_grad_hess = shared_hist_packed[i];
const int64_t packed_grad_hess_int64 = (static_cast<int64_t>(static_cast<int16_t>(packed_grad_hess >> 16)) << 32) | (static_cast<int64_t>(packed_grad_hess & 0x0000ffff));
atomicAdd_system(feature_histogram_ptr + i, (atomic_add_long_t)(packed_grad_hess_int64));
}
}
}
template <typename BIN_TYPE, typename DATA_PTR_TYPE, int SHARED_HIST_SIZE, bool USE_16BIT_HIST>
__global__ void CUDAConstructDiscretizedHistogramSparseKernel(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const int32_t* cuda_gradients_and_hessians,
const BIN_TYPE* data,
const DATA_PTR_TYPE* row_ptr,
const DATA_PTR_TYPE* partition_ptr,
const uint32_t* column_hist_offsets_full,
const data_size_t num_data) {
const int dim_y = static_cast<int>(gridDim.y * blockDim.y);
const data_size_t num_data_in_smaller_leaf = smaller_leaf_splits->num_data_in_leaf;
const data_size_t num_data_per_thread = (num_data_in_smaller_leaf + dim_y - 1) / dim_y;
const data_size_t* data_indices_ref = smaller_leaf_splits->data_indices_in_leaf;
__shared__ int16_t shared_hist[SHARED_HIST_SIZE];
int32_t* shared_hist_packed = reinterpret_cast<int32_t*>(shared_hist);
const unsigned int num_threads_per_block = blockDim.x * blockDim.y;
const DATA_PTR_TYPE* block_row_ptr = row_ptr + blockIdx.x * (num_data + 1);
const BIN_TYPE* data_ptr = data + partition_ptr[blockIdx.x];
const uint32_t partition_hist_start = column_hist_offsets_full[blockIdx.x];
const uint32_t partition_hist_end = column_hist_offsets_full[blockIdx.x + 1];
const uint32_t num_items_in_partition = (partition_hist_end - partition_hist_start);
const unsigned int thread_idx = threadIdx.x + threadIdx.y * blockDim.x;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
shared_hist_packed[i] = 0.0f;
}
__syncthreads();
const unsigned int threadIdx_y = threadIdx.y;
const unsigned int blockIdx_y = blockIdx.y;
const data_size_t block_start = (blockIdx_y * blockDim.y) * num_data_per_thread;
const data_size_t* data_indices_ref_this_block = data_indices_ref + block_start;
data_size_t block_num_data = max(0, min(num_data_in_smaller_leaf - block_start, num_data_per_thread * static_cast<data_size_t>(blockDim.y)));
const data_size_t num_iteration_total = (block_num_data + blockDim.y - 1) / blockDim.y;
const data_size_t remainder = block_num_data % blockDim.y;
const data_size_t num_iteration_this = remainder == 0 ? num_iteration_total : num_iteration_total - static_cast<data_size_t>(threadIdx_y >= remainder);
data_size_t inner_data_index = static_cast<data_size_t>(threadIdx_y);
for (data_size_t i = 0; i < num_iteration_this; ++i) {
const data_size_t data_index = data_indices_ref_this_block[inner_data_index];
const DATA_PTR_TYPE row_start = block_row_ptr[data_index];
const DATA_PTR_TYPE row_end = block_row_ptr[data_index + 1];
const DATA_PTR_TYPE row_size = row_end - row_start;
if (threadIdx.x < row_size) {
const int32_t grad_and_hess = cuda_gradients_and_hessians[data_index];
const uint32_t bin = static_cast<uint32_t>(data_ptr[row_start + threadIdx.x]);
int32_t* pos_ptr = shared_hist_packed + bin;
atomicAdd_block(pos_ptr, grad_and_hess);
}
inner_data_index += blockDim.y;
}
__syncthreads();
if (USE_16BIT_HIST) {
int32_t* feature_histogram_ptr = reinterpret_cast<int32_t*>(smaller_leaf_splits->hist_in_leaf) + partition_hist_start;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
const int32_t packed_grad_hess = shared_hist_packed[i];
atomicAdd_system(feature_histogram_ptr + i, packed_grad_hess);
}
} else {
atomic_add_long_t* feature_histogram_ptr = reinterpret_cast<atomic_add_long_t*>(smaller_leaf_splits->hist_in_leaf) + partition_hist_start;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
const int32_t packed_grad_hess = shared_hist_packed[i];
const int64_t packed_grad_hess_int64 = (static_cast<int64_t>(static_cast<int16_t>(packed_grad_hess >> 16)) << 32) | (static_cast<int64_t>(packed_grad_hess & 0x0000ffff));
atomicAdd_system(feature_histogram_ptr + i, (atomic_add_long_t)(packed_grad_hess_int64));
}
}
}
template <typename BIN_TYPE, int SHARED_HIST_SIZE, bool USE_16BIT_HIST>
__global__ void CUDAConstructDiscretizedHistogramDenseKernel_GlobalMemory(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const int32_t* cuda_gradients_and_hessians,
const BIN_TYPE* data,
const uint32_t* column_hist_offsets,
const uint32_t* column_hist_offsets_full,
const int* feature_partition_column_index_offsets,
const data_size_t num_data,
int32_t* global_hist_buffer) {
const int dim_y = static_cast<int>(gridDim.y * blockDim.y);
const data_size_t num_data_in_smaller_leaf = smaller_leaf_splits->num_data_in_leaf;
const data_size_t num_data_per_thread = (num_data_in_smaller_leaf + dim_y - 1) / dim_y;
const data_size_t* data_indices_ref = smaller_leaf_splits->data_indices_in_leaf;
const unsigned int num_threads_per_block = blockDim.x * blockDim.y;
const int partition_column_start = feature_partition_column_index_offsets[blockIdx.x];
const int partition_column_end = feature_partition_column_index_offsets[blockIdx.x + 1];
const BIN_TYPE* data_ptr = data + partition_column_start * num_data;
const int num_columns_in_partition = partition_column_end - partition_column_start;
const uint32_t partition_hist_start = column_hist_offsets_full[blockIdx.x];
const uint32_t partition_hist_end = column_hist_offsets_full[blockIdx.x + 1];
const uint32_t num_items_in_partition = (partition_hist_end - partition_hist_start);
const int num_total_bin = column_hist_offsets_full[gridDim.x];
int32_t* shared_hist_packed = global_hist_buffer + (blockIdx.y * num_total_bin + partition_hist_start);
const unsigned int thread_idx = threadIdx.x + threadIdx.y * blockDim.x;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
shared_hist_packed[i] = 0;
}
__syncthreads();
const unsigned int threadIdx_y = threadIdx.y;
const unsigned int blockIdx_y = blockIdx.y;
const data_size_t block_start = (blockIdx_y * blockDim.y) * num_data_per_thread;
const data_size_t* data_indices_ref_this_block = data_indices_ref + block_start;
data_size_t block_num_data = max(0, min(num_data_in_smaller_leaf - block_start, num_data_per_thread * static_cast<data_size_t>(blockDim.y)));
const data_size_t num_iteration_total = (block_num_data + blockDim.y - 1) / blockDim.y;
const data_size_t remainder = block_num_data % blockDim.y;
const data_size_t num_iteration_this = remainder == 0 ? num_iteration_total : num_iteration_total - static_cast<data_size_t>(threadIdx_y >= remainder);
data_size_t inner_data_index = static_cast<data_size_t>(threadIdx_y);
const int column_index = static_cast<int>(threadIdx.x) + partition_column_start;
if (threadIdx.x < static_cast<unsigned int>(num_columns_in_partition)) {
int32_t* shared_hist_ptr = shared_hist_packed + (column_hist_offsets[column_index]);
for (data_size_t i = 0; i < num_iteration_this; ++i) {
const data_size_t data_index = data_indices_ref_this_block[inner_data_index];
const int32_t grad_and_hess = cuda_gradients_and_hessians[data_index];
const uint32_t bin = static_cast<uint32_t>(data_ptr[data_index * num_columns_in_partition + threadIdx.x]);
int32_t* pos_ptr = shared_hist_ptr + bin;
atomicAdd_block(pos_ptr, grad_and_hess);
inner_data_index += blockDim.y;
}
}
__syncthreads();
if (USE_16BIT_HIST) {
int32_t* feature_histogram_ptr = reinterpret_cast<int32_t*>(smaller_leaf_splits->hist_in_leaf) + partition_hist_start;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
const int32_t packed_grad_hess = shared_hist_packed[i];
atomicAdd_system(feature_histogram_ptr + i, packed_grad_hess);
}
} else {
atomic_add_long_t* feature_histogram_ptr = reinterpret_cast<atomic_add_long_t*>(smaller_leaf_splits->hist_in_leaf) + partition_hist_start;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
const int32_t packed_grad_hess = shared_hist_packed[i];
const int64_t packed_grad_hess_int64 = (static_cast<int64_t>(static_cast<int16_t>(packed_grad_hess >> 16)) << 32) | (static_cast<int64_t>(packed_grad_hess & 0x0000ffff));
atomicAdd_system(feature_histogram_ptr + i, (atomic_add_long_t)(packed_grad_hess_int64));
}
}
}
template <typename BIN_TYPE, typename DATA_PTR_TYPE, int SHARED_HIST_SIZE, bool USE_16BIT_HIST>
__global__ void CUDAConstructDiscretizedHistogramSparseKernel_GlobalMemory(
const CUDALeafSplitsStruct* smaller_leaf_splits,
const int32_t* cuda_gradients_and_hessians,
const BIN_TYPE* data,
const DATA_PTR_TYPE* row_ptr,
const DATA_PTR_TYPE* partition_ptr,
const uint32_t* column_hist_offsets_full,
const data_size_t num_data,
int32_t* global_hist_buffer) {
const int dim_y = static_cast<int>(gridDim.y * blockDim.y);
const data_size_t num_data_in_smaller_leaf = smaller_leaf_splits->num_data_in_leaf;
const data_size_t num_data_per_thread = (num_data_in_smaller_leaf + dim_y - 1) / dim_y;
const data_size_t* data_indices_ref = smaller_leaf_splits->data_indices_in_leaf;
const int num_total_bin = column_hist_offsets_full[gridDim.x];
const unsigned int num_threads_per_block = blockDim.x * blockDim.y;
const DATA_PTR_TYPE* block_row_ptr = row_ptr + blockIdx.x * (num_data + 1);
const BIN_TYPE* data_ptr = data + partition_ptr[blockIdx.x];
const uint32_t partition_hist_start = column_hist_offsets_full[blockIdx.x];
const uint32_t partition_hist_end = column_hist_offsets_full[blockIdx.x + 1];
const uint32_t num_items_in_partition = (partition_hist_end - partition_hist_start);
const unsigned int thread_idx = threadIdx.x + threadIdx.y * blockDim.x;
int32_t* shared_hist_packed = global_hist_buffer + (blockIdx.y * num_total_bin + partition_hist_start);
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
shared_hist_packed[i] = 0.0f;
}
__syncthreads();
const unsigned int threadIdx_y = threadIdx.y;
const unsigned int blockIdx_y = blockIdx.y;
const data_size_t block_start = (blockIdx_y * blockDim.y) * num_data_per_thread;
const data_size_t* data_indices_ref_this_block = data_indices_ref + block_start;
data_size_t block_num_data = max(0, min(num_data_in_smaller_leaf - block_start, num_data_per_thread * static_cast<data_size_t>(blockDim.y)));
const data_size_t num_iteration_total = (block_num_data + blockDim.y - 1) / blockDim.y;
const data_size_t remainder = block_num_data % blockDim.y;
const data_size_t num_iteration_this = remainder == 0 ? num_iteration_total : num_iteration_total - static_cast<data_size_t>(threadIdx_y >= remainder);
data_size_t inner_data_index = static_cast<data_size_t>(threadIdx_y);
for (data_size_t i = 0; i < num_iteration_this; ++i) {
const data_size_t data_index = data_indices_ref_this_block[inner_data_index];
const DATA_PTR_TYPE row_start = block_row_ptr[data_index];
const DATA_PTR_TYPE row_end = block_row_ptr[data_index + 1];
const DATA_PTR_TYPE row_size = row_end - row_start;
if (threadIdx.x < row_size) {
const int32_t grad_and_hess = cuda_gradients_and_hessians[data_index];
const uint32_t bin = static_cast<uint32_t>(data_ptr[row_start + threadIdx.x]);
int32_t* pos_ptr = shared_hist_packed + bin;
atomicAdd_block(pos_ptr, grad_and_hess);
}
inner_data_index += blockDim.y;
}
__syncthreads();
if (USE_16BIT_HIST) {
int32_t* feature_histogram_ptr = reinterpret_cast<int32_t*>(smaller_leaf_splits->hist_in_leaf) + partition_hist_start;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
const int32_t packed_grad_hess = shared_hist_packed[i];
atomicAdd_system(feature_histogram_ptr + i, packed_grad_hess);
}
} else {
atomic_add_long_t* feature_histogram_ptr = reinterpret_cast<atomic_add_long_t*>(smaller_leaf_splits->hist_in_leaf) + partition_hist_start;
for (unsigned int i = thread_idx; i < num_items_in_partition; i += num_threads_per_block) {
const int32_t packed_grad_hess = shared_hist_packed[i];
const int64_t packed_grad_hess_int64 = (static_cast<int64_t>(static_cast<int16_t>(packed_grad_hess >> 16)) << 32) | (static_cast<int64_t>(packed_grad_hess & 0x0000ffff));
atomicAdd_system(feature_histogram_ptr + i, (atomic_add_long_t)(packed_grad_hess_int64));
}
}
}
void CUDAHistogramConstructor::LaunchConstructHistogramKernel(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins) {
if (cuda_row_data_->shared_hist_size() == DP_SHARED_HIST_SIZE && gpu_use_dp_) {
LaunchConstructHistogramKernelInner<double, DP_SHARED_HIST_SIZE>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else if (cuda_row_data_->shared_hist_size() == SP_SHARED_HIST_SIZE && !gpu_use_dp_) {
LaunchConstructHistogramKernelInner<float, SP_SHARED_HIST_SIZE>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else {
Log::Fatal("Unknown shared histogram size %d", cuda_row_data_->shared_hist_size());
}
}
template <typename HIST_TYPE, size_t SHARED_HIST_SIZE>
void CUDAHistogramConstructor::LaunchConstructHistogramKernelInner(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins) {
if (cuda_row_data_->bit_type() == 8) {
LaunchConstructHistogramKernelInner0<HIST_TYPE, SHARED_HIST_SIZE, uint8_t>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else if (cuda_row_data_->bit_type() == 16) {
LaunchConstructHistogramKernelInner0<HIST_TYPE, SHARED_HIST_SIZE, uint16_t>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else if (cuda_row_data_->bit_type() == 32) {
LaunchConstructHistogramKernelInner0<HIST_TYPE, SHARED_HIST_SIZE, uint32_t>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else {
Log::Fatal("Unknown bit_type = %d", cuda_row_data_->bit_type());
}
}
template <typename HIST_TYPE, size_t SHARED_HIST_SIZE, typename BIN_TYPE>
void CUDAHistogramConstructor::LaunchConstructHistogramKernelInner0(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins) {
if (cuda_row_data_->row_ptr_bit_type() == 16) {
LaunchConstructHistogramKernelInner1<HIST_TYPE, SHARED_HIST_SIZE, BIN_TYPE, uint16_t>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else if (cuda_row_data_->row_ptr_bit_type() == 32) {
LaunchConstructHistogramKernelInner1<HIST_TYPE, SHARED_HIST_SIZE, BIN_TYPE, uint32_t>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else if (cuda_row_data_->row_ptr_bit_type() == 64) {
LaunchConstructHistogramKernelInner1<HIST_TYPE, SHARED_HIST_SIZE, BIN_TYPE, uint64_t>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else {
if (!cuda_row_data_->is_sparse()) {
LaunchConstructHistogramKernelInner1<HIST_TYPE, SHARED_HIST_SIZE, BIN_TYPE, uint16_t>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else {
Log::Fatal("Unknown row_ptr_bit_type = %d", cuda_row_data_->row_ptr_bit_type());
}
}
}
template <typename HIST_TYPE, size_t SHARED_HIST_SIZE, typename BIN_TYPE, typename PTR_TYPE>
void CUDAHistogramConstructor::LaunchConstructHistogramKernelInner1(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins) {
if (cuda_row_data_->NumLargeBinPartition() == 0) {
LaunchConstructHistogramKernelInner2<HIST_TYPE, SHARED_HIST_SIZE, BIN_TYPE, PTR_TYPE, false>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
} else {
LaunchConstructHistogramKernelInner2<HIST_TYPE, SHARED_HIST_SIZE, BIN_TYPE, PTR_TYPE, true>(cuda_smaller_leaf_splits, num_data_in_smaller_leaf, num_bits_in_histogram_bins);
}
}
template <typename HIST_TYPE, size_t SHARED_HIST_SIZE, typename BIN_TYPE, typename PTR_TYPE, bool USE_GLOBAL_MEM_BUFFER>
void CUDAHistogramConstructor::LaunchConstructHistogramKernelInner2(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins) {
int grid_dim_x = 0;
int grid_dim_y = 0;
int block_dim_x = 0;
int block_dim_y = 0;
CalcConstructHistogramKernelDim(&grid_dim_x, &grid_dim_y, &block_dim_x, &block_dim_y, num_data_in_smaller_leaf);
dim3 grid_dim(grid_dim_x, grid_dim_y);
dim3 block_dim(block_dim_x, block_dim_y);
if (use_quantized_grad_) {
if (USE_GLOBAL_MEM_BUFFER) {
if (cuda_row_data_->is_sparse()) {
if (num_bits_in_histogram_bins <= 16) {
CUDAConstructDiscretizedHistogramSparseKernel_GlobalMemory<BIN_TYPE, PTR_TYPE, SHARED_HIST_SIZE, true><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
reinterpret_cast<const int32_t*>(cuda_gradients_),
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->GetRowPtr<PTR_TYPE>(),
cuda_row_data_->GetPartitionPtr<PTR_TYPE>(),
cuda_row_data_->cuda_partition_hist_offsets(),
num_data_,
reinterpret_cast<int32_t*>(cuda_hist_buffer_.RawData()));
} else {
CUDAConstructDiscretizedHistogramSparseKernel_GlobalMemory<BIN_TYPE, PTR_TYPE, SHARED_HIST_SIZE, false><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
reinterpret_cast<const int32_t*>(cuda_gradients_),
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->GetRowPtr<PTR_TYPE>(),
cuda_row_data_->GetPartitionPtr<PTR_TYPE>(),
cuda_row_data_->cuda_partition_hist_offsets(),
num_data_,
reinterpret_cast<int32_t*>(cuda_hist_buffer_.RawData()));
}
} else {
if (num_bits_in_histogram_bins <= 16) {
CUDAConstructDiscretizedHistogramDenseKernel_GlobalMemory<BIN_TYPE, SHARED_HIST_SIZE, true><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
reinterpret_cast<const int32_t*>(cuda_gradients_),
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->cuda_column_hist_offsets(),
cuda_row_data_->cuda_partition_hist_offsets(),
cuda_row_data_->cuda_feature_partition_column_index_offsets(),
num_data_,
reinterpret_cast<int32_t*>(cuda_hist_buffer_.RawData()));
} else {
CUDAConstructDiscretizedHistogramDenseKernel_GlobalMemory<BIN_TYPE, SHARED_HIST_SIZE, false><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
reinterpret_cast<const int32_t*>(cuda_gradients_),
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->cuda_column_hist_offsets(),
cuda_row_data_->cuda_partition_hist_offsets(),
cuda_row_data_->cuda_feature_partition_column_index_offsets(),
num_data_,
reinterpret_cast<int32_t*>(cuda_hist_buffer_.RawData()));
}
}
} else {
if (cuda_row_data_->is_sparse()) {
if (num_bits_in_histogram_bins <= 16) {
CUDAConstructDiscretizedHistogramSparseKernel<BIN_TYPE, PTR_TYPE, SHARED_HIST_SIZE, true><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
reinterpret_cast<const int32_t*>(cuda_gradients_),
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->GetRowPtr<PTR_TYPE>(),
cuda_row_data_->GetPartitionPtr<PTR_TYPE>(),
cuda_row_data_->cuda_partition_hist_offsets(),
num_data_);
} else {
CUDAConstructDiscretizedHistogramSparseKernel<BIN_TYPE, PTR_TYPE, SHARED_HIST_SIZE, false><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
reinterpret_cast<const int32_t*>(cuda_gradients_),
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->GetRowPtr<PTR_TYPE>(),
cuda_row_data_->GetPartitionPtr<PTR_TYPE>(),
cuda_row_data_->cuda_partition_hist_offsets(),
num_data_);
}
} else {
if (num_bits_in_histogram_bins <= 16) {
CUDAConstructDiscretizedHistogramDenseKernel<BIN_TYPE, SHARED_HIST_SIZE, true><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
reinterpret_cast<const int32_t*>(cuda_gradients_),
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->cuda_column_hist_offsets(),
cuda_row_data_->cuda_partition_hist_offsets(),
cuda_row_data_->cuda_feature_partition_column_index_offsets(),
num_data_);
} else {
CUDAConstructDiscretizedHistogramDenseKernel<BIN_TYPE, SHARED_HIST_SIZE, false><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
reinterpret_cast<const int32_t*>(cuda_gradients_),
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->cuda_column_hist_offsets(),
cuda_row_data_->cuda_partition_hist_offsets(),
cuda_row_data_->cuda_feature_partition_column_index_offsets(),
num_data_);
}
}
}
} else {
if (!USE_GLOBAL_MEM_BUFFER) {
if (cuda_row_data_->is_sparse()) {
CUDAConstructHistogramSparseKernel<BIN_TYPE, PTR_TYPE, HIST_TYPE, SHARED_HIST_SIZE><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
cuda_gradients_, cuda_hessians_,
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->GetRowPtr<PTR_TYPE>(),
cuda_row_data_->GetPartitionPtr<PTR_TYPE>(),
cuda_row_data_->cuda_partition_hist_offsets(),
num_data_);
} else {
CUDAConstructHistogramDenseKernel<BIN_TYPE, HIST_TYPE, SHARED_HIST_SIZE><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
cuda_gradients_, cuda_hessians_,
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->cuda_column_hist_offsets(),
cuda_row_data_->cuda_partition_hist_offsets(),
cuda_row_data_->cuda_feature_partition_column_index_offsets(),
num_data_);
}
} else {
if (cuda_row_data_->is_sparse()) {
CUDAConstructHistogramSparseKernel_GlobalMemory<BIN_TYPE, HIST_TYPE, PTR_TYPE><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
cuda_gradients_, cuda_hessians_,
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->GetRowPtr<PTR_TYPE>(),
cuda_row_data_->GetPartitionPtr<PTR_TYPE>(),
cuda_row_data_->cuda_partition_hist_offsets(),
num_data_,
reinterpret_cast<HIST_TYPE*>(cuda_hist_buffer_.RawData()));
} else {
CUDAConstructHistogramDenseKernel_GlobalMemory<BIN_TYPE, HIST_TYPE><<<grid_dim, block_dim, 0, cuda_stream_>>>(
cuda_smaller_leaf_splits,
cuda_gradients_, cuda_hessians_,
cuda_row_data_->GetBin<BIN_TYPE>(),
cuda_row_data_->cuda_column_hist_offsets(),
cuda_row_data_->cuda_partition_hist_offsets(),
cuda_row_data_->cuda_feature_partition_column_index_offsets(),
num_data_,
reinterpret_cast<HIST_TYPE*>(cuda_hist_buffer_.RawData()));
}
}
}
}
__global__ void SubtractHistogramKernel(
const int num_total_bin,
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const CUDALeafSplitsStruct* cuda_larger_leaf_splits) {
const unsigned int global_thread_index = threadIdx.x + blockIdx.x * blockDim.x;
const int cuda_larger_leaf_index = cuda_larger_leaf_splits->leaf_index;
if (cuda_larger_leaf_index >= 0) {
const hist_t* smaller_leaf_hist = cuda_smaller_leaf_splits->hist_in_leaf;
hist_t* larger_leaf_hist = cuda_larger_leaf_splits->hist_in_leaf;
if (global_thread_index < 2 * num_total_bin) {
larger_leaf_hist[global_thread_index] -= smaller_leaf_hist[global_thread_index];
}
}
}
__global__ void FixHistogramKernel(
const uint32_t* cuda_feature_num_bins,
const uint32_t* cuda_feature_hist_offsets,
const uint32_t* cuda_feature_most_freq_bins,
const int* cuda_need_fix_histogram_features,
const uint32_t* cuda_need_fix_histogram_features_num_bin_aligned,
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits) {
__shared__ hist_t shared_mem_buffer[WARPSIZE];
const unsigned int blockIdx_x = blockIdx.x;
const int feature_index = cuda_need_fix_histogram_features[blockIdx_x];
const uint32_t num_bin_aligned = cuda_need_fix_histogram_features_num_bin_aligned[blockIdx_x];
const uint32_t feature_hist_offset = cuda_feature_hist_offsets[feature_index];
const uint32_t most_freq_bin = cuda_feature_most_freq_bins[feature_index];
const double leaf_sum_gradients = cuda_smaller_leaf_splits->sum_of_gradients;
const double leaf_sum_hessians = cuda_smaller_leaf_splits->sum_of_hessians;
hist_t* feature_hist = cuda_smaller_leaf_splits->hist_in_leaf + feature_hist_offset * 2;
const unsigned int threadIdx_x = threadIdx.x;
const uint32_t num_bin = cuda_feature_num_bins[feature_index];
const uint32_t hist_pos = threadIdx_x << 1;
const hist_t bin_gradient = (threadIdx_x < num_bin && threadIdx_x != most_freq_bin) ? feature_hist[hist_pos] : 0.0f;
const hist_t bin_hessian = (threadIdx_x < num_bin && threadIdx_x != most_freq_bin) ? feature_hist[hist_pos + 1] : 0.0f;
const hist_t sum_gradient = ShuffleReduceSum<hist_t>(bin_gradient, shared_mem_buffer, num_bin_aligned);
const hist_t sum_hessian = ShuffleReduceSum<hist_t>(bin_hessian, shared_mem_buffer, num_bin_aligned);
if (threadIdx_x == 0) {
feature_hist[most_freq_bin << 1] = leaf_sum_gradients - sum_gradient;
feature_hist[(most_freq_bin << 1) + 1] = leaf_sum_hessians - sum_hessian;
}
}
template <bool SMALLER_USE_16BIT_HIST, bool LARGER_USE_16BIT_HIST, bool PARENT_USE_16BIT_HIST>
__global__ void SubtractHistogramDiscretizedKernel(
const int num_total_bin,
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const CUDALeafSplitsStruct* cuda_larger_leaf_splits,
hist_t* num_bit_change_buffer) {
const unsigned int global_thread_index = threadIdx.x + blockIdx.x * blockDim.x;
const int cuda_larger_leaf_index_ref = cuda_larger_leaf_splits->leaf_index;
if (cuda_larger_leaf_index_ref >= 0) {
if (PARENT_USE_16BIT_HIST) {
const int32_t* smaller_leaf_hist = reinterpret_cast<const int32_t*>(cuda_smaller_leaf_splits->hist_in_leaf);
int32_t* larger_leaf_hist = reinterpret_cast<int32_t*>(cuda_larger_leaf_splits->hist_in_leaf);
if (global_thread_index < num_total_bin) {
larger_leaf_hist[global_thread_index] -= smaller_leaf_hist[global_thread_index];
}
} else if (LARGER_USE_16BIT_HIST) {
int32_t* buffer = reinterpret_cast<int32_t*>(num_bit_change_buffer);
const int32_t* smaller_leaf_hist = reinterpret_cast<const int32_t*>(cuda_smaller_leaf_splits->hist_in_leaf);
int64_t* larger_leaf_hist = reinterpret_cast<int64_t*>(cuda_larger_leaf_splits->hist_in_leaf);
if (global_thread_index < num_total_bin) {
const int64_t parent_hist_item = larger_leaf_hist[global_thread_index];
const int32_t smaller_hist_item = smaller_leaf_hist[global_thread_index];
const int64_t smaller_hist_item_int64 = (static_cast<int64_t>(static_cast<int16_t>(smaller_hist_item >> 16)) << 32) |
static_cast<int64_t>(smaller_hist_item & 0x0000ffff);
const int64_t larger_hist_item = parent_hist_item - smaller_hist_item_int64;
buffer[global_thread_index] = static_cast<int32_t>(static_cast<int16_t>(larger_hist_item >> 32) << 16) |
static_cast<int32_t>(larger_hist_item & 0x000000000000ffff);
}
} else if (SMALLER_USE_16BIT_HIST) {
const int32_t* smaller_leaf_hist = reinterpret_cast<const int32_t*>(cuda_smaller_leaf_splits->hist_in_leaf);
int64_t* larger_leaf_hist = reinterpret_cast<int64_t*>(cuda_larger_leaf_splits->hist_in_leaf);
if (global_thread_index < num_total_bin) {
const int64_t parent_hist_item = larger_leaf_hist[global_thread_index];
const int32_t smaller_hist_item = smaller_leaf_hist[global_thread_index];
const int64_t smaller_hist_item_int64 = (static_cast<int64_t>(static_cast<int16_t>(smaller_hist_item >> 16)) << 32) |
static_cast<int64_t>(smaller_hist_item & 0x0000ffff);
const int64_t larger_hist_item = parent_hist_item - smaller_hist_item_int64;
larger_leaf_hist[global_thread_index] = larger_hist_item;
}
} else {
const int64_t* smaller_leaf_hist = reinterpret_cast<const int64_t*>(cuda_smaller_leaf_splits->hist_in_leaf);
int64_t* larger_leaf_hist = reinterpret_cast<int64_t*>(cuda_larger_leaf_splits->hist_in_leaf);
if (global_thread_index < num_total_bin) {
larger_leaf_hist[global_thread_index] -= smaller_leaf_hist[global_thread_index];
}
}
}
}
__global__ void CopyChangedNumBitHistogram(
const int num_total_bin,
const CUDALeafSplitsStruct* cuda_larger_leaf_splits,
hist_t* num_bit_change_buffer) {
int32_t* hist_dst = reinterpret_cast<int32_t*>(cuda_larger_leaf_splits->hist_in_leaf);
const int32_t* hist_src = reinterpret_cast<const int32_t*>(num_bit_change_buffer);
const unsigned int global_thread_index = threadIdx.x + blockIdx.x * blockDim.x;
if (global_thread_index < static_cast<unsigned int>(num_total_bin)) {
hist_dst[global_thread_index] = hist_src[global_thread_index];
}
}
template <bool USE_16BIT_HIST>
__global__ void FixHistogramDiscretizedKernel(
const uint32_t* cuda_feature_num_bins,
const uint32_t* cuda_feature_hist_offsets,
const uint32_t* cuda_feature_most_freq_bins,
const int* cuda_need_fix_histogram_features,
const uint32_t* cuda_need_fix_histogram_features_num_bin_aligned,
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits) {
__shared__ int64_t shared_mem_buffer[WARPSIZE];
const unsigned int blockIdx_x = blockIdx.x;
const int feature_index = cuda_need_fix_histogram_features[blockIdx_x];
const uint32_t num_bin_aligned = cuda_need_fix_histogram_features_num_bin_aligned[blockIdx_x];
const uint32_t feature_hist_offset = cuda_feature_hist_offsets[feature_index];
const uint32_t most_freq_bin = cuda_feature_most_freq_bins[feature_index];
if (USE_16BIT_HIST) {
const int64_t leaf_sum_gradients_hessians_int64 = cuda_smaller_leaf_splits->sum_of_gradients_hessians;
const int32_t leaf_sum_gradients_hessians =
(static_cast<int32_t>(leaf_sum_gradients_hessians_int64 >> 32) << 16) | static_cast<int32_t>(leaf_sum_gradients_hessians_int64 & 0x000000000000ffff);
int32_t* feature_hist = reinterpret_cast<int32_t*>(cuda_smaller_leaf_splits->hist_in_leaf) + feature_hist_offset;
const unsigned int threadIdx_x = threadIdx.x;
const uint32_t num_bin = cuda_feature_num_bins[feature_index];
const int32_t bin_gradient_hessian = (threadIdx_x < num_bin && threadIdx_x != most_freq_bin) ? feature_hist[threadIdx_x] : 0;
const int32_t sum_gradient_hessian = ShuffleReduceSum<int32_t>(
bin_gradient_hessian,
reinterpret_cast<int32_t*>(shared_mem_buffer),
num_bin_aligned);
if (threadIdx_x == 0) {
feature_hist[most_freq_bin] = leaf_sum_gradients_hessians - sum_gradient_hessian;
}
} else {
const int64_t leaf_sum_gradients_hessians = cuda_smaller_leaf_splits->sum_of_gradients_hessians;
int64_t* feature_hist = reinterpret_cast<int64_t*>(cuda_smaller_leaf_splits->hist_in_leaf) + feature_hist_offset;
const unsigned int threadIdx_x = threadIdx.x;
const uint32_t num_bin = cuda_feature_num_bins[feature_index];
const int64_t bin_gradient_hessian = (threadIdx_x < num_bin && threadIdx_x != most_freq_bin) ? feature_hist[threadIdx_x] : 0;
const int64_t sum_gradient_hessian = ShuffleReduceSum<int64_t>(bin_gradient_hessian, shared_mem_buffer, num_bin_aligned);
if (threadIdx_x == 0) {
feature_hist[most_freq_bin] = leaf_sum_gradients_hessians - sum_gradient_hessian;
}
}
}
void CUDAHistogramConstructor::LaunchSubtractHistogramKernel(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const CUDALeafSplitsStruct* cuda_larger_leaf_splits,
const bool use_discretized_grad,
const uint8_t parent_num_bits_in_histogram_bins,
const uint8_t smaller_num_bits_in_histogram_bins,
const uint8_t larger_num_bits_in_histogram_bins) {
if (!use_discretized_grad) {
const int num_subtract_threads = 2 * num_total_bin_;
const int num_subtract_blocks = (num_subtract_threads + SUBTRACT_BLOCK_SIZE - 1) / SUBTRACT_BLOCK_SIZE;
global_timer.Start("CUDAHistogramConstructor::FixHistogramKernel");
if (need_fix_histogram_features_.size() > 0) {
FixHistogramKernel<<<need_fix_histogram_features_.size(), FIX_HISTOGRAM_BLOCK_SIZE, 0, cuda_stream_>>>(
cuda_feature_num_bins_.RawData(),
cuda_feature_hist_offsets_.RawData(),
cuda_feature_most_freq_bins_.RawData(),
cuda_need_fix_histogram_features_.RawData(),
cuda_need_fix_histogram_features_num_bin_aligned_.RawData(),
cuda_smaller_leaf_splits);
}
global_timer.Stop("CUDAHistogramConstructor::FixHistogramKernel");
global_timer.Start("CUDAHistogramConstructor::SubtractHistogramKernel");
SubtractHistogramKernel<<<num_subtract_blocks, SUBTRACT_BLOCK_SIZE, 0, cuda_stream_>>>(
num_total_bin_,
cuda_smaller_leaf_splits,
cuda_larger_leaf_splits);
global_timer.Stop("CUDAHistogramConstructor::SubtractHistogramKernel");
} else {
const int num_subtract_threads = num_total_bin_;
const int num_subtract_blocks = (num_subtract_threads + SUBTRACT_BLOCK_SIZE - 1) / SUBTRACT_BLOCK_SIZE;
global_timer.Start("CUDAHistogramConstructor::FixHistogramDiscretizedKernel");
if (need_fix_histogram_features_.size() > 0) {
if (smaller_num_bits_in_histogram_bins <= 16) {
FixHistogramDiscretizedKernel<true><<<need_fix_histogram_features_.size(), FIX_HISTOGRAM_BLOCK_SIZE, 0, cuda_stream_>>>(
cuda_feature_num_bins_.RawData(),
cuda_feature_hist_offsets_.RawData(),
cuda_feature_most_freq_bins_.RawData(),
cuda_need_fix_histogram_features_.RawData(),
cuda_need_fix_histogram_features_num_bin_aligned_.RawData(),
cuda_smaller_leaf_splits);
} else {
FixHistogramDiscretizedKernel<false><<<need_fix_histogram_features_.size(), FIX_HISTOGRAM_BLOCK_SIZE, 0, cuda_stream_>>>(
cuda_feature_num_bins_.RawData(),
cuda_feature_hist_offsets_.RawData(),
cuda_feature_most_freq_bins_.RawData(),
cuda_need_fix_histogram_features_.RawData(),
cuda_need_fix_histogram_features_num_bin_aligned_.RawData(),
cuda_smaller_leaf_splits);
}
}
global_timer.Stop("CUDAHistogramConstructor::FixHistogramDiscretizedKernel");
global_timer.Start("CUDAHistogramConstructor::SubtractHistogramDiscretizedKernel");
if (parent_num_bits_in_histogram_bins <= 16) {
CHECK_LE(smaller_num_bits_in_histogram_bins, 16);
CHECK_LE(larger_num_bits_in_histogram_bins, 16);
SubtractHistogramDiscretizedKernel<true, true, true><<<num_subtract_blocks, SUBTRACT_BLOCK_SIZE, 0, cuda_stream_>>>(
num_total_bin_,
cuda_smaller_leaf_splits,
cuda_larger_leaf_splits,
hist_buffer_for_num_bit_change_.RawData());
} else if (larger_num_bits_in_histogram_bins <= 16) {
CHECK_LE(smaller_num_bits_in_histogram_bins, 16);
SubtractHistogramDiscretizedKernel<true, true, false><<<num_subtract_blocks, SUBTRACT_BLOCK_SIZE, 0, cuda_stream_>>>(
num_total_bin_,
cuda_smaller_leaf_splits,
cuda_larger_leaf_splits,
hist_buffer_for_num_bit_change_.RawData());
CopyChangedNumBitHistogram<<<num_subtract_blocks, SUBTRACT_BLOCK_SIZE, 0, cuda_stream_>>>(
num_total_bin_,
cuda_larger_leaf_splits,
hist_buffer_for_num_bit_change_.RawData());
} else if (smaller_num_bits_in_histogram_bins <= 16) {
SubtractHistogramDiscretizedKernel<true, false, false><<<num_subtract_blocks, SUBTRACT_BLOCK_SIZE, 0, cuda_stream_>>>(
num_total_bin_,
cuda_smaller_leaf_splits,
cuda_larger_leaf_splits,
hist_buffer_for_num_bit_change_.RawData());
} else {
SubtractHistogramDiscretizedKernel<false, false, false><<<num_subtract_blocks, SUBTRACT_BLOCK_SIZE, 0, cuda_stream_>>>(
num_total_bin_,
cuda_smaller_leaf_splits,
cuda_larger_leaf_splits,
hist_buffer_for_num_bit_change_.RawData());
}
global_timer.Stop("CUDAHistogramConstructor::SubtractHistogramDiscretizedKernel");
}
}
} // namespace LightGBM
#endif // USE_CUDA
src/treelearner/cuda/cuda_histogram_constructor.hpp
+199
View File
@@ -0,0 +1,199 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifndef LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_HISTOGRAM_CONSTRUCTOR_HPP_
#define LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_HISTOGRAM_CONSTRUCTOR_HPP_
#ifdef USE_CUDA
#include <LightGBM/cuda/cuda_row_data.hpp>
#include <LightGBM/cuda/cuda_utils.hu>
#include <LightGBM/feature_group.h>
#include <LightGBM/tree.h>
#include <memory>
#include <vector>
#include "cuda_leaf_splits.hpp"
#define NUM_DATA_PER_THREAD (400)
#define NUM_THREADS_PER_BLOCK (504)
#define NUM_FEATURE_PER_THREAD_GROUP (28)
#define SUBTRACT_BLOCK_SIZE (1024)
#define FIX_HISTOGRAM_SHARED_MEM_SIZE (1024)
#define FIX_HISTOGRAM_BLOCK_SIZE (512)
#define USED_HISTOGRAM_BUFFER_NUM (8)
namespace LightGBM {
class CUDAHistogramConstructor {
public:
CUDAHistogramConstructor(
const Dataset* train_data,
const int num_leaves,
const int num_threads,
const std::vector<uint32_t>& feature_hist_offsets,
const int min_data_in_leaf,
const double min_sum_hessian_in_leaf,
const int gpu_device_id,
const bool gpu_use_dp,
const bool use_discretized_grad,
const int grad_discretized_bins);
~CUDAHistogramConstructor();
void Init(const Dataset* train_data, TrainingShareStates* share_state);
void ConstructHistogramForLeaf(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const CUDALeafSplitsStruct* cuda_larger_leaf_splits,
const data_size_t global_num_data_in_smaller_leaf,
const data_size_t global_num_data_in_larger_leaf,
const data_size_t num_data_in_smaller_leaf,
const data_size_t num_data_in_larger_leaf,
const double sum_hessians_in_smaller_leaf,
const double sum_hessians_in_larger_leaf,
const uint8_t num_bits_in_histogram_bins);
void SubtractHistogramForLeaf(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const CUDALeafSplitsStruct* cuda_larger_leaf_splits,
const bool use_discretized_grad,
const uint8_t parent_num_bits_in_histogram_bins,
const uint8_t smaller_num_bits_in_histogram_bins,
const uint8_t larger_num_bits_in_histogram_bins);
void ResetTrainingData(const Dataset* train_data, TrainingShareStates* share_states);
void ResetConfig(const Config* config);
void BeforeTrain(const score_t* gradients, const score_t* hessians);
const hist_t* cuda_hist() const { return cuda_hist_.RawData(); }
hist_t* cuda_hist_pointer() { return cuda_hist_.RawData(); }
private:
void InitFeatureMetaInfo(const Dataset* train_data, const std::vector<uint32_t>& feature_hist_offsets);
void CalcConstructHistogramKernelDim(
int* grid_dim_x,
int* grid_dim_y,
int* block_dim_x,
int* block_dim_y,
const data_size_t num_data_in_smaller_leaf);
template <typename HIST_TYPE, size_t SHARED_HIST_SIZE>
void LaunchConstructHistogramKernelInner(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins);
template <typename HIST_TYPE, size_t SHARED_HIST_SIZE, typename BIN_TYPE>
void LaunchConstructHistogramKernelInner0(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins);
template <typename HIST_TYPE, size_t SHARED_HIST_SIZE, typename BIN_TYPE, typename PTR_TYPE>
void LaunchConstructHistogramKernelInner1(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins);
template <typename HIST_TYPE, size_t SHARED_HIST_SIZE, typename BIN_TYPE, typename PTR_TYPE, bool USE_GLOBAL_MEM_BUFFER>
void LaunchConstructHistogramKernelInner2(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins);
void LaunchConstructHistogramKernel(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const data_size_t num_data_in_smaller_leaf,
const uint8_t num_bits_in_histogram_bins);
void LaunchSubtractHistogramKernel(
const CUDALeafSplitsStruct* cuda_smaller_leaf_splits,
const CUDALeafSplitsStruct* cuda_larger_leaf_splits,
const bool use_discretized_grad,
const uint8_t parent_num_bits_in_histogram_bins,
const uint8_t smaller_num_bits_in_histogram_bins,
const uint8_t larger_num_bits_in_histogram_bins);
// Host memory
/*! \brief size of training data */
data_size_t num_data_;
/*! \brief number of features in training data */
int num_features_;
/*! \brief maximum number of leaves */
int num_leaves_;
/*! \brief number of threads */
int num_threads_;
/*! \brief total number of bins in histogram */
int num_total_bin_;
/*! \brief number of bins per feature */
std::vector<uint32_t> feature_num_bins_;
/*! \brief offsets in histogram of all features */
std::vector<uint32_t> feature_hist_offsets_;
/*! \brief most frequent bins in each feature */
std::vector<uint32_t> feature_most_freq_bins_;
/*! \brief minimum number of data allowed per leaf */
int min_data_in_leaf_;
/*! \brief minimum sum value of hessians allowed per leaf */
double min_sum_hessian_in_leaf_;
/*! \brief cuda stream for histogram construction */
cudaStream_t cuda_stream_;
/*! \brief indices of feature whose histograms need to be fixed */
std::vector<int> need_fix_histogram_features_;
/*! \brief aligned number of bins of the features whose histograms need to be fixed */
std::vector<uint32_t> need_fix_histogram_features_num_bin_aligend_;
/*! \brief minimum number of blocks allowed in the y dimension */
const int min_grid_dim_y_ = 160;
// CUDA memory, held by this object
/*! \brief CUDA row wise data */
std::unique_ptr<CUDARowData> cuda_row_data_;
/*! \brief number of bins per feature */
CUDAVector<uint32_t> cuda_feature_num_bins_;
/*! \brief offsets in histogram of all features */
CUDAVector<uint32_t> cuda_feature_hist_offsets_;
/*! \brief most frequent bins in each feature */
CUDAVector<uint32_t> cuda_feature_most_freq_bins_;
/*! \brief CUDA histograms */
CUDAVector<hist_t> cuda_hist_;
/*! \brief CUDA histograms buffer for each block */
CUDAVector<float> cuda_hist_buffer_;
/*! \brief indices of feature whose histograms need to be fixed */
CUDAVector<int> cuda_need_fix_histogram_features_;
/*! \brief aligned number of bins of the features whose histograms need to be fixed */
CUDAVector<uint32_t> cuda_need_fix_histogram_features_num_bin_aligned_;
/*! \brief histogram buffer used in histogram subtraction with different number of bits for histogram bins */
CUDAVector<hist_t> hist_buffer_for_num_bit_change_;
// CUDA memory, held by other object
/*! \brief gradients on CUDA */
const score_t* cuda_gradients_;
/*! \brief hessians on CUDA */
const score_t* cuda_hessians_;
/*! \brief GPU device index */
const int gpu_device_id_;
/*! \brief use double precision histogram per block */
const bool gpu_use_dp_;
/*! \brief whether to use quantized gradients */
const bool use_quantized_grad_;
/*! \brief the number of bins to quantized gradients */
const int num_grad_quant_bins_;
};
} // namespace LightGBM
#endif // USE_CUDA
#endif // LIGHTGBM_SRC_TREELEARNER_CUDA_CUDA_HISTOGRAM_CONSTRUCTOR_HPP_
src/treelearner/cuda/cuda_leaf_splits.cpp
+79
View File
@@ -0,0 +1,79 @@
/*!
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for
* license information.
*/
#ifdef USE_CUDA
#include "cuda_leaf_splits.hpp"
namespace LightGBM {
CUDALeafSplits::CUDALeafSplits(const data_size_t num_data):
num_data_(num_data) {}
CUDALeafSplits::~CUDALeafSplits() {}
void CUDALeafSplits::Init(const bool use_quantized_grad) {
num_blocks_init_from_gradients_ = (num_data_ + NUM_THREADS_PER_BLOCK_LEAF_SPLITS - 1) / NUM_THREADS_PER_BLOCK_LEAF_SPLITS;
// allocate more memory for sum reduction in CUDA
// only the first element records the final sum
cuda_sum_of_gradients_buffer_.Resize(static_cast<size_t>(num_blocks_init_from_gradients_));
cuda_sum_of_hessians_buffer_.Resize(static_cast<size_t>(num_blocks_init_from_gradients_));
if (use_quantized_grad) {
cuda_sum_of_gradients_hessians_buffer_.Resize(static_cast<size_t>(num_blocks_init_from_gradients_));
}
cuda_struct_.Resize(1);
}
void CUDALeafSplits::InitValues() {
LaunchInitValuesEmptyKernel();
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDALeafSplits::InitValues(
const double lambda_l1, const double lambda_l2,
const score_t* cuda_gradients, const score_t* cuda_hessians,
const data_size_t* cuda_bagging_data_indices, const data_size_t* cuda_data_indices_in_leaf,
const data_size_t num_used_indices, hist_t* cuda_hist_in_leaf,
double* root_sum_gradients, double* root_sum_hessians) {
cuda_gradients_ = cuda_gradients;
cuda_hessians_ = cuda_hessians;
cuda_sum_of_gradients_buffer_.SetValue(0);
cuda_sum_of_hessians_buffer_.SetValue(0);
LaunchInitValuesKernel(lambda_l1, lambda_l2, cuda_bagging_data_indices, cuda_data_indices_in_leaf, num_used_indices, cuda_hist_in_leaf);
CopyFromCUDADeviceToHost<double>(root_sum_gradients, cuda_sum_of_gradients_buffer_.RawData(), 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<double>(root_sum_hessians, cuda_sum_of_hessians_buffer_.RawData(), 1, __FILE__, __LINE__);
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDALeafSplits::InitValues(
const double lambda_l1, const double lambda_l2,
const int16_t* cuda_gradients_and_hessians,
const data_size_t* cuda_bagging_data_indices,
const data_size_t* cuda_data_indices_in_leaf, const data_size_t num_used_indices,
hist_t* cuda_hist_in_leaf, double* root_sum_gradients, double* root_sum_hessians,
const score_t* grad_scale, const score_t* hess_scale) {
cuda_gradients_ = reinterpret_cast<const score_t*>(cuda_gradients_and_hessians);
cuda_hessians_ = nullptr;
LaunchInitValuesKernel(lambda_l1, lambda_l2, cuda_bagging_data_indices, cuda_data_indices_in_leaf, num_used_indices, cuda_hist_in_leaf, grad_scale, hess_scale);
CopyFromCUDADeviceToHost<double>(root_sum_gradients, cuda_sum_of_gradients_buffer_.RawData(), 1, __FILE__, __LINE__);
CopyFromCUDADeviceToHost<double>(root_sum_hessians, cuda_sum_of_hessians_buffer_.RawData(), 1, __FILE__, __LINE__);
SynchronizeCUDADevice(__FILE__, __LINE__);
}
void CUDALeafSplits::Resize(const data_size_t num_data) {
num_data_ = num_data;
num_blocks_init_from_gradients_ = (num_data + NUM_THREADS_PER_BLOCK_LEAF_SPLITS - 1) / NUM_THREADS_PER_BLOCK_LEAF_SPLITS;
cuda_sum_of_gradients_buffer_.Resize(static_cast<size_t>(num_blocks_init_from_gradients_));
cuda_sum_of_hessians_buffer_.Resize(static_cast<size_t>(num_blocks_init_from_gradients_));
cuda_sum_of_gradients_hessians_buffer_.Resize(static_cast<size_t>(num_blocks_init_from_gradients_));
}
} // namespace LightGBM
#endif // USE_CUDA

Some files were not shown because too many files have changed in this diff Show More