chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,5 @@
|
||||
data=../data/categorical.data
|
||||
|
||||
input_model=LightGBM_model.txt
|
||||
|
||||
task=predict
|
||||
@@ -0,0 +1,7 @@
|
||||
# coding: utf-8
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
preds = [np.loadtxt(str(name)) for name in Path(__file__).absolute().parent.glob("*.pred")]
|
||||
np.testing.assert_allclose(preds[0], preds[1])
|
||||
@@ -0,0 +1,54 @@
|
||||
/*!
|
||||
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <LightGBM/meta.h>
|
||||
#include <LightGBM/utils/array_args.h>
|
||||
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
using LightGBM::data_size_t;
|
||||
using LightGBM::score_t;
|
||||
using LightGBM::ArrayArgs;
|
||||
|
||||
|
||||
TEST(Partition, JustWorks) {
|
||||
std::vector<score_t> gradients({0.5f, 5.0f, 1.0f, 2.0f, 2.0f});
|
||||
data_size_t middle_begin, middle_end;
|
||||
|
||||
ArrayArgs<score_t>::Partition(&gradients, 0, static_cast<int>(gradients.size()), &middle_begin, &middle_end);
|
||||
|
||||
EXPECT_EQ(gradients[middle_begin + 1], gradients[middle_end - 1]);
|
||||
EXPECT_GT(gradients[0], gradients[middle_begin + 1]);
|
||||
EXPECT_GT(gradients[middle_begin + 1], gradients.back());
|
||||
}
|
||||
|
||||
TEST(Partition, PartitionOneElement) {
|
||||
std::vector<score_t> gradients({0.5f});
|
||||
data_size_t middle_begin, middle_end;
|
||||
ArrayArgs<score_t>::Partition(&gradients, 0, static_cast<int>(gradients.size()), &middle_begin, &middle_end);
|
||||
EXPECT_EQ(gradients[middle_begin + 1], gradients[middle_end - 1]);
|
||||
}
|
||||
|
||||
TEST(Partition, Empty) {
|
||||
std::vector<score_t> gradients;
|
||||
data_size_t middle_begin, middle_end;
|
||||
ArrayArgs<score_t>::Partition(&gradients, 0, static_cast<int>(gradients.size()), &middle_begin, &middle_end);
|
||||
|
||||
EXPECT_EQ(middle_begin, -1);
|
||||
EXPECT_EQ(middle_end, 0);
|
||||
}
|
||||
|
||||
TEST(Partition, AllEqual) {
|
||||
std::vector<score_t> gradients({0.5f, 0.5f, 0.5f});
|
||||
data_size_t middle_begin, middle_end;
|
||||
ArrayArgs<score_t>::Partition(&gradients, 0, static_cast<int>(gradients.size()), &middle_begin, &middle_end);
|
||||
|
||||
EXPECT_EQ(gradients[middle_begin + 1], gradients[middle_end - 1]);
|
||||
EXPECT_EQ(middle_begin, -1);
|
||||
EXPECT_EQ(middle_end, 3);
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
/*!
|
||||
* Copyright (c) 2023-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2023-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*
|
||||
* Author: Oliver Borchert
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <nanoarrow/nanoarrow.hpp>
|
||||
|
||||
#include "../../src/arrow/array.hpp"
|
||||
|
||||
using LightGBM::ArrowChunkedArray;
|
||||
|
||||
namespace {
|
||||
|
||||
// Build an ArrowArrayStream from a schema and a list of chunk arrays. Takes ownership of the
|
||||
// passed schema and chunks.
|
||||
nanoarrow::UniqueArrayStream MakeStream(nanoarrow::UniqueSchema schema,
|
||||
std::vector<nanoarrow::UniqueArray> chunks) {
|
||||
nanoarrow::UniqueArrayStream stream;
|
||||
nanoarrow::VectorArrayStream(schema.get(), std::move(chunks)).ToArrayStream(stream.get());
|
||||
return stream;
|
||||
}
|
||||
|
||||
nanoarrow::UniqueSchema MakePrimitiveSchema(ArrowType type) {
|
||||
nanoarrow::UniqueSchema schema;
|
||||
EXPECT_EQ(ArrowSchemaInitFromType(schema.get(), type), NANOARROW_OK);
|
||||
return schema;
|
||||
}
|
||||
|
||||
nanoarrow::UniqueSchema MakeStructSchema(const std::vector<ArrowType>& field_types) {
|
||||
nanoarrow::UniqueSchema schema;
|
||||
ArrowSchemaInit(schema.get());
|
||||
EXPECT_EQ(ArrowSchemaSetTypeStruct(schema.get(), field_types.size()), NANOARROW_OK);
|
||||
for (size_t i = 0; i < field_types.size(); ++i) {
|
||||
EXPECT_EQ(ArrowSchemaSetType(schema->children[i], field_types[i]), NANOARROW_OK);
|
||||
}
|
||||
return schema;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
nanoarrow::UniqueArray MakePrimitiveArray(ArrowType type, const std::vector<T>& values,
|
||||
const std::vector<int64_t>& null_indices = {},
|
||||
int64_t offset = 0) {
|
||||
nanoarrow::UniqueArray array;
|
||||
EXPECT_EQ(ArrowArrayInitFromType(array.get(), type), NANOARROW_OK);
|
||||
EXPECT_EQ(ArrowArrayStartAppending(array.get()), NANOARROW_OK);
|
||||
size_t null_idx_pos = 0;
|
||||
for (size_t i = 0; i < values.size(); ++i) {
|
||||
if (null_idx_pos < null_indices.size() &&
|
||||
null_indices[null_idx_pos] == static_cast<int64_t>(i)) {
|
||||
EXPECT_EQ(ArrowArrayAppendNull(array.get(), 1), NANOARROW_OK);
|
||||
++null_idx_pos;
|
||||
} else {
|
||||
if (type == NANOARROW_TYPE_BOOL) {
|
||||
EXPECT_EQ(ArrowArrayAppendInt(array.get(), values[i] ? 1 : 0), NANOARROW_OK);
|
||||
} else {
|
||||
EXPECT_EQ(ArrowArrayAppendDouble(array.get(), static_cast<double>(values[i])),
|
||||
NANOARROW_OK);
|
||||
}
|
||||
}
|
||||
}
|
||||
EXPECT_EQ(ArrowArrayFinishBuildingDefault(array.get(), nullptr), NANOARROW_OK);
|
||||
|
||||
// Apply slicing offset (tests the consumer's handling of `array->offset`).
|
||||
if (offset > 0) {
|
||||
array->offset += offset;
|
||||
array->length -= offset;
|
||||
}
|
||||
return array;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(ArrowChunkedArrayTest, GetLength) {
|
||||
// Single chunk
|
||||
{
|
||||
auto schema = MakePrimitiveSchema(NANOARROW_TYPE_FLOAT);
|
||||
std::vector<nanoarrow::UniqueArray> chunks;
|
||||
chunks.emplace_back(MakePrimitiveArray<float>(NANOARROW_TYPE_FLOAT, {1, 2}));
|
||||
ArrowChunkedArray chunked_array(MakeStream(std::move(schema), std::move(chunks)).get());
|
||||
ASSERT_EQ(chunked_array.get_length(), 2);
|
||||
}
|
||||
|
||||
// Multiple chunks
|
||||
{
|
||||
auto schema = MakePrimitiveSchema(NANOARROW_TYPE_FLOAT);
|
||||
std::vector<nanoarrow::UniqueArray> chunks;
|
||||
chunks.emplace_back(MakePrimitiveArray<float>(NANOARROW_TYPE_FLOAT, {1, 2}));
|
||||
chunks.emplace_back(MakePrimitiveArray<float>(NANOARROW_TYPE_FLOAT, {3, 4, 5, 6}));
|
||||
ArrowChunkedArray chunked_array(MakeStream(std::move(schema), std::move(chunks)).get());
|
||||
ASSERT_EQ(chunked_array.get_length(), 6);
|
||||
}
|
||||
|
||||
// Sliced chunk via offset
|
||||
{
|
||||
auto schema = MakePrimitiveSchema(NANOARROW_TYPE_BOOL);
|
||||
std::vector<nanoarrow::UniqueArray> chunks;
|
||||
chunks.emplace_back(
|
||||
MakePrimitiveArray<bool>(NANOARROW_TYPE_BOOL, {true, false, true, true}, {}, 1));
|
||||
ArrowChunkedArray chunked_array(MakeStream(std::move(schema), std::move(chunks)).get());
|
||||
ASSERT_EQ(chunked_array.get_length(), 3);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(ArrowChunkedArrayTest, GetFields) {
|
||||
auto schema = MakeStructSchema({NANOARROW_TYPE_FLOAT, NANOARROW_TYPE_FLOAT});
|
||||
|
||||
nanoarrow::UniqueArray array;
|
||||
ASSERT_EQ(ArrowArrayInitFromSchema(array.get(), schema.get(), nullptr), NANOARROW_OK);
|
||||
ASSERT_EQ(ArrowArrayStartAppending(array.get()), NANOARROW_OK);
|
||||
std::vector<float> dat1 = {1, 2, 3};
|
||||
std::vector<float> dat2 = {4, 5, 6};
|
||||
for (size_t i = 0; i < dat1.size(); ++i) {
|
||||
ASSERT_EQ(ArrowArrayAppendDouble(array->children[0], dat1[i]), NANOARROW_OK);
|
||||
ASSERT_EQ(ArrowArrayAppendDouble(array->children[1], dat2[i]), NANOARROW_OK);
|
||||
ASSERT_EQ(ArrowArrayFinishElement(array.get()), NANOARROW_OK);
|
||||
}
|
||||
ASSERT_EQ(ArrowArrayFinishBuildingDefault(array.get(), nullptr), NANOARROW_OK);
|
||||
|
||||
std::vector<nanoarrow::UniqueArray> chunks;
|
||||
chunks.emplace_back(std::move(array));
|
||||
ArrowChunkedArray chunked_array(MakeStream(std::move(schema), std::move(chunks)).get());
|
||||
|
||||
ASSERT_EQ(chunked_array.get_length(), 3);
|
||||
ASSERT_EQ(chunked_array.get_num_fields(), 2);
|
||||
|
||||
int32_t first0 = 0, first1 = 0;
|
||||
chunked_array.view().field(0).visit<int32_t>([&](auto v) { first0 = *v.begin(); });
|
||||
chunked_array.view().field(1).visit<int32_t>([&](auto v) { first1 = *v.begin(); });
|
||||
ASSERT_EQ(first0, 1);
|
||||
ASSERT_EQ(first1, 4);
|
||||
}
|
||||
|
||||
TEST(ArrowChunkedArrayTest, IteratorArithmetic) {
|
||||
auto schema = MakePrimitiveSchema(NANOARROW_TYPE_FLOAT);
|
||||
std::vector<nanoarrow::UniqueArray> chunks;
|
||||
chunks.emplace_back(MakePrimitiveArray<float>(NANOARROW_TYPE_FLOAT, {1, 2}));
|
||||
chunks.emplace_back(MakePrimitiveArray<float>(NANOARROW_TYPE_FLOAT, {3, 4, 5, 6}));
|
||||
chunks.emplace_back(MakePrimitiveArray<float>(NANOARROW_TYPE_FLOAT, {7}));
|
||||
ArrowChunkedArray chunked_array(MakeStream(std::move(schema), std::move(chunks)).get());
|
||||
|
||||
chunked_array.view().visit<int32_t>([](auto v) {
|
||||
auto it = v.begin();
|
||||
EXPECT_EQ(*it, 1);
|
||||
++it;
|
||||
EXPECT_EQ(*it, 2);
|
||||
++it;
|
||||
EXPECT_EQ(*it, 3);
|
||||
it += 2;
|
||||
EXPECT_EQ(*it, 5);
|
||||
it += 2;
|
||||
EXPECT_EQ(*it, 7);
|
||||
|
||||
auto begin = v.begin();
|
||||
EXPECT_EQ(begin[0], 1);
|
||||
EXPECT_EQ(begin[1], 2);
|
||||
EXPECT_EQ(begin[2], 3);
|
||||
EXPECT_EQ(begin[6], 7);
|
||||
|
||||
auto end = v.end();
|
||||
EXPECT_EQ(end - it, 1);
|
||||
EXPECT_EQ(end - v.begin(), 7);
|
||||
});
|
||||
}
|
||||
|
||||
TEST(ArrowChunkedArrayTest, BooleanIterator) {
|
||||
auto schema = MakePrimitiveSchema(NANOARROW_TYPE_BOOL);
|
||||
std::vector<nanoarrow::UniqueArray> chunks;
|
||||
chunks.emplace_back(MakePrimitiveArray<bool>(NANOARROW_TYPE_BOOL, {false, true, false}, {2}));
|
||||
chunks.emplace_back(MakePrimitiveArray<bool>(
|
||||
NANOARROW_TYPE_BOOL, {false, false, false, false, true, true, true, true, false, true}, {},
|
||||
1));
|
||||
ArrowChunkedArray chunked_array(MakeStream(std::move(schema), std::move(chunks)).get());
|
||||
|
||||
chunked_array.view().visit<float>([](auto v) {
|
||||
auto it = v.begin();
|
||||
// First chunk
|
||||
EXPECT_EQ(*it, 0);
|
||||
EXPECT_EQ(*(++it), 1);
|
||||
EXPECT_TRUE(std::isnan(*(++it)));
|
||||
|
||||
// Second chunk
|
||||
EXPECT_EQ(*(++it), 0);
|
||||
it += 3;
|
||||
EXPECT_EQ(*it, 1);
|
||||
it += 4;
|
||||
EXPECT_EQ(*it, 0);
|
||||
EXPECT_EQ(*(++it), 1);
|
||||
|
||||
EXPECT_EQ(++it, v.end());
|
||||
});
|
||||
}
|
||||
|
||||
TEST(ArrowChunkedArrayTest, OffsetAndValidity) {
|
||||
auto schema = MakePrimitiveSchema(NANOARROW_TYPE_FLOAT);
|
||||
std::vector<nanoarrow::UniqueArray> chunks;
|
||||
chunks.emplace_back(
|
||||
MakePrimitiveArray<float>(NANOARROW_TYPE_FLOAT, {0, 1, 2, 3, 4, 5, 6}, {2, 3}, 2));
|
||||
ArrowChunkedArray chunked_array(MakeStream(std::move(schema), std::move(chunks)).get());
|
||||
|
||||
chunked_array.view().visit<double>([](auto v) {
|
||||
auto it = v.begin();
|
||||
EXPECT_TRUE(std::isnan(*it));
|
||||
EXPECT_TRUE(std::isnan(*(++it)));
|
||||
EXPECT_EQ(it[2], 4);
|
||||
EXPECT_EQ(it[4], 6);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
/*!
|
||||
* Copyright (c) 2026-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2026-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*
|
||||
* Author: Oliver Borchert
|
||||
*/
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(push)
|
||||
#pragma warning(disable : 4996)
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
||||
#endif
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <LightGBM/c_api.h>
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include <nanoarrow/nanoarrow.hpp>
|
||||
|
||||
namespace {
|
||||
|
||||
nanoarrow::UniqueSchema MakePrimitiveSchema(ArrowType type) {
|
||||
nanoarrow::UniqueSchema schema;
|
||||
EXPECT_EQ(ArrowSchemaInitFromType(schema.get(), type), NANOARROW_OK);
|
||||
return schema;
|
||||
}
|
||||
|
||||
nanoarrow::UniqueSchema MakeFloatStructSchema(int n_fields) {
|
||||
nanoarrow::UniqueSchema schema;
|
||||
ArrowSchemaInit(schema.get());
|
||||
EXPECT_EQ(ArrowSchemaSetTypeStruct(schema.get(), n_fields), NANOARROW_OK);
|
||||
for (int i = 0; i < n_fields; ++i) {
|
||||
EXPECT_EQ(ArrowSchemaSetType(schema->children[i], NANOARROW_TYPE_FLOAT), NANOARROW_OK);
|
||||
}
|
||||
return schema;
|
||||
}
|
||||
|
||||
nanoarrow::UniqueArray MakeFloatArray(const std::vector<float>& values) {
|
||||
nanoarrow::UniqueArray array;
|
||||
EXPECT_EQ(ArrowArrayInitFromType(array.get(), NANOARROW_TYPE_FLOAT), NANOARROW_OK);
|
||||
EXPECT_EQ(ArrowArrayStartAppending(array.get()), NANOARROW_OK);
|
||||
for (auto v : values) {
|
||||
EXPECT_EQ(ArrowArrayAppendDouble(array.get(), v), NANOARROW_OK);
|
||||
}
|
||||
EXPECT_EQ(ArrowArrayFinishBuildingDefault(array.get(), nullptr), NANOARROW_OK);
|
||||
return array;
|
||||
}
|
||||
|
||||
nanoarrow::UniqueArray MakeFloatStructArray(const struct ArrowSchema* schema,
|
||||
const std::vector<std::vector<float>>& columns) {
|
||||
nanoarrow::UniqueArray array;
|
||||
EXPECT_EQ(ArrowArrayInitFromSchema(array.get(), schema, nullptr), NANOARROW_OK);
|
||||
EXPECT_EQ(ArrowArrayStartAppending(array.get()), NANOARROW_OK);
|
||||
const size_t n = columns[0].size();
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
for (size_t c = 0; c < columns.size(); ++c) {
|
||||
EXPECT_EQ(ArrowArrayAppendDouble(array->children[c], columns[c][i]), NANOARROW_OK);
|
||||
}
|
||||
EXPECT_EQ(ArrowArrayFinishElement(array.get()), NANOARROW_OK);
|
||||
}
|
||||
EXPECT_EQ(ArrowArrayFinishBuildingDefault(array.get(), nullptr), NANOARROW_OK);
|
||||
return array;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(ArrowDeprecatedTest, DatasetCreateFromArrow) {
|
||||
auto schema = MakeFloatStructSchema(2);
|
||||
std::vector<std::vector<float>> columns = {
|
||||
{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f},
|
||||
{6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f}};
|
||||
auto array = MakeFloatStructArray(schema.get(), columns);
|
||||
|
||||
// Move ownership of schema and array out of the unique wrappers; the
|
||||
// deprecated API takes ownership of both.
|
||||
ArrowSchema raw_schema;
|
||||
schema.move(&raw_schema);
|
||||
std::vector<ArrowArray> raw_chunks(1);
|
||||
array.move(&raw_chunks[0]);
|
||||
|
||||
DatasetHandle handle = nullptr;
|
||||
int result = LGBM_DatasetCreateFromArrow(
|
||||
static_cast<int64_t>(raw_chunks.size()), raw_chunks.data(), &raw_schema,
|
||||
"max_bin=15", nullptr, &handle);
|
||||
ASSERT_EQ(result, 0);
|
||||
ASSERT_NE(handle, nullptr);
|
||||
|
||||
int num_data = 0;
|
||||
int num_feature = 0;
|
||||
ASSERT_EQ(LGBM_DatasetGetNumData(handle, &num_data), 0);
|
||||
ASSERT_EQ(LGBM_DatasetGetNumFeature(handle, &num_feature), 0);
|
||||
EXPECT_EQ(num_data, 6);
|
||||
EXPECT_EQ(num_feature, 2);
|
||||
|
||||
ASSERT_EQ(LGBM_DatasetFree(handle), 0);
|
||||
}
|
||||
|
||||
TEST(ArrowDeprecatedTest, DatasetSetFieldFromArrow) {
|
||||
// Create a small dataset from a dense matrix.
|
||||
std::vector<double> data = {1.0, 2.0,
|
||||
3.0, 4.0,
|
||||
5.0, 6.0,
|
||||
7.0, 8.0};
|
||||
DatasetHandle handle = nullptr;
|
||||
ASSERT_EQ(LGBM_DatasetCreateFromMat(data.data(), C_API_DTYPE_FLOAT64, 4, 2, 1,
|
||||
"max_bin=15", nullptr, &handle),
|
||||
0);
|
||||
|
||||
// Set the label using the deprecated Arrow API.
|
||||
std::vector<float> label_values = {0.0f, 1.0f, 0.0f, 1.0f};
|
||||
auto label_schema = MakePrimitiveSchema(NANOARROW_TYPE_FLOAT);
|
||||
auto label_array = MakeFloatArray(label_values);
|
||||
|
||||
ArrowSchema raw_schema;
|
||||
label_schema.move(&raw_schema);
|
||||
std::vector<ArrowArray> raw_chunks(1);
|
||||
label_array.move(&raw_chunks[0]);
|
||||
|
||||
ASSERT_EQ(LGBM_DatasetSetFieldFromArrow(
|
||||
handle, "label", static_cast<int64_t>(raw_chunks.size()),
|
||||
raw_chunks.data(), &raw_schema),
|
||||
0);
|
||||
|
||||
int out_len = 0;
|
||||
const void* out_ptr = nullptr;
|
||||
int out_type = 0;
|
||||
ASSERT_EQ(LGBM_DatasetGetField(handle, "label", &out_len, &out_ptr, &out_type), 0);
|
||||
EXPECT_EQ(out_type, C_API_DTYPE_FLOAT32);
|
||||
ASSERT_EQ(out_len, static_cast<int>(label_values.size()));
|
||||
const float* read = static_cast<const float*>(out_ptr);
|
||||
for (size_t i = 0; i < label_values.size(); ++i) {
|
||||
EXPECT_FLOAT_EQ(read[i], label_values[i]);
|
||||
}
|
||||
|
||||
ASSERT_EQ(LGBM_DatasetFree(handle), 0);
|
||||
}
|
||||
|
||||
TEST(ArrowDeprecatedTest, BoosterPredictForArrow) {
|
||||
// Train a tiny booster.
|
||||
const int nrow = 8;
|
||||
const int ncol = 2;
|
||||
std::vector<double> data = {1.0, 1.0,
|
||||
2.0, 2.0,
|
||||
3.0, 3.0,
|
||||
4.0, 4.0,
|
||||
5.0, 5.0,
|
||||
6.0, 6.0,
|
||||
7.0, 7.0,
|
||||
8.0, 8.0};
|
||||
std::vector<float> labels = {0, 0, 0, 0, 1, 1, 1, 1};
|
||||
|
||||
DatasetHandle dataset = nullptr;
|
||||
ASSERT_EQ(LGBM_DatasetCreateFromMat(data.data(), C_API_DTYPE_FLOAT64, nrow, ncol, 1,
|
||||
"max_bin=15", nullptr, &dataset),
|
||||
0);
|
||||
ASSERT_EQ(LGBM_DatasetSetField(dataset, "label", labels.data(),
|
||||
static_cast<int>(labels.size()), C_API_DTYPE_FLOAT32),
|
||||
0);
|
||||
|
||||
BoosterHandle booster = nullptr;
|
||||
ASSERT_EQ(LGBM_BoosterCreate(dataset,
|
||||
"objective=binary metric=auc num_leaves=3 verbose=-1",
|
||||
&booster),
|
||||
0);
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
int finished = 0;
|
||||
ASSERT_EQ(LGBM_BoosterUpdateOneIter(booster, &finished), 0);
|
||||
}
|
||||
|
||||
// Predict using the deprecated Arrow API.
|
||||
auto schema = MakeFloatStructSchema(ncol);
|
||||
std::vector<std::vector<float>> columns = {
|
||||
{1.0f, 4.0f, 8.0f},
|
||||
{1.0f, 4.0f, 8.0f}};
|
||||
auto array = MakeFloatStructArray(schema.get(), columns);
|
||||
|
||||
ArrowSchema raw_schema;
|
||||
schema.move(&raw_schema);
|
||||
std::vector<ArrowArray> raw_chunks(1);
|
||||
array.move(&raw_chunks[0]);
|
||||
|
||||
const int n_predict_rows = static_cast<int>(columns[0].size());
|
||||
std::vector<double> arrow_out(n_predict_rows, 0.0);
|
||||
int64_t arrow_written = 0;
|
||||
ASSERT_EQ(LGBM_BoosterPredictForArrow(
|
||||
booster, static_cast<int64_t>(raw_chunks.size()), raw_chunks.data(),
|
||||
&raw_schema, C_API_PREDICT_NORMAL, 0, -1, "", &arrow_written,
|
||||
arrow_out.data()),
|
||||
0);
|
||||
ASSERT_EQ(arrow_written, n_predict_rows);
|
||||
|
||||
// Compare against LGBM_BoosterPredictForMat with equivalent data.
|
||||
std::vector<double> mat_data = {1.0, 1.0,
|
||||
4.0, 4.0,
|
||||
8.0, 8.0};
|
||||
std::vector<double> mat_out(n_predict_rows, 0.0);
|
||||
int64_t mat_written = 0;
|
||||
ASSERT_EQ(LGBM_BoosterPredictForMat(booster, mat_data.data(), C_API_DTYPE_FLOAT64,
|
||||
n_predict_rows, ncol, 1, C_API_PREDICT_NORMAL, 0,
|
||||
-1, "", &mat_written, mat_out.data()),
|
||||
0);
|
||||
ASSERT_EQ(mat_written, n_predict_rows);
|
||||
for (int i = 0; i < n_predict_rows; ++i) {
|
||||
EXPECT_DOUBLE_EQ(arrow_out[i], mat_out[i]);
|
||||
}
|
||||
|
||||
ASSERT_EQ(LGBM_BoosterFree(booster), 0);
|
||||
ASSERT_EQ(LGBM_DatasetFree(dataset), 0);
|
||||
}
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(pop)
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
@@ -0,0 +1,73 @@
|
||||
/*!
|
||||
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <LightGBM/utils/byte_buffer.h>
|
||||
|
||||
#include <memory>
|
||||
#include <random>
|
||||
|
||||
using LightGBM::ByteBuffer;
|
||||
|
||||
|
||||
TEST(ByteBuffer, JustWorks) {
|
||||
std::unique_ptr<ByteBuffer> buffer;
|
||||
buffer.reset(new ByteBuffer());
|
||||
|
||||
int cumulativeSize = 0;
|
||||
EXPECT_EQ(cumulativeSize, buffer->GetSize());
|
||||
|
||||
int8_t int8Val = 34;
|
||||
cumulativeSize += sizeof(int8_t);
|
||||
buffer->Write(&int8Val, sizeof(int8_t));
|
||||
EXPECT_EQ(cumulativeSize, buffer->GetSize());
|
||||
EXPECT_EQ(int8Val, buffer->GetAt(cumulativeSize - 1));
|
||||
|
||||
int16_t int16Val = 33;
|
||||
cumulativeSize += sizeof(int16_t);
|
||||
buffer->Write(&int16Val, sizeof(int16_t));
|
||||
EXPECT_EQ(cumulativeSize, buffer->GetSize());
|
||||
int16_t serializedInt16 = 0;
|
||||
char* int16Ptr = reinterpret_cast<char*>(&serializedInt16);
|
||||
for (unsigned int i = 0; i < sizeof(int16_t); i++) {
|
||||
int16Ptr[i] = buffer->GetAt(cumulativeSize - (sizeof(int16_t) - i));
|
||||
}
|
||||
EXPECT_EQ(int16Val, serializedInt16);
|
||||
|
||||
int64_t int64Val = 35;
|
||||
cumulativeSize += sizeof(int64_t);
|
||||
buffer->Write(&int64Val, sizeof(int64_t));
|
||||
EXPECT_EQ(cumulativeSize, buffer->GetSize());
|
||||
int64_t serializedInt64 = 0;
|
||||
char* int64Ptr = reinterpret_cast<char*>(&serializedInt64);
|
||||
for (unsigned int i = 0; i < sizeof(int64_t); i++) {
|
||||
int64Ptr[i] = buffer->GetAt(cumulativeSize - (sizeof(int64_t) - i));
|
||||
}
|
||||
EXPECT_EQ(int64Val, serializedInt64);
|
||||
|
||||
double doubleVal = 36.6;
|
||||
cumulativeSize += sizeof(double);
|
||||
buffer->Write(&doubleVal, sizeof(doubleVal));
|
||||
EXPECT_EQ(cumulativeSize, buffer->GetSize());
|
||||
double serializedDouble = 0;
|
||||
char* doublePtr = reinterpret_cast<char*>(&serializedDouble);
|
||||
for (unsigned int i = 0; i < sizeof(double); i++) {
|
||||
doublePtr[i] = buffer->GetAt(cumulativeSize - (sizeof(double) - i));
|
||||
}
|
||||
EXPECT_EQ(doubleVal, serializedDouble);
|
||||
|
||||
const int charSize = 3;
|
||||
char charArrayVal[charSize] = { 'a', 'b', 'c' };
|
||||
cumulativeSize += charSize;
|
||||
buffer->Write(charArrayVal, charSize);
|
||||
EXPECT_EQ(cumulativeSize, buffer->GetSize());
|
||||
for (int i = 0; i < charSize; i++) {
|
||||
EXPECT_EQ(charArrayVal[i], buffer->GetAt(cumulativeSize - (charSize - i)));
|
||||
}
|
||||
|
||||
// Test that Data() points to first value written
|
||||
EXPECT_EQ(int8Val, *buffer->Data());
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
/*!
|
||||
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*
|
||||
* Author: Alberto Ferreira
|
||||
*/
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "../include/LightGBM/utils/chunked_array.hpp"
|
||||
|
||||
using LightGBM::ChunkedArray;
|
||||
|
||||
/*!
|
||||
Helper util to compare two vectors.
|
||||
|
||||
Don't compare floating point vectors this way!
|
||||
*/
|
||||
template <typename T>
|
||||
testing::AssertionResult are_vectors_equal(const std::vector<T> &a, const std::vector<T> &b) {
|
||||
if (a.size() != b.size()) {
|
||||
return testing::AssertionFailure()
|
||||
<< "Vectors differ in size: "
|
||||
<< a.size() << " != " << b.size();
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < a.size(); ++i) {
|
||||
if (a[i] != b[i]) {
|
||||
return testing::AssertionFailure()
|
||||
<< "Vectors differ at least at position " << i << ": "
|
||||
<< a[i] << " != " << b[i];
|
||||
}
|
||||
}
|
||||
|
||||
return testing::AssertionSuccess();
|
||||
}
|
||||
|
||||
|
||||
class ChunkedArrayTest : public testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
}
|
||||
|
||||
void add_items_to_array(const std::vector<int> &vec, ChunkedArray<int> *ca) {
|
||||
for (auto v : vec) {
|
||||
ca->add(v);
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
Ensures that if coalesce_to() is called upon the ChunkedArray,
|
||||
it would yield the same contents as vec
|
||||
*/
|
||||
testing::AssertionResult coalesced_output_equals_vec(const ChunkedArray<int> &ca, const std::vector<int> &vec,
|
||||
const bool all_addresses = false) {
|
||||
std::vector<int> out(vec.size());
|
||||
ca.coalesce_to(out.data(), all_addresses);
|
||||
return are_vectors_equal(out, vec);
|
||||
}
|
||||
|
||||
// Constants
|
||||
const std::vector<int> REF_VEC = {1, 5, 2, 4, 9, 8, 7};
|
||||
const size_t CHUNK_SIZE = 3;
|
||||
const size_t OUT_OF_BOUNDS_OFFSET = 4;
|
||||
|
||||
ChunkedArray<int> ca_ = ChunkedArray<int>(CHUNK_SIZE); //<! Re-used for many tests.
|
||||
};
|
||||
|
||||
|
||||
/*! ChunkedArray cannot be built from chunks of size 0. */
|
||||
TEST_F(ChunkedArrayTest, constructorWithChunkSize0Throws) {
|
||||
ASSERT_THROW(ChunkedArray<int> chunked_array(0), std::runtime_error);
|
||||
}
|
||||
|
||||
/*! get_chunk_size() should return the size used in the constructor */
|
||||
TEST_F(ChunkedArrayTest, constructorWithChunkSize) {
|
||||
for (size_t chunk_size = 1; chunk_size < 10; ++chunk_size) {
|
||||
ChunkedArray<int> chunked_array(chunk_size);
|
||||
ASSERT_EQ(chunked_array.get_chunk_size(), chunk_size);
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
get_chunk_size() should return the size used in the constructor
|
||||
independently of array manipulations.
|
||||
*/
|
||||
TEST_F(ChunkedArrayTest, getChunkSizeIsConstant) {
|
||||
for (size_t i = 0; i < 3 * CHUNK_SIZE; ++i) {
|
||||
ASSERT_EQ(ca_.get_chunk_size(), CHUNK_SIZE);
|
||||
ca_.add(0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*!
|
||||
get_add_count() should return the number of add calls,
|
||||
independently of the number of chunks used.
|
||||
*/
|
||||
TEST_F(ChunkedArrayTest, getChunksCount) {
|
||||
ASSERT_EQ(ca_.get_chunks_count(), 1); // ChunkedArray always starts with 1 chunk.
|
||||
|
||||
for (size_t i = 0; i < 3 * CHUNK_SIZE; ++i) {
|
||||
ca_.add(0);
|
||||
int expected_chunks = static_cast<int>(i / CHUNK_SIZE) + 1;
|
||||
ASSERT_EQ(ca_.get_chunks_count(), expected_chunks) << "with " << i << " add() call(s) "
|
||||
<< "and CHUNK_SIZE==" << CHUNK_SIZE << ".";
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
get_add_count() should return the number of add calls,
|
||||
independently of the number of chunks used.
|
||||
*/
|
||||
TEST_F(ChunkedArrayTest, getAddCount) {
|
||||
for (size_t i = 0; i < 3 * CHUNK_SIZE; ++i) {
|
||||
ASSERT_EQ(ca_.get_add_count(), i);
|
||||
ca_.add(0);
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
Ensure coalesce_to() works and dumps all the inserted data correctly.
|
||||
|
||||
If the ChunkedArray is created from a sequence of add() calls, coalescing to
|
||||
an output array after multiple add operations should yield the same
|
||||
exact data at both input and output.
|
||||
*/
|
||||
TEST_F(ChunkedArrayTest, coalesceTo) {
|
||||
std::vector<int> out(REF_VEC.size());
|
||||
add_items_to_array(REF_VEC, &ca_);
|
||||
|
||||
ca_.coalesce_to(out.data());
|
||||
|
||||
ASSERT_TRUE(are_vectors_equal(REF_VEC, out));
|
||||
}
|
||||
|
||||
/*!
|
||||
After clear the ChunkedArray() should still be usable.
|
||||
*/
|
||||
TEST_F(ChunkedArrayTest, clear) {
|
||||
const std::vector<int> ref_vec2 = {1, 2, 5, -1};
|
||||
add_items_to_array(REF_VEC, &ca_);
|
||||
// Start with some content:
|
||||
ASSERT_TRUE(coalesced_output_equals_vec(ca_, REF_VEC));
|
||||
|
||||
// Clear & re-use:
|
||||
ca_.clear();
|
||||
add_items_to_array(ref_vec2, &ca_);
|
||||
|
||||
// Output should match new content:
|
||||
ASSERT_TRUE(coalesced_output_equals_vec(ca_, ref_vec2));
|
||||
}
|
||||
|
||||
/*!
|
||||
Ensure ChunkedArray is safe against double-frees.
|
||||
*/
|
||||
TEST_F(ChunkedArrayTest, doubleFreeSafe) {
|
||||
ca_.release(); // Cannot be used any longer from now on.
|
||||
ca_.release(); // Ensure we don't segfault.
|
||||
|
||||
SUCCEED();
|
||||
}
|
||||
|
||||
/*!
|
||||
Ensure size computations in the getters are correct.
|
||||
*/
|
||||
TEST_F(ChunkedArrayTest, totalArraySizeMatchesLastChunkAddCount) {
|
||||
add_items_to_array(REF_VEC, &ca_);
|
||||
|
||||
const size_t first_chunks_add_count = (ca_.get_chunks_count() - 1) * ca_.get_chunk_size();
|
||||
const size_t last_chunk_add_count = ca_.get_last_chunk_add_count();
|
||||
|
||||
EXPECT_EQ(first_chunks_add_count, static_cast<int>(REF_VEC.size() / CHUNK_SIZE) * CHUNK_SIZE);
|
||||
EXPECT_EQ(last_chunk_add_count, REF_VEC.size() % CHUNK_SIZE);
|
||||
EXPECT_EQ(first_chunks_add_count + last_chunk_add_count, ca_.get_add_count());
|
||||
}
|
||||
|
||||
/*!
|
||||
Assert all values are correct and at the expected addresses throughout the
|
||||
several chunks.
|
||||
|
||||
This uses getitem() to reach each individual address of any of the chunks.
|
||||
|
||||
A sentinel value of -1 is used to check for invalid addresses.
|
||||
This would occur if there was an improper data layout with the chunks.
|
||||
*/
|
||||
TEST_F(ChunkedArrayTest, dataLayoutTestThroughGetitem) {
|
||||
add_items_to_array(REF_VEC, &ca_);
|
||||
|
||||
for (size_t i = 0, chunk = 0, in_chunk_idx = 0; i < REF_VEC.size(); ++i) {
|
||||
int value = ca_.getitem(chunk, in_chunk_idx, -1); // -1 works as sentinel value (bad layout found)
|
||||
|
||||
EXPECT_EQ(value, REF_VEC[i]) << " for address (chunk,in_chunk_idx) = (" << chunk << "," << in_chunk_idx << ")";
|
||||
|
||||
if (++in_chunk_idx == ca_.get_chunk_size()) {
|
||||
in_chunk_idx = 0;
|
||||
++chunk;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
Perform an array of setitem & getitem at valid and invalid addresses.
|
||||
We use several random addresses and trials to avoid writing much code.
|
||||
|
||||
By testing a random number of addresses many more times than the size of the test space
|
||||
we are almost guaranteed to cover all possible search addresses.
|
||||
|
||||
We also gradually add more chunks to the ChunkedArray and re-run more trials
|
||||
to ensure the valid/invalid addresses are updated.
|
||||
|
||||
With each valid update we add to a "memory" vector the latest inserted values.
|
||||
This is used at the end to ensure all values were stored properly, including after
|
||||
value overrides.
|
||||
*/
|
||||
TEST_F(ChunkedArrayTest, testDataLayoutWithAdvancedInsertionAPI) {
|
||||
const size_t MAX_CHUNKS_SEARCH = 5;
|
||||
const size_t MAX_IN_CHUNK_SEARCH_IDX = 2 * CHUNK_SIZE;
|
||||
// Number of trials for each new ChunkedArray configuration. Pass 100 times over the search space:
|
||||
const size_t N_TRIALS = MAX_CHUNKS_SEARCH * MAX_IN_CHUNK_SEARCH_IDX * 100;
|
||||
const int INVALID = -1; // A negative value signaling the requested value lives in an invalid address.
|
||||
const int UNINITIALIZED = -99; // A negative value to signal this was never updated.
|
||||
std::vector<int> ref_values(MAX_CHUNKS_SEARCH * CHUNK_SIZE, UNINITIALIZED); // Memorize latest inserted values.
|
||||
|
||||
// Each outer loop iteration changes the test by adding +1 chunk. We start with 1 chunk only:
|
||||
for (size_t chunks = 1; chunks < MAX_CHUNKS_SEARCH; ++chunks) {
|
||||
EXPECT_EQ(ca_.get_chunks_count(), chunks);
|
||||
|
||||
// Sweep valid and invalid addresses with a ChunkedArray with `chunks` chunks:
|
||||
for (size_t trial = 0; trial < N_TRIALS; ++trial) {
|
||||
// Compute a new trial address & value & if it is a valid address:
|
||||
const size_t trial_chunk = std::rand() % MAX_CHUNKS_SEARCH;
|
||||
const size_t trial_in_chunk_idx = std::rand() % MAX_IN_CHUNK_SEARCH_IDX;
|
||||
const int trial_value = std::rand() % 99999;
|
||||
const bool valid_address = (trial_chunk < chunks) & (trial_in_chunk_idx < CHUNK_SIZE);
|
||||
|
||||
// Insert item. If at a valid address, 0 is returned, otherwise, -1 is returned:
|
||||
EXPECT_EQ(ca_.setitem(trial_chunk, trial_in_chunk_idx, trial_value),
|
||||
valid_address ? 0 : -1);
|
||||
// If at valid address, check that the stored value is correct & remember it for the future:
|
||||
if (valid_address) {
|
||||
// Check the just-stored value with getitem():
|
||||
EXPECT_EQ(ca_.getitem(trial_chunk, trial_in_chunk_idx, INVALID), trial_value);
|
||||
|
||||
// Also store the just-stored value for future tracking:
|
||||
ref_values[trial_chunk * CHUNK_SIZE + trial_in_chunk_idx] = trial_value;
|
||||
}
|
||||
}
|
||||
|
||||
ca_.new_chunk(); // Just finished a round of trials. Now add a new chunk. Valid addresses will be expanded.
|
||||
}
|
||||
|
||||
// Final check: ensure even with overrides, all valid insertions store the latest value at that address:
|
||||
std::vector<int> coalesced_out(MAX_CHUNKS_SEARCH * CHUNK_SIZE, UNINITIALIZED);
|
||||
ca_.coalesce_to(coalesced_out.data(), true); // Export all valid addresses.
|
||||
for (size_t i = 0; i < ref_values.size(); ++i) {
|
||||
if (ref_values[i] != UNINITIALIZED) {
|
||||
// Test in 2 ways that the values are correctly laid out in memory:
|
||||
EXPECT_EQ(ca_.getitem(i / CHUNK_SIZE, i % CHUNK_SIZE, INVALID), ref_values[i]);
|
||||
EXPECT_EQ(coalesced_out[i], ref_values[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+154
@@ -0,0 +1,154 @@
|
||||
/*!
|
||||
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*/
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <limits>
|
||||
|
||||
#include "../include/LightGBM/utils/common.h"
|
||||
|
||||
|
||||
// This is a basic test for floating number parsing.
|
||||
// Most of the test cases come from:
|
||||
// https://github.com/dmlc/xgboost/blob/master/tests/cpp/common/test_charconv.cc
|
||||
// https://github.com/Alexhuszagh/rust-lexical/blob/master/data/test-parse-unittests/strtod_tests.toml
|
||||
class AtofPreciseTest : public testing::Test {
|
||||
public:
|
||||
struct AtofTestCase {
|
||||
const char* data;
|
||||
double expected;
|
||||
};
|
||||
|
||||
static double TestAtofPrecise(
|
||||
const char* data, double expected, bool test_eq = true) {
|
||||
double got = 0;
|
||||
const char* end = LightGBM::Common::AtofPrecise(data, &got);
|
||||
EXPECT_TRUE(end != data) << "fail to parse: " << data;
|
||||
EXPECT_EQ(*end, '\0') << "not parsing to end: " << data;
|
||||
if (test_eq) {
|
||||
EXPECT_EQ(expected, got) << "parse string: " << data;
|
||||
}
|
||||
return got;
|
||||
}
|
||||
|
||||
static double Int64Bits2Double(uint64_t v) {
|
||||
union {
|
||||
uint64_t i;
|
||||
double d;
|
||||
} conv;
|
||||
conv.i = v;
|
||||
return conv.d;
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(AtofPreciseTest, Basic) {
|
||||
AtofTestCase test_cases[] = {
|
||||
{ "0", 0.0 },
|
||||
{ "0E0", 0.0 },
|
||||
{ "-0E0", 0.0 },
|
||||
{ "-0", -0.0 },
|
||||
{ "1", 1.0 },
|
||||
{ "1E0", 1.0 },
|
||||
{ "-1", -1.0 },
|
||||
{ "-1E0", -1.0 },
|
||||
{ "123456.0", 123456.0 },
|
||||
{ "432E1", 432E1 },
|
||||
{ "1.2345678", 1.2345678 },
|
||||
{ "2.4414062E-4", 2.4414062E-4 },
|
||||
{ "3.0540412E5", 3.0540412E5 },
|
||||
{ "3.355445E7", 3.355445E7 },
|
||||
{ "1.1754944E-38", 1.1754944E-38 },
|
||||
};
|
||||
|
||||
for (auto const& test : test_cases) {
|
||||
TestAtofPrecise(test.data, test.expected);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(AtofPreciseTest, CornerCases) {
|
||||
AtofTestCase test_cases[] = {
|
||||
{ "1e-400", 0.0 },
|
||||
{ "2.4703282292062326e-324", 0.0 },
|
||||
{ "4.9406564584124654e-324", Int64Bits2Double(0x0000000000000001LU) },
|
||||
{ "8.44291197326099e-309", Int64Bits2Double(0x0006123400000001LU) },
|
||||
// FLT_MAX
|
||||
{ "3.40282346638528859811704183484516925440e38",
|
||||
static_cast<double>(std::numeric_limits<float>::max()) },
|
||||
// FLT_MIN
|
||||
{ "1.1754943508222875079687365372222456778186655567720875215087517062784172594547271728515625e-38",
|
||||
static_cast<double>(std::numeric_limits<float>::min()) },
|
||||
// DBL_MAX (1 + (1 - 2^-52)) * 2^1023 = (2^53 - 1) * 2^971
|
||||
{ "17976931348623157081452742373170435679807056752584499659891747680315"
|
||||
"72607800285387605895586327668781715404589535143824642343213268894641"
|
||||
"82768467546703537516986049910576551282076245490090389328944075868508"
|
||||
"45513394230458323690322294816580855933212334827479782620414472316873"
|
||||
"8177180919299881250404026184124858368", std::numeric_limits<double>::max() },
|
||||
{ "1.7976931348623158e+308", std::numeric_limits<double>::max() },
|
||||
// 2^971 * (2^53 - 1 + 1/2) : the smallest number resolving to inf
|
||||
{"179769313486231580793728971405303415079934132710037826936173778980444"
|
||||
"968292764750946649017977587207096330286416692887910946555547851940402"
|
||||
"630657488671505820681908902000708383676273854845817711531764475730270"
|
||||
"069855571366959622842914819860834936475292719074168444365510704342711"
|
||||
"559699508093042880177904174497792", std::numeric_limits<double>::infinity() },
|
||||
// Near DBL_MIN
|
||||
{ "2.2250738585072009e-308", Int64Bits2Double(0x000fffffffffffffLU) },
|
||||
// DBL_MIN 2^-1022
|
||||
{ "2.2250738585072012e-308", std::numeric_limits<double>::min() },
|
||||
{ "2.2250738585072014e-308", std::numeric_limits<double>::min() },
|
||||
};
|
||||
|
||||
for (auto const& test : test_cases) {
|
||||
TestAtofPrecise(test.data, test.expected);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(AtofPreciseTest, ErrorInput) {
|
||||
double got = 0;
|
||||
ASSERT_THROW(LightGBM::Common::AtofPrecise("x1", &got), std::runtime_error);
|
||||
}
|
||||
|
||||
TEST_F(AtofPreciseTest, NaN) {
|
||||
AtofTestCase test_cases[] = {
|
||||
{ "nan", std::numeric_limits<double>::quiet_NaN() },
|
||||
{ "NaN", std::numeric_limits<double>::quiet_NaN() },
|
||||
{ "NAN", std::numeric_limits<double>::quiet_NaN() },
|
||||
// The behavior for parsing -nan depends on implementation.
|
||||
// Thus we skip binary check for negative nan.
|
||||
{ "-nan", -std::numeric_limits<double>::quiet_NaN() },
|
||||
{ "-NaN", -std::numeric_limits<double>::quiet_NaN() },
|
||||
{ "-NAN", -std::numeric_limits<double>::quiet_NaN() },
|
||||
};
|
||||
|
||||
for (auto const& test : test_cases) {
|
||||
double got = TestAtofPrecise(test.data, test.expected, false);
|
||||
|
||||
EXPECT_TRUE(std::isnan(got)) << "not parsed as NaN: " << test.data;
|
||||
if (got > 0) {
|
||||
// See comment in test_cases.
|
||||
EXPECT_EQ(memcmp(&got, &test.expected, sizeof(test.expected)), 0)
|
||||
<< "parsed NaN is not the same for every bit: " << test.data;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(AtofPreciseTest, Inf) {
|
||||
AtofTestCase test_cases[] = {
|
||||
{ "inf", std::numeric_limits<double>::infinity() },
|
||||
{ "Inf", std::numeric_limits<double>::infinity() },
|
||||
{ "INF", std::numeric_limits<double>::infinity() },
|
||||
{ "-inf", -std::numeric_limits<double>::infinity() },
|
||||
{ "-Inf", -std::numeric_limits<double>::infinity() },
|
||||
{ "-INF", -std::numeric_limits<double>::infinity() },
|
||||
};
|
||||
|
||||
for (auto const& test : test_cases) {
|
||||
double got = TestAtofPrecise(test.data, test.expected, false);
|
||||
|
||||
EXPECT_EQ(LightGBM::Common::Sign(test.expected), LightGBM::Common::Sign(got)) << "sign differs parsing: " << test.data;
|
||||
EXPECT_TRUE(std::isinf(got)) << "not parsed as infinite: " << test.data;
|
||||
EXPECT_EQ(memcmp(&got, &test.expected, sizeof(test.expected)), 0)
|
||||
<< "parsed infinite is not the same for every bit: " << test.data;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
/*!
|
||||
* Copyright (c) 2021-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2021-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*/
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
testing::FLAGS_gtest_death_test_style = "threadsafe";
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/*!
|
||||
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <testutils.h>
|
||||
#include <LightGBM/utils/byte_buffer.h>
|
||||
#include <LightGBM/utils/log.h>
|
||||
#include <LightGBM/c_api.h>
|
||||
#include <LightGBM/dataset.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
using LightGBM::ByteBuffer;
|
||||
using LightGBM::Dataset;
|
||||
using LightGBM::Log;
|
||||
using LightGBM::TestUtils;
|
||||
|
||||
TEST(Serialization, JustWorks) {
|
||||
// Load some test data
|
||||
DatasetHandle dataset_handle;
|
||||
const char* params = "max_bin=15";
|
||||
int result = TestUtils::LoadDatasetFromExamples("binary_classification/binary.test", params, &dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LoadDatasetFromExamples result code: " << result;
|
||||
|
||||
Dataset* dataset;
|
||||
bool succeeded = true;
|
||||
std::string exceptionText("");
|
||||
try {
|
||||
dataset = static_cast<Dataset*>(dataset_handle);
|
||||
|
||||
// Serialize the reference
|
||||
ByteBufferHandle buffer_handle;
|
||||
int32_t buffer_len;
|
||||
result = LGBM_DatasetSerializeReferenceToBinary(dataset_handle, &buffer_handle, &buffer_len);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetSerializeReferenceToBinary result code: " << result;
|
||||
|
||||
ByteBuffer* buffer = nullptr;
|
||||
Dataset* deserialized_dataset = nullptr;
|
||||
try {
|
||||
buffer = static_cast<ByteBuffer*>(buffer_handle);
|
||||
|
||||
// Deserialize the reference
|
||||
DatasetHandle deserialized_dataset_handle;
|
||||
result = LGBM_DatasetCreateFromSerializedReference(buffer->Data(),
|
||||
static_cast<int32_t>(buffer->GetSize()),
|
||||
dataset->num_data(),
|
||||
0, // num_classes
|
||||
params,
|
||||
&deserialized_dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetCreateFromSerializedReference result code: " << result;
|
||||
|
||||
// Confirm 1 successful API call
|
||||
deserialized_dataset = static_cast<Dataset*>(deserialized_dataset_handle);
|
||||
EXPECT_EQ(dataset->num_data(), deserialized_dataset->num_data());
|
||||
} catch (std::exception& ex) {
|
||||
succeeded = false;
|
||||
exceptionText = std::string(ex.what());
|
||||
}
|
||||
|
||||
// Free memory
|
||||
if (buffer) {
|
||||
result = LGBM_ByteBufferFree(buffer);
|
||||
EXPECT_EQ(0, result) << "LGBM_ByteBufferFree result code: " << result;
|
||||
}
|
||||
if (deserialized_dataset) {
|
||||
result = LGBM_DatasetFree(deserialized_dataset);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetFree result code: " << result;
|
||||
}
|
||||
} catch (std::exception& ex) {
|
||||
succeeded = false;
|
||||
exceptionText = std::string(ex.what());
|
||||
}
|
||||
|
||||
if (dataset) {
|
||||
result = LGBM_DatasetFree(dataset);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetFree result code: " << result;
|
||||
}
|
||||
|
||||
if (!succeeded) {
|
||||
FAIL() << "Test Serialization failed with exception: " << exceptionText;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
/*!
|
||||
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <testutils.h>
|
||||
#include <LightGBM/c_api.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
|
||||
using LightGBM::TestUtils;
|
||||
|
||||
void test_predict_type(int predict_type, int num_predicts) {
|
||||
// Load some test data
|
||||
int result;
|
||||
|
||||
DatasetHandle train_dataset;
|
||||
result = TestUtils::LoadDatasetFromExamples("binary_classification/binary.train", "max_bin=15", &train_dataset);
|
||||
EXPECT_EQ(0, result) << "LoadDatasetFromExamples train result code: " << result;
|
||||
|
||||
BoosterHandle booster_handle;
|
||||
result = LGBM_BoosterCreate(train_dataset, "app=binary metric=auc num_leaves=31 verbose=0", &booster_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_BoosterCreate result code: " << result;
|
||||
|
||||
for (int i = 0; i < 51; i++) {
|
||||
int produced_empty_tree;
|
||||
result = LGBM_BoosterUpdateOneIter(
|
||||
booster_handle,
|
||||
&produced_empty_tree);
|
||||
EXPECT_EQ(0, result) << "LGBM_BoosterUpdateOneIter result code: " << result;
|
||||
}
|
||||
|
||||
int n_features;
|
||||
result = LGBM_BoosterGetNumFeature(
|
||||
booster_handle,
|
||||
&n_features);
|
||||
EXPECT_EQ(0, result) << "LGBM_BoosterGetNumFeature result code: " << result;
|
||||
EXPECT_EQ(28, n_features) << "LGBM_BoosterGetNumFeature number of features: " << n_features;
|
||||
|
||||
// Run a single row prediction and compare with regular Mat prediction:
|
||||
int64_t output_size;
|
||||
result = LGBM_BoosterCalcNumPredict(
|
||||
booster_handle,
|
||||
1,
|
||||
predict_type, // predict_type
|
||||
0, // start_iteration
|
||||
-1, // num_iteration
|
||||
&output_size);
|
||||
EXPECT_EQ(0, result) << "LGBM_BoosterCalcNumPredict result code: " << result;
|
||||
EXPECT_EQ(num_predicts, output_size) << "LGBM_BoosterCalcNumPredict output size: " << output_size;
|
||||
|
||||
std::ifstream test_file("examples/binary_classification/binary.test");
|
||||
std::vector<double> test;
|
||||
double x;
|
||||
int test_set_size = 0;
|
||||
while (test_file >> x) {
|
||||
if (test_set_size % (n_features + 1) == 0) {
|
||||
// Drop the result from the dataset, we only care about checking that prediction results are equal
|
||||
// in both cases
|
||||
test_file >> x;
|
||||
test_set_size++;
|
||||
}
|
||||
test.push_back(x);
|
||||
test_set_size++;
|
||||
}
|
||||
EXPECT_EQ(test_set_size % (n_features + 1), 0) << "Test size mismatch with dataset size (%)";
|
||||
test_set_size /= (n_features + 1);
|
||||
EXPECT_EQ(test_set_size, 500) << "Improperly parsed test file (test_set_size)";
|
||||
EXPECT_EQ(test.size(), test_set_size * n_features) << "Improperly parsed test file (test len)";
|
||||
|
||||
std::vector<double> mat_output(output_size * test_set_size, -1);
|
||||
int64_t written;
|
||||
result = LGBM_BoosterPredictForMat(
|
||||
booster_handle,
|
||||
&test[0],
|
||||
C_API_DTYPE_FLOAT64,
|
||||
test_set_size, // nrow
|
||||
n_features, // ncol
|
||||
1, // is_row_major
|
||||
predict_type, // predict_type
|
||||
0, // start_iteration
|
||||
-1, // num_iteration
|
||||
"",
|
||||
&written,
|
||||
&mat_output[0]);
|
||||
EXPECT_EQ(0, result) << "LGBM_BoosterPredictForMat result code: " << result;
|
||||
|
||||
// Test LGBM_BoosterPredictForMat in multi-threaded mode
|
||||
const int kNThreads = 10;
|
||||
const int numIterations = 5;
|
||||
std::vector<std::thread> predict_for_mat_threads(kNThreads);
|
||||
for (int i = 0; i < kNThreads; i++) {
|
||||
predict_for_mat_threads[i] = std::thread(
|
||||
[
|
||||
i, test_set_size, output_size, n_features,
|
||||
test = &test[0], booster_handle, predict_type, numIterations
|
||||
]() {
|
||||
for (int j = 0; j < numIterations; j++) {
|
||||
int result;
|
||||
std::vector<double> mat_output(output_size * test_set_size, -1);
|
||||
int64_t written;
|
||||
result = LGBM_BoosterPredictForMat(
|
||||
booster_handle,
|
||||
&test[0],
|
||||
C_API_DTYPE_FLOAT64,
|
||||
test_set_size, // nrow
|
||||
n_features, // ncol
|
||||
1, // is_row_major
|
||||
predict_type, // predict_type
|
||||
0, // start_iteration
|
||||
-1, // num_iteration
|
||||
"",
|
||||
&written,
|
||||
&mat_output[0]);
|
||||
EXPECT_EQ(0, result) << "LGBM_BoosterPredictForMat result code: " << result;
|
||||
}
|
||||
});
|
||||
}
|
||||
for (std::thread& t : predict_for_mat_threads) {
|
||||
t.join();
|
||||
}
|
||||
|
||||
// Now let's run with the single row fast prediction API:
|
||||
FastConfigHandle fast_configs[kNThreads];
|
||||
for (int i = 0; i < kNThreads; i++) {
|
||||
result = LGBM_BoosterPredictForMatSingleRowFastInit(
|
||||
booster_handle,
|
||||
predict_type, // predict_type
|
||||
0, // start_iteration
|
||||
-1, // num_iteration
|
||||
C_API_DTYPE_FLOAT64,
|
||||
n_features,
|
||||
"",
|
||||
&fast_configs[i]);
|
||||
EXPECT_EQ(0, result) << "LGBM_BoosterPredictForMatSingleRowFastInit result code: " << result;
|
||||
}
|
||||
|
||||
std::vector<double> single_row_output(output_size * test_set_size, -1);
|
||||
std::vector<std::thread> single_row_threads(kNThreads);
|
||||
int batch_size = (test_set_size + kNThreads - 1) / kNThreads; // round up
|
||||
for (int i = 0; i < kNThreads; i++) {
|
||||
single_row_threads[i] = std::thread(
|
||||
[
|
||||
i, batch_size, test_set_size, output_size, n_features,
|
||||
test = &test[0], fast_configs = &fast_configs[0], single_row_output = &single_row_output[0]
|
||||
]() {
|
||||
int result;
|
||||
int64_t written;
|
||||
for (int j = i * batch_size; j < std::min((i + 1) * batch_size, test_set_size); j++) {
|
||||
result = LGBM_BoosterPredictForMatSingleRowFast(
|
||||
fast_configs[i],
|
||||
&test[j * n_features],
|
||||
&written,
|
||||
&single_row_output[j * output_size]);
|
||||
EXPECT_EQ(0, result) << "LGBM_BoosterPredictForMatSingleRowFast result code: " << result;
|
||||
EXPECT_EQ(written, output_size) << "LGBM_BoosterPredictForMatSingleRowFast unexpected written output size";
|
||||
}
|
||||
});
|
||||
}
|
||||
for (std::thread& t : single_row_threads) {
|
||||
t.join();
|
||||
}
|
||||
|
||||
EXPECT_EQ(single_row_output, mat_output) << "LGBM_BoosterPredictForMatSingleRowFast output mismatch with LGBM_BoosterPredictForMat";
|
||||
|
||||
// Free all:
|
||||
for (int i = 0; i < kNThreads; i++) {
|
||||
result = LGBM_FastConfigFree(fast_configs[i]);
|
||||
EXPECT_EQ(0, result) << "LGBM_FastConfigFree result code: " << result;
|
||||
}
|
||||
|
||||
result = LGBM_BoosterFree(booster_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_BoosterFree result code: " << result;
|
||||
|
||||
result = LGBM_DatasetFree(train_dataset);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetFree result code: " << result;
|
||||
}
|
||||
|
||||
TEST(SingleRow, Normal) {
|
||||
test_predict_type(C_API_PREDICT_NORMAL, 1);
|
||||
}
|
||||
|
||||
TEST(SingleRow, Contrib) {
|
||||
test_predict_type(C_API_PREDICT_CONTRIB, 29);
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
/*!
|
||||
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <testutils.h>
|
||||
#include <LightGBM/utils/log.h>
|
||||
#include <LightGBM/c_api.h>
|
||||
#include <LightGBM/dataset.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using LightGBM::Dataset;
|
||||
using LightGBM::Log;
|
||||
using LightGBM::TestUtils;
|
||||
|
||||
void test_stream_dense(
|
||||
int8_t creation_type,
|
||||
DatasetHandle ref_dataset_handle,
|
||||
int32_t nrows,
|
||||
int32_t ncols,
|
||||
int32_t nclasses,
|
||||
int batch_count,
|
||||
const std::vector<double>* features,
|
||||
const std::vector<float>* labels,
|
||||
const std::vector<float>* weights,
|
||||
const std::vector<double>* init_scores,
|
||||
const std::vector<int32_t>* groups) {
|
||||
Log::Info("Streaming %d rows dense data with a batch size of %d", nrows, batch_count);
|
||||
DatasetHandle dataset_handle = nullptr;
|
||||
Dataset* dataset = nullptr;
|
||||
|
||||
int has_weights = weights != nullptr;
|
||||
int has_init_scores = init_scores != nullptr;
|
||||
int has_queries = groups != nullptr;
|
||||
|
||||
bool succeeded = true;
|
||||
std::string exceptionText("");
|
||||
|
||||
try {
|
||||
int result = 0;
|
||||
switch (creation_type) {
|
||||
case 0: {
|
||||
Log::Info("Creating Dataset using LGBM_DatasetCreateFromSampledColumn, %d rows dense data with a batch size of %d", nrows, batch_count);
|
||||
|
||||
// construct sample data first (use all data for convenience and since size is small)
|
||||
std::vector<std::vector<double>> sample_values(ncols);
|
||||
std::vector<std::vector<int>> sample_idx(ncols);
|
||||
const double* current_val = features->data();
|
||||
for (int32_t idx = 0; idx < nrows; ++idx) {
|
||||
for (int32_t k = 0; k < ncols; ++k) {
|
||||
if (std::fabs(*current_val) > 1e-35f || std::isnan(*current_val)) {
|
||||
sample_values[k].emplace_back(*current_val);
|
||||
sample_idx[k].emplace_back(static_cast<int>(idx));
|
||||
}
|
||||
current_val++;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<int> sample_sizes;
|
||||
std::vector<double*> sample_values_ptrs;
|
||||
std::vector<int*> sample_idx_ptrs;
|
||||
for (int32_t i = 0; i < ncols; ++i) {
|
||||
sample_values_ptrs.push_back(sample_values[i].data());
|
||||
sample_idx_ptrs.push_back(sample_idx[i].data());
|
||||
sample_sizes.push_back(static_cast<int>(sample_values[i].size()));
|
||||
}
|
||||
|
||||
result = LGBM_DatasetCreateFromSampledColumn(
|
||||
sample_values_ptrs.data(),
|
||||
sample_idx_ptrs.data(),
|
||||
ncols,
|
||||
sample_sizes.data(),
|
||||
nrows,
|
||||
nrows,
|
||||
nrows,
|
||||
"max_bin=15",
|
||||
&dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetCreateFromSampledColumn result code: " << result;
|
||||
|
||||
result = LGBM_DatasetInitStreaming(dataset_handle, has_weights, has_init_scores, has_queries, nclasses, 1, -1);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetInitStreaming result code: " << result;
|
||||
break;
|
||||
}
|
||||
|
||||
case 1:
|
||||
Log::Info("Creating Dataset using LGBM_DatasetCreateByReference, %d rows dense data with a batch size of %d", nrows, batch_count);
|
||||
result = LGBM_DatasetCreateByReference(ref_dataset_handle, nrows, &dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetCreateByReference result code: " << result;
|
||||
break;
|
||||
}
|
||||
|
||||
dataset = static_cast<Dataset*>(dataset_handle);
|
||||
|
||||
Log::Info("Streaming dense dataset, %d rows dense data with a batch size of %d", nrows, batch_count);
|
||||
TestUtils::StreamDenseDataset(
|
||||
dataset_handle,
|
||||
nrows,
|
||||
ncols,
|
||||
nclasses,
|
||||
batch_count,
|
||||
features,
|
||||
labels,
|
||||
weights,
|
||||
init_scores,
|
||||
groups);
|
||||
|
||||
dataset->FinishLoad();
|
||||
|
||||
TestUtils::AssertMetadata(&dataset->metadata(),
|
||||
labels,
|
||||
weights,
|
||||
init_scores,
|
||||
groups);
|
||||
}
|
||||
catch (std::exception& ex) {
|
||||
succeeded = false;
|
||||
exceptionText = std::string(ex.what());
|
||||
}
|
||||
|
||||
if (dataset_handle) {
|
||||
int result = LGBM_DatasetFree(dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetFree result code: " << result;
|
||||
}
|
||||
|
||||
if (!succeeded) {
|
||||
FAIL() << "Test Dense Stream failed with exception: " << exceptionText;
|
||||
}
|
||||
}
|
||||
|
||||
void test_stream_sparse(
|
||||
int8_t creation_type,
|
||||
DatasetHandle ref_dataset_handle,
|
||||
int32_t nrows,
|
||||
int32_t ncols,
|
||||
int32_t nclasses,
|
||||
int batch_count,
|
||||
const std::vector<int32_t>* indptr,
|
||||
const std::vector<int32_t>* indices,
|
||||
const std::vector<double>* vals,
|
||||
const std::vector<float>* labels,
|
||||
const std::vector<float>* weights,
|
||||
const std::vector<double>* init_scores,
|
||||
const std::vector<int32_t>* groups) {
|
||||
Log::Info("Streaming %d rows sparse data with a batch size of %d", nrows, batch_count);
|
||||
DatasetHandle dataset_handle = nullptr;
|
||||
Dataset* dataset = nullptr;
|
||||
|
||||
int has_weights = weights != nullptr;
|
||||
int has_init_scores = init_scores != nullptr;
|
||||
int has_queries = groups != nullptr;
|
||||
|
||||
bool succeeded = true;
|
||||
std::string exceptionText("");
|
||||
|
||||
try {
|
||||
int result = 0;
|
||||
switch (creation_type) {
|
||||
case 0: {
|
||||
Log::Info("Creating Dataset using LGBM_DatasetCreateFromSampledColumn, %d rows sparse data with a batch size of %d", nrows, batch_count);
|
||||
|
||||
std::vector<std::vector<double>> sample_values(ncols);
|
||||
std::vector<std::vector<int>> sample_idx(ncols);
|
||||
for (size_t i = 0; i < indptr->size() - 1; ++i) {
|
||||
int start_index = indptr->at(i);
|
||||
int stop_index = indptr->at(i + 1);
|
||||
for (int32_t j = start_index; j < stop_index; ++j) {
|
||||
auto val = vals->at(j);
|
||||
auto idx = indices->at(j);
|
||||
if (std::fabs(val) > 1e-35f || std::isnan(val)) {
|
||||
sample_values[idx].emplace_back(val);
|
||||
sample_idx[idx].emplace_back(static_cast<int>(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<int> sample_sizes;
|
||||
std::vector<double*> sample_values_ptrs;
|
||||
std::vector<int*> sample_idx_ptrs;
|
||||
for (int32_t i = 0; i < ncols; ++i) {
|
||||
sample_values_ptrs.push_back(sample_values[i].data());
|
||||
sample_idx_ptrs.push_back(sample_idx[i].data());
|
||||
sample_sizes.push_back(static_cast<int>(sample_values[i].size()));
|
||||
}
|
||||
|
||||
result = LGBM_DatasetCreateFromSampledColumn(
|
||||
sample_values_ptrs.data(),
|
||||
sample_idx_ptrs.data(),
|
||||
ncols,
|
||||
sample_sizes.data(),
|
||||
nrows,
|
||||
nrows,
|
||||
nrows,
|
||||
"max_bin=15",
|
||||
&dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetCreateFromSampledColumn result code: " << result;
|
||||
|
||||
dataset = static_cast<Dataset*>(dataset_handle);
|
||||
dataset->InitStreaming(nrows, has_weights, has_init_scores, has_queries, nclasses, 2, -1);
|
||||
break;
|
||||
}
|
||||
|
||||
case 1:
|
||||
Log::Info("Creating Dataset using LGBM_DatasetCreateByReference, %d rows sparse data with a batch size of %d", nrows, batch_count);
|
||||
result = LGBM_DatasetCreateByReference(ref_dataset_handle, nrows, &dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetCreateByReference result code: " << result;
|
||||
break;
|
||||
}
|
||||
|
||||
dataset = static_cast<Dataset*>(dataset_handle);
|
||||
|
||||
Log::Info("Streaming sparse dataset, %d rows sparse data with a batch size of %d", nrows, batch_count);
|
||||
TestUtils::StreamSparseDataset(
|
||||
dataset_handle,
|
||||
nrows,
|
||||
nclasses,
|
||||
batch_count,
|
||||
indptr,
|
||||
indices,
|
||||
vals,
|
||||
labels,
|
||||
weights,
|
||||
init_scores,
|
||||
groups);
|
||||
|
||||
dataset->FinishLoad();
|
||||
|
||||
TestUtils::AssertMetadata(&dataset->metadata(),
|
||||
labels,
|
||||
weights,
|
||||
init_scores,
|
||||
groups);
|
||||
}
|
||||
catch (std::exception& ex) {
|
||||
succeeded = false;
|
||||
exceptionText = std::string(ex.what());
|
||||
}
|
||||
|
||||
if (dataset_handle) {
|
||||
int result = LGBM_DatasetFree(dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetFree result code: " << result;
|
||||
}
|
||||
|
||||
if (!succeeded) {
|
||||
FAIL() << "Test Sparse Stream failed with exception: " << exceptionText;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Stream, PushDenseRowsWithMetadata) {
|
||||
// Load some test data
|
||||
DatasetHandle ref_dataset_handle;
|
||||
const char* params = "max_bin=15";
|
||||
// Use the smaller ".test" data because we don't care about the actual data and it's smaller
|
||||
int result = TestUtils::LoadDatasetFromExamples("binary_classification/binary.test", params, &ref_dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LoadDatasetFromExamples result code: " << result;
|
||||
|
||||
Dataset* ref_dataset = static_cast<Dataset*>(ref_dataset_handle);
|
||||
auto noriginalrows = ref_dataset->num_data();
|
||||
Log::Info("Row count: %d", noriginalrows);
|
||||
Log::Info("Feature group count: %d", ref_dataset->num_features());
|
||||
|
||||
// Add some fake initial_scores and groups so we can test streaming them
|
||||
int nclasses = 2; // choose > 1 just to test multi-class handling
|
||||
std::vector<double> unused_init_scores;
|
||||
unused_init_scores.resize(noriginalrows * nclasses);
|
||||
std::vector<int32_t> unused_groups;
|
||||
unused_groups.assign(noriginalrows, 1);
|
||||
result = LGBM_DatasetSetField(ref_dataset_handle, "init_score", unused_init_scores.data(), noriginalrows * nclasses, 1);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetSetField init_score result code: " << result;
|
||||
result = LGBM_DatasetSetField(ref_dataset_handle, "group", unused_groups.data(), noriginalrows, 2);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetSetField group result code: " << result;
|
||||
|
||||
// Now use the reference dataset schema to make some testable Datasets with N rows each
|
||||
int32_t nrows = 1000;
|
||||
int32_t ncols = ref_dataset->num_features();
|
||||
std::vector<double> features;
|
||||
std::vector<float> labels;
|
||||
std::vector<float> weights;
|
||||
std::vector<double> init_scores;
|
||||
std::vector<int32_t> groups;
|
||||
|
||||
Log::Info("Creating random data");
|
||||
TestUtils::CreateRandomDenseData(nrows, ncols, nclasses, &features, &labels, &weights, &init_scores, &groups);
|
||||
|
||||
const std::vector<int32_t> batch_counts = { 1, nrows / 100, nrows / 10, nrows };
|
||||
const std::vector<int8_t> creation_types = { 0, 1 };
|
||||
|
||||
for (size_t i = 0; i < creation_types.size(); ++i) { // from sampled data or reference
|
||||
for (size_t j = 0; j < batch_counts.size(); ++j) {
|
||||
auto type = creation_types[i];
|
||||
auto batch_count = batch_counts[j];
|
||||
test_stream_dense(type, ref_dataset_handle, nrows, ncols, nclasses, batch_count, &features, &labels, &weights, &init_scores, &groups);
|
||||
}
|
||||
}
|
||||
|
||||
result = LGBM_DatasetFree(ref_dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetFree result code: " << result;
|
||||
}
|
||||
|
||||
TEST(Stream, PushSparseRowsWithMetadata) {
|
||||
// Load some test data
|
||||
DatasetHandle ref_dataset_handle;
|
||||
const char* params = "max_bin=15";
|
||||
// Use the smaller ".test" data because we don't care about the actual data and it's smaller
|
||||
int result = TestUtils::LoadDatasetFromExamples("binary_classification/binary.test", params, &ref_dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LoadDatasetFromExamples result code: " << result;
|
||||
|
||||
Dataset* ref_dataset = static_cast<Dataset*>(ref_dataset_handle);
|
||||
auto noriginalrows = ref_dataset->num_data();
|
||||
Log::Info("Row count: %d", noriginalrows);
|
||||
Log::Info("Feature group count: %d", ref_dataset->num_features());
|
||||
|
||||
// Add some fake initial_scores and groups so we can test streaming them
|
||||
int32_t nclasses = 2;
|
||||
std::vector<double> unused_init_scores;
|
||||
unused_init_scores.resize(noriginalrows * nclasses);
|
||||
std::vector<int32_t> unused_groups;
|
||||
unused_groups.assign(noriginalrows, 1);
|
||||
result = LGBM_DatasetSetField(ref_dataset_handle, "init_score", unused_init_scores.data(), noriginalrows * nclasses, 1);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetSetField init_score result code: " << result;
|
||||
result = LGBM_DatasetSetField(ref_dataset_handle, "group", unused_groups.data(), noriginalrows, 2);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetSetField group result code: " << result;
|
||||
|
||||
// Now use the reference dataset schema to make some testable Datasets with N rows each
|
||||
int32_t nrows = 1000;
|
||||
int32_t ncols = ref_dataset->num_features();
|
||||
std::vector<int32_t> indptr;
|
||||
std::vector<int32_t> indices;
|
||||
std::vector<double> vals;
|
||||
std::vector<float> labels;
|
||||
std::vector<float> weights;
|
||||
std::vector<double> init_scores;
|
||||
std::vector<int32_t> groups;
|
||||
|
||||
Log::Info("Creating random data");
|
||||
float sparse_percent = .1f;
|
||||
TestUtils::CreateRandomSparseData(nrows, ncols, nclasses, sparse_percent, &indptr, &indices, &vals, &labels, &weights, &init_scores, &groups);
|
||||
|
||||
const std::vector<int32_t> batch_counts = { 1, nrows / 100, nrows / 10, nrows };
|
||||
const std::vector<int8_t> creation_types = { 0, 1 };
|
||||
|
||||
for (size_t i = 0; i < creation_types.size(); ++i) { // from sampled data or reference
|
||||
for (size_t j = 0; j < batch_counts.size(); ++j) {
|
||||
auto type = creation_types[i];
|
||||
auto batch_count = batch_counts[j];
|
||||
test_stream_sparse(type, ref_dataset_handle, nrows, ncols, nclasses, batch_count, &indptr, &indices, &vals, &labels, &weights, &init_scores, &groups);
|
||||
}
|
||||
}
|
||||
|
||||
result = LGBM_DatasetFree(ref_dataset_handle);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetFree result code: " << result;
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
/*!
|
||||
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <testutils.h>
|
||||
#include <LightGBM/c_api.h>
|
||||
#include <LightGBM/utils/random.h>
|
||||
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
using LightGBM::Log;
|
||||
using LightGBM::Random;
|
||||
|
||||
namespace LightGBM {
|
||||
|
||||
/*!
|
||||
* Creates a Dataset from the internal repository examples.
|
||||
*/
|
||||
int TestUtils::LoadDatasetFromExamples(const char* filename, const char* config, DatasetHandle* out) {
|
||||
std::string fullPath("examples/");
|
||||
fullPath += filename;
|
||||
Log::Info("Debug sample data path: %s", fullPath.c_str());
|
||||
return LGBM_DatasetCreateFromFile(
|
||||
fullPath.c_str(),
|
||||
config,
|
||||
nullptr,
|
||||
out);
|
||||
}
|
||||
|
||||
/*!
|
||||
* Creates fake data in the passed vectors.
|
||||
*/
|
||||
void TestUtils::CreateRandomDenseData(
|
||||
int32_t nrows,
|
||||
int32_t ncols,
|
||||
int32_t nclasses,
|
||||
std::vector<double>* features,
|
||||
std::vector<float>* labels,
|
||||
std::vector<float>* weights,
|
||||
std::vector<double>* init_scores,
|
||||
std::vector<int32_t>* groups) {
|
||||
Random rand(42);
|
||||
features->reserve(nrows * ncols);
|
||||
|
||||
for (int32_t row = 0; row < nrows; row++) {
|
||||
for (int32_t col = 0; col < ncols; col++) {
|
||||
features->push_back(rand.NextFloat());
|
||||
}
|
||||
}
|
||||
|
||||
CreateRandomMetadata(nrows, nclasses, labels, weights, init_scores, groups);
|
||||
}
|
||||
|
||||
/*!
|
||||
* Creates fake data in the passed vectors.
|
||||
*/
|
||||
void TestUtils::CreateRandomSparseData(
|
||||
int32_t nrows,
|
||||
int32_t ncols,
|
||||
int32_t nclasses,
|
||||
float sparse_percent,
|
||||
std::vector<int32_t>* indptr,
|
||||
std::vector<int32_t>* indices,
|
||||
std::vector<double>* values,
|
||||
std::vector<float>* labels,
|
||||
std::vector<float>* weights,
|
||||
std::vector<double>* init_scores,
|
||||
std::vector<int32_t>* groups) {
|
||||
Random rand(42);
|
||||
indptr->reserve(static_cast<int32_t>(nrows + 1));
|
||||
indices->reserve(static_cast<int32_t>(sparse_percent * nrows * ncols));
|
||||
values->reserve(static_cast<int32_t>(sparse_percent * nrows * ncols));
|
||||
|
||||
indptr->push_back(0);
|
||||
for (int32_t row = 0; row < nrows; row++) {
|
||||
for (int32_t col = 0; col < ncols; col++) {
|
||||
float rnd = rand.NextFloat();
|
||||
if (rnd < sparse_percent) {
|
||||
indices->push_back(col);
|
||||
values->push_back(rand.NextFloat());
|
||||
}
|
||||
}
|
||||
indptr->push_back(static_cast<int32_t>(indices->size() - 1));
|
||||
}
|
||||
|
||||
CreateRandomMetadata(nrows, nclasses, labels, weights, init_scores, groups);
|
||||
}
|
||||
|
||||
/*!
|
||||
* Creates fake data in the passed vectors.
|
||||
*/
|
||||
void TestUtils::CreateRandomMetadata(int32_t nrows,
|
||||
int32_t nclasses,
|
||||
std::vector<float>* labels,
|
||||
std::vector<float>* weights,
|
||||
std::vector<double>* init_scores,
|
||||
std::vector<int32_t>* groups) {
|
||||
Random rand(42);
|
||||
labels->reserve(nrows);
|
||||
if (weights) {
|
||||
weights->reserve(nrows);
|
||||
}
|
||||
if (init_scores) {
|
||||
init_scores->reserve(nrows * nclasses);
|
||||
}
|
||||
if (groups) {
|
||||
groups->reserve(nrows);
|
||||
}
|
||||
|
||||
int32_t group = 0;
|
||||
|
||||
for (int32_t row = 0; row < nrows; row++) {
|
||||
labels->push_back(rand.NextFloat());
|
||||
if (weights) {
|
||||
weights->push_back(rand.NextFloat());
|
||||
}
|
||||
if (init_scores) {
|
||||
for (int32_t i = 0; i < nclasses; i++) {
|
||||
init_scores->push_back(rand.NextFloat());
|
||||
}
|
||||
}
|
||||
if (groups) {
|
||||
if (rand.NextFloat() > 0.95) {
|
||||
group++;
|
||||
}
|
||||
groups->push_back(group);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TestUtils::StreamDenseDataset(DatasetHandle dataset_handle,
|
||||
int32_t nrows,
|
||||
int32_t ncols,
|
||||
int32_t nclasses,
|
||||
int32_t batch_count,
|
||||
const std::vector<double>* features,
|
||||
const std::vector<float>* labels,
|
||||
const std::vector<float>* weights,
|
||||
const std::vector<double>* init_scores,
|
||||
const std::vector<int32_t>* groups) {
|
||||
int result = LGBM_DatasetSetWaitForManualFinish(dataset_handle, 1);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetSetWaitForManualFinish result code: " << result;
|
||||
|
||||
Log::Info(" Begin StreamDenseDataset");
|
||||
if ((nrows % batch_count) != 0) {
|
||||
Log::Fatal("This utility method only handles nrows that are a multiple of batch_count");
|
||||
}
|
||||
|
||||
const double* features_ptr = features->data();
|
||||
const float* labels_ptr = labels->data();
|
||||
const float* weights_ptr = nullptr;
|
||||
if (weights) {
|
||||
weights_ptr = weights->data();
|
||||
}
|
||||
|
||||
// Since init_scores are in a column format, but need to be pushed as rows, we have to extract each batch
|
||||
std::vector<double> init_score_batch;
|
||||
const double* init_scores_ptr = nullptr;
|
||||
if (init_scores) {
|
||||
init_score_batch.reserve(nclasses * batch_count);
|
||||
init_scores_ptr = init_score_batch.data();
|
||||
}
|
||||
|
||||
const int32_t* groups_ptr = nullptr;
|
||||
if (groups) {
|
||||
groups_ptr = groups->data();
|
||||
}
|
||||
|
||||
auto start_time = std::chrono::steady_clock::now();
|
||||
|
||||
for (int32_t i = 0; i < nrows; i += batch_count) {
|
||||
if (init_scores) {
|
||||
init_scores_ptr = CreateInitScoreBatch(&init_score_batch, i, nrows, nclasses, batch_count, init_scores);
|
||||
}
|
||||
|
||||
result = LGBM_DatasetPushRowsWithMetadata(dataset_handle,
|
||||
features_ptr,
|
||||
1,
|
||||
batch_count,
|
||||
ncols,
|
||||
i,
|
||||
labels_ptr,
|
||||
weights_ptr,
|
||||
init_scores_ptr,
|
||||
groups_ptr,
|
||||
0);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetPushRowsWithMetadata result code: " << result;
|
||||
if (result != 0) {
|
||||
FAIL() << "LGBM_DatasetPushRowsWithMetadata failed"; // This forces an immediate failure, which EXPECT_EQ does not
|
||||
}
|
||||
|
||||
features_ptr += batch_count * ncols;
|
||||
labels_ptr += batch_count;
|
||||
if (weights_ptr) {
|
||||
weights_ptr += batch_count;
|
||||
}
|
||||
if (groups_ptr) {
|
||||
groups_ptr += batch_count;
|
||||
}
|
||||
}
|
||||
|
||||
auto cur_time = std::chrono::steady_clock::now();
|
||||
Log::Info(" Time: %d", cur_time - start_time);
|
||||
}
|
||||
|
||||
void TestUtils::StreamSparseDataset(DatasetHandle dataset_handle,
|
||||
int32_t nrows,
|
||||
int32_t nclasses,
|
||||
int32_t batch_count,
|
||||
const std::vector<int32_t>* indptr,
|
||||
const std::vector<int32_t>* indices,
|
||||
const std::vector<double>* values,
|
||||
const std::vector<float>* labels,
|
||||
const std::vector<float>* weights,
|
||||
const std::vector<double>* init_scores,
|
||||
const std::vector<int32_t>* groups) {
|
||||
int result = LGBM_DatasetSetWaitForManualFinish(dataset_handle, 1);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetSetWaitForManualFinish result code: " << result;
|
||||
|
||||
Log::Info(" Begin StreamSparseDataset");
|
||||
if ((nrows % batch_count) != 0) {
|
||||
Log::Fatal("This utility method only handles nrows that are a multiple of batch_count");
|
||||
}
|
||||
|
||||
const int32_t* indptr_ptr = indptr->data();
|
||||
const int32_t* indices_ptr = indices->data();
|
||||
const double* values_ptr = values->data();
|
||||
const float* labels_ptr = labels->data();
|
||||
const float* weights_ptr = nullptr;
|
||||
if (weights) {
|
||||
weights_ptr = weights->data();
|
||||
}
|
||||
|
||||
const int32_t* groups_ptr = nullptr;
|
||||
if (groups) {
|
||||
groups_ptr = groups->data();
|
||||
}
|
||||
|
||||
auto start_time = std::chrono::steady_clock::now();
|
||||
|
||||
// Use multiple threads to test concurrency
|
||||
int thread_count = 2;
|
||||
if (nrows == batch_count) {
|
||||
thread_count = 1; // If pushing all rows in 1 batch, we cannot have multiple threads
|
||||
}
|
||||
std::vector<std::thread> threads;
|
||||
threads.reserve(thread_count);
|
||||
for (int32_t t = 0; t < thread_count; ++t) {
|
||||
std::thread th(TestUtils::PushSparseBatch,
|
||||
dataset_handle,
|
||||
nrows,
|
||||
nclasses,
|
||||
batch_count,
|
||||
indptr,
|
||||
indptr_ptr,
|
||||
indices_ptr,
|
||||
values_ptr,
|
||||
labels_ptr,
|
||||
weights_ptr,
|
||||
init_scores,
|
||||
groups_ptr,
|
||||
thread_count,
|
||||
t);
|
||||
threads.push_back(std::move(th));
|
||||
}
|
||||
|
||||
for (auto& t : threads) t.join();
|
||||
|
||||
auto cur_time = std::chrono::steady_clock::now();
|
||||
Log::Info(" Time: %d", cur_time - start_time);
|
||||
}
|
||||
|
||||
/*!
|
||||
* Pushes data from 1 thread into a Dataset based on thread_id and nrows.
|
||||
* e.g. with 100 rows, thread 0 will push rows 0-49, and thread 2 will push rows 50-99.
|
||||
* Note that rows are still pushed in microbatches within their range.
|
||||
*/
|
||||
void TestUtils::PushSparseBatch(DatasetHandle dataset_handle,
|
||||
int32_t nrows,
|
||||
int32_t nclasses,
|
||||
int32_t batch_count,
|
||||
const std::vector<int32_t>* indptr,
|
||||
const int32_t* indptr_ptr,
|
||||
const int32_t* indices_ptr,
|
||||
const double* values_ptr,
|
||||
const float* labels_ptr,
|
||||
const float* weights_ptr,
|
||||
const std::vector<double>* init_scores,
|
||||
const int32_t* groups_ptr,
|
||||
int32_t thread_count,
|
||||
int32_t thread_id) {
|
||||
int32_t threadChunkSize = nrows / thread_count;
|
||||
int32_t startIndex = threadChunkSize * thread_id;
|
||||
int32_t stopIndex = startIndex + threadChunkSize;
|
||||
|
||||
indptr_ptr += threadChunkSize * thread_id;
|
||||
labels_ptr += threadChunkSize * thread_id;
|
||||
if (weights_ptr) {
|
||||
weights_ptr += threadChunkSize * thread_id;
|
||||
}
|
||||
if (groups_ptr) {
|
||||
groups_ptr += threadChunkSize * thread_id;
|
||||
}
|
||||
|
||||
for (int32_t i = startIndex; i < stopIndex; i += batch_count) {
|
||||
// Since init_scores are in a column format, but need to be pushed as rows, we have to extract each batch
|
||||
std::vector<double> init_score_batch;
|
||||
const double* init_scores_ptr = nullptr;
|
||||
if (init_scores) {
|
||||
init_score_batch.reserve(nclasses * batch_count);
|
||||
init_scores_ptr = CreateInitScoreBatch(&init_score_batch, i, nrows, nclasses, batch_count, init_scores);
|
||||
}
|
||||
|
||||
int32_t nelem = indptr->at(i + batch_count - 1) - indptr->at(i);
|
||||
|
||||
int result = LGBM_DatasetPushRowsByCSRWithMetadata(dataset_handle,
|
||||
indptr_ptr,
|
||||
2,
|
||||
indices_ptr,
|
||||
values_ptr,
|
||||
1,
|
||||
batch_count + 1,
|
||||
nelem,
|
||||
i,
|
||||
labels_ptr,
|
||||
weights_ptr,
|
||||
init_scores_ptr,
|
||||
groups_ptr,
|
||||
thread_id);
|
||||
EXPECT_EQ(0, result) << "LGBM_DatasetPushRowsByCSRWithMetadata result code: " << result;
|
||||
if (result != 0) {
|
||||
FAIL() << "LGBM_DatasetPushRowsByCSRWithMetadata failed"; // This forces an immediate failure, which EXPECT_EQ does not
|
||||
}
|
||||
|
||||
indptr_ptr += batch_count;
|
||||
labels_ptr += batch_count;
|
||||
if (weights_ptr) {
|
||||
weights_ptr += batch_count;
|
||||
}
|
||||
if (groups_ptr) {
|
||||
groups_ptr += batch_count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void TestUtils::AssertMetadata(const Metadata* metadata,
|
||||
const std::vector<float>* ref_labels,
|
||||
const std::vector<float>* ref_weights,
|
||||
const std::vector<double>* ref_init_scores,
|
||||
const std::vector<int32_t>* ref_groups) {
|
||||
const float* labels = metadata->label();
|
||||
auto nTotal = static_cast<int32_t>(ref_labels->size());
|
||||
for (auto i = 0; i < nTotal; i++) {
|
||||
EXPECT_EQ(ref_labels->at(i), labels[i]) << "Inserted data: " << ref_labels->at(i) << " at " << i;
|
||||
if (ref_labels->at(i) != labels[i]) {
|
||||
FAIL() << "Mismatched labels"; // This forces an immediate failure, which EXPECT_EQ does not
|
||||
}
|
||||
}
|
||||
|
||||
const float* weights = metadata->weights();
|
||||
if (weights) {
|
||||
if (!ref_weights) {
|
||||
FAIL() << "Expected null weights";
|
||||
}
|
||||
for (auto i = 0; i < nTotal; i++) {
|
||||
EXPECT_EQ(ref_weights->at(i), weights[i]) << "Inserted data: " << ref_weights->at(i);
|
||||
if (ref_weights->at(i) != weights[i]) {
|
||||
FAIL() << "Mismatched weights"; // This forces an immediate failure, which EXPECT_EQ does not
|
||||
}
|
||||
}
|
||||
} else if (ref_weights) {
|
||||
FAIL() << "Expected non-null weights";
|
||||
}
|
||||
|
||||
const double* init_scores = metadata->init_score();
|
||||
if (init_scores) {
|
||||
if (!ref_init_scores) {
|
||||
FAIL() << "Expected null init_scores";
|
||||
}
|
||||
for (size_t i = 0; i < ref_init_scores->size(); i++) {
|
||||
EXPECT_EQ(ref_init_scores->at(i), init_scores[i]) << "Inserted data: " << ref_init_scores->at(i) << " Index: " << i;
|
||||
if (ref_init_scores->at(i) != init_scores[i]) {
|
||||
FAIL() << "Mismatched init_scores"; // This forces an immediate failure, which EXPECT_EQ does not
|
||||
}
|
||||
}
|
||||
} else if (ref_init_scores) {
|
||||
FAIL() << "Expected non-null init_scores";
|
||||
}
|
||||
|
||||
const int32_t* query_boundaries = metadata->query_boundaries();
|
||||
if (query_boundaries) {
|
||||
if (!ref_groups) {
|
||||
FAIL() << "Expected null query_boundaries";
|
||||
}
|
||||
// Calculate expected boundaries
|
||||
std::vector<int32_t> ref_query_boundaries;
|
||||
ref_query_boundaries.push_back(0);
|
||||
int group_val = ref_groups->at(0);
|
||||
for (auto i = 1; i < nTotal; i++) {
|
||||
if (ref_groups->at(i) != group_val) {
|
||||
ref_query_boundaries.push_back(i);
|
||||
group_val = ref_groups->at(i);
|
||||
}
|
||||
}
|
||||
ref_query_boundaries.push_back(nTotal);
|
||||
|
||||
for (size_t i = 0; i < ref_query_boundaries.size(); i++) {
|
||||
EXPECT_EQ(ref_query_boundaries[i], query_boundaries[i]) << "Inserted data: " << ref_query_boundaries[i];
|
||||
if (ref_query_boundaries[i] != query_boundaries[i]) {
|
||||
FAIL() << "Mismatched query_boundaries"; // This forces an immediate failure, which EXPECT_EQ does not
|
||||
}
|
||||
}
|
||||
} else if (ref_groups) {
|
||||
FAIL() << "Expected non-null query_boundaries";
|
||||
}
|
||||
}
|
||||
|
||||
const double* TestUtils::CreateInitScoreBatch(std::vector<double>* init_score_batch,
|
||||
int32_t index,
|
||||
int32_t nrows,
|
||||
int32_t nclasses,
|
||||
int32_t batch_count,
|
||||
const std::vector<double>* original_init_scores) {
|
||||
// Extract a set of rows from the column-based format (still maintaining column based format)
|
||||
init_score_batch->clear();
|
||||
for (int32_t c = 0; c < nclasses; c++) {
|
||||
for (int32_t row = index; row < index + batch_count; row++) {
|
||||
init_score_batch->push_back(original_init_scores->at(row + nrows * c));
|
||||
}
|
||||
}
|
||||
return init_score_batch->data();
|
||||
}
|
||||
|
||||
} // namespace LightGBM
|
||||
@@ -0,0 +1,125 @@
|
||||
/*!
|
||||
* Copyright (c) 2022-2026 Microsoft Corporation. All rights reserved.
|
||||
* Copyright (c) 2022-2026 The LightGBM developers. All rights reserved.
|
||||
* Licensed under the MIT License. See LICENSE file in the project root for license information.
|
||||
*/
|
||||
#ifndef LIGHTGBM_TESTS_CPP_TESTS_TESTUTILS_H_
|
||||
#define LIGHTGBM_TESTS_CPP_TESTS_TESTUTILS_H_
|
||||
|
||||
#include <LightGBM/c_api.h>
|
||||
#include <LightGBM/dataset.h>
|
||||
|
||||
#include <vector>
|
||||
|
||||
using LightGBM::Metadata;
|
||||
|
||||
namespace LightGBM {
|
||||
|
||||
class TestUtils {
|
||||
public:
|
||||
/*!
|
||||
* Creates a Dataset from the internal repository examples.
|
||||
*/
|
||||
static int LoadDatasetFromExamples(const char* filename, const char* config, DatasetHandle* out);
|
||||
|
||||
|
||||
/*!
|
||||
* Creates a dense Dataset of random values.
|
||||
*/
|
||||
static void CreateRandomDenseData(int32_t nrows,
|
||||
int32_t ncols,
|
||||
int32_t nclasses,
|
||||
std::vector<double>* features,
|
||||
std::vector<float>* labels,
|
||||
std::vector<float>* weights,
|
||||
std::vector<double>* init_scores,
|
||||
std::vector<int32_t>* groups);
|
||||
|
||||
/*!
|
||||
* Creates a CSR sparse Dataset of random values.
|
||||
*/
|
||||
static void CreateRandomSparseData(int32_t nrows,
|
||||
int32_t ncols,
|
||||
int32_t nclasses,
|
||||
float sparse_percent,
|
||||
std::vector<int32_t>* indptr,
|
||||
std::vector<int32_t>* indices,
|
||||
std::vector<double>* values,
|
||||
std::vector<float>* labels,
|
||||
std::vector<float>* weights,
|
||||
std::vector<double>* init_scores,
|
||||
std::vector<int32_t>* groups);
|
||||
|
||||
/*!
|
||||
* Creates a batch of Metadata of random values.
|
||||
*/
|
||||
static void CreateRandomMetadata(int32_t nrows,
|
||||
int32_t nclasses,
|
||||
std::vector<float>* labels,
|
||||
std::vector<float>* weights,
|
||||
std::vector<double>* init_scores,
|
||||
std::vector<int32_t>* groups);
|
||||
|
||||
/*!
|
||||
* Pushes nrows of data to a Dataset in batches of batch_count.
|
||||
*/
|
||||
static void StreamDenseDataset(DatasetHandle dataset_handle,
|
||||
int32_t nrows,
|
||||
int32_t ncols,
|
||||
int32_t nclasses,
|
||||
int32_t batch_count,
|
||||
const std::vector<double>* features,
|
||||
const std::vector<float>* labels,
|
||||
const std::vector<float>* weights,
|
||||
const std::vector<double>* init_scores,
|
||||
const std::vector<int32_t>* groups);
|
||||
|
||||
/*!
|
||||
* Pushes nrows of data to a Dataset in batches of batch_count.
|
||||
*/
|
||||
static void StreamSparseDataset(DatasetHandle dataset_handle,
|
||||
int32_t nrows,
|
||||
int32_t nclasses,
|
||||
int32_t batch_count,
|
||||
const std::vector<int32_t>* indptr,
|
||||
const std::vector<int32_t>* indices,
|
||||
const std::vector<double>* values,
|
||||
const std::vector<float>* labels,
|
||||
const std::vector<float>* weights,
|
||||
const std::vector<double>* init_scores,
|
||||
const std::vector<int32_t>* groups);
|
||||
|
||||
/*!
|
||||
* Validates metadata against reference vectors.
|
||||
*/
|
||||
static void AssertMetadata(const Metadata* metadata,
|
||||
const std::vector<float>* labels,
|
||||
const std::vector<float>* weights,
|
||||
const std::vector<double>* init_scores,
|
||||
const std::vector<int32_t>* groups);
|
||||
|
||||
static const double* CreateInitScoreBatch(std::vector<double>* init_score_batch,
|
||||
int32_t index,
|
||||
int32_t nrows,
|
||||
int32_t nclasses,
|
||||
int32_t batch_count,
|
||||
const std::vector<double>* original_init_scores);
|
||||
|
||||
private:
|
||||
static void PushSparseBatch(DatasetHandle dataset_handle,
|
||||
int32_t nrows,
|
||||
int32_t nclasses,
|
||||
int32_t batch_count,
|
||||
const std::vector<int32_t>* indptr,
|
||||
const int32_t* indptr_ptr,
|
||||
const int32_t* indices_ptr,
|
||||
const double* values_ptr,
|
||||
const float* labels_ptr,
|
||||
const float* weights_ptr,
|
||||
const std::vector<double>* init_scores,
|
||||
const int32_t* groups_ptr,
|
||||
int32_t thread_count,
|
||||
int32_t thread_id);
|
||||
};
|
||||
} // namespace LightGBM
|
||||
#endif // LIGHTGBM_TESTS_CPP_TESTS_TESTUTILS_H_
|
||||
@@ -0,0 +1,7 @@
|
||||
data=../data/categorical.data
|
||||
|
||||
app=binary
|
||||
|
||||
num_trees=10
|
||||
|
||||
categorical_column=0,1,4,5,6
|
||||
Reference in New Issue
Block a user