wehub-resource-sync
1593 lines
58 KiB
C++
1593 lines
58 KiB
C++
// Copyright 2025-present the zvec project
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#include "zvec/db/doc.h"
|
|
#include <cstdint>
|
|
#include <limits>
|
|
#include <gtest/gtest.h>
|
|
#include <zvec/ailego/utility/float_helper.h>
|
|
#include "utils/utils.h"
|
|
#include "zvec/db/index_params.h"
|
|
#include "zvec/db/status.h"
|
|
#include "zvec/db/type.h"
|
|
|
|
|
|
using namespace zvec;
|
|
|
|
class DocDetailedTest : public ::testing::Test {
|
|
protected:
|
|
void SetUp() override {
|
|
test_doc_ = std::make_shared<Doc>();
|
|
test_doc_->set_pk("test_pk");
|
|
test_doc_->set_doc_id(12345);
|
|
test_doc_->set_score(0.95f);
|
|
test_doc_->set_operator(Operator::INSERT);
|
|
}
|
|
|
|
Doc::Ptr test_doc_;
|
|
};
|
|
|
|
// Test serialization and deserialization of basic data types
|
|
TEST_F(DocDetailedTest, BasicTypeSerializationDeserialization) {
|
|
// Test boundary values
|
|
test_doc_->set("bool_true", true);
|
|
test_doc_->set("bool_false", false);
|
|
test_doc_->set("int32_min", std::numeric_limits<int32_t>::min());
|
|
test_doc_->set("int32_max", std::numeric_limits<int32_t>::max());
|
|
test_doc_->set("uint32_min", std::numeric_limits<uint32_t>::min());
|
|
test_doc_->set("uint32_max", std::numeric_limits<uint32_t>::max());
|
|
test_doc_->set("int64_min", std::numeric_limits<int64_t>::min());
|
|
test_doc_->set("int64_max", std::numeric_limits<int64_t>::max());
|
|
test_doc_->set("uint64_min", std::numeric_limits<uint64_t>::min());
|
|
test_doc_->set("uint64_max", std::numeric_limits<uint64_t>::max());
|
|
test_doc_->set("float_min", std::numeric_limits<float>::min());
|
|
test_doc_->set("float_max", std::numeric_limits<float>::max());
|
|
test_doc_->set("float_lowest", std::numeric_limits<float>::lowest());
|
|
test_doc_->set("double_min", std::numeric_limits<double>::min());
|
|
test_doc_->set("double_max", std::numeric_limits<double>::max());
|
|
test_doc_->set("double_lowest", std::numeric_limits<double>::lowest());
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
ASSERT_FALSE(serialized.empty());
|
|
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
ASSERT_NE(deserialized_doc, nullptr);
|
|
|
|
EXPECT_EQ(deserialized_doc->get<bool>("bool_true").value(), true);
|
|
EXPECT_EQ(deserialized_doc->get<bool>("bool_false").value(), false);
|
|
EXPECT_EQ(deserialized_doc->get<int32_t>("int32_min").value(),
|
|
std::numeric_limits<int32_t>::min());
|
|
EXPECT_EQ(deserialized_doc->get<int32_t>("int32_max").value(),
|
|
std::numeric_limits<int32_t>::max());
|
|
EXPECT_EQ(deserialized_doc->get<uint32_t>("uint32_min").value(),
|
|
std::numeric_limits<uint32_t>::min());
|
|
EXPECT_EQ(deserialized_doc->get<uint32_t>("uint32_max").value(),
|
|
std::numeric_limits<uint32_t>::max());
|
|
EXPECT_EQ(deserialized_doc->get<int64_t>("int64_min").value(),
|
|
std::numeric_limits<int64_t>::min());
|
|
EXPECT_EQ(deserialized_doc->get<int64_t>("int64_max").value(),
|
|
std::numeric_limits<int64_t>::max());
|
|
EXPECT_EQ(deserialized_doc->get<uint64_t>("uint64_min").value(),
|
|
std::numeric_limits<uint64_t>::min());
|
|
EXPECT_EQ(deserialized_doc->get<uint64_t>("uint64_max").value(),
|
|
std::numeric_limits<uint64_t>::max());
|
|
|
|
// For floating point numbers, use approximate comparison
|
|
EXPECT_FLOAT_EQ(deserialized_doc->get<float>("float_min").value(),
|
|
std::numeric_limits<float>::min());
|
|
EXPECT_FLOAT_EQ(deserialized_doc->get<float>("float_max").value(),
|
|
std::numeric_limits<float>::max());
|
|
EXPECT_FLOAT_EQ(deserialized_doc->get<float>("float_lowest").value(),
|
|
std::numeric_limits<float>::lowest());
|
|
EXPECT_DOUBLE_EQ(deserialized_doc->get<double>("double_min").value(),
|
|
std::numeric_limits<double>::min());
|
|
EXPECT_DOUBLE_EQ(deserialized_doc->get<double>("double_max").value(),
|
|
std::numeric_limits<double>::max());
|
|
EXPECT_DOUBLE_EQ(deserialized_doc->get<double>("double_lowest").value(),
|
|
std::numeric_limits<double>::lowest());
|
|
}
|
|
|
|
// Test various cases of string types
|
|
TEST_F(DocDetailedTest, StringTypeSerializationDeserialization) {
|
|
// Test empty string
|
|
test_doc_->set("empty_string", std::string(""));
|
|
|
|
// Test long string
|
|
std::string long_string(10000, 'a');
|
|
test_doc_->set("long_string", long_string);
|
|
|
|
// Test string with special characters
|
|
test_doc_->set("special_chars",
|
|
std::string("Special characters\t\n\r\0included", 15));
|
|
|
|
// Test string with binary data
|
|
std::string binary_string;
|
|
for (int i = 0; i < 256; ++i) {
|
|
binary_string.push_back(static_cast<char>(i));
|
|
}
|
|
test_doc_->set("binary_string", binary_string);
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
ASSERT_FALSE(serialized.empty());
|
|
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
ASSERT_NE(deserialized_doc, nullptr);
|
|
|
|
EXPECT_EQ(deserialized_doc->get<std::string>("empty_string").value(), "");
|
|
EXPECT_EQ(deserialized_doc->get<std::string>("long_string").value(),
|
|
long_string);
|
|
EXPECT_EQ(deserialized_doc->get<std::string>("special_chars").value(),
|
|
std::string("Special characters\t\n\r\0included", 15));
|
|
EXPECT_EQ(deserialized_doc->get<std::string>("binary_string").value(),
|
|
binary_string);
|
|
}
|
|
|
|
|
|
// Test vector<bool> type
|
|
TEST_F(DocDetailedTest, VectorBoolSerializationDeserialization) {
|
|
std::vector<bool> bool_vec;
|
|
// Create a vector<bool> with a large number of elements
|
|
for (int i = 0; i < 1000; ++i) {
|
|
bool_vec.push_back(i % 2 == 0);
|
|
}
|
|
test_doc_->set("bool_vec", bool_vec);
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
ASSERT_FALSE(serialized.empty());
|
|
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
ASSERT_NE(deserialized_doc, nullptr);
|
|
|
|
auto deserialized_vec =
|
|
deserialized_doc->get<std::vector<bool>>("bool_vec").value();
|
|
|
|
ASSERT_EQ(deserialized_vec.size(), bool_vec.size());
|
|
for (size_t i = 0; i < bool_vec.size(); ++i) {
|
|
EXPECT_EQ(deserialized_vec[i], bool_vec[i]) << "Mismatch at index " << i;
|
|
}
|
|
}
|
|
|
|
// Test numeric vector types
|
|
TEST_F(DocDetailedTest, NumericVectorSerializationDeserialization) {
|
|
// Test int8_t vector
|
|
std::vector<int8_t> int8_vec = {std::numeric_limits<int8_t>::min(), -1, 0, 1,
|
|
std::numeric_limits<int8_t>::max()};
|
|
test_doc_->set("int8_vec", int8_vec);
|
|
|
|
// Test int16_t vector
|
|
std::vector<int16_t> int16_vec = {std::numeric_limits<int16_t>::min(), -1, 0,
|
|
1, std::numeric_limits<int16_t>::max()};
|
|
test_doc_->set("int16_vec", int16_vec);
|
|
|
|
// Test int32_t vector
|
|
std::vector<int32_t> int32_vec = {std::numeric_limits<int32_t>::min(), -1, 0,
|
|
1, std::numeric_limits<int32_t>::max()};
|
|
test_doc_->set("int32_vec", int32_vec);
|
|
|
|
// Test int64_t vector
|
|
std::vector<int64_t> int64_vec = {std::numeric_limits<int64_t>::min(), -1, 0,
|
|
1, std::numeric_limits<int64_t>::max()};
|
|
test_doc_->set("int64_vec", int64_vec);
|
|
|
|
// Test uint32_t vector
|
|
std::vector<uint32_t> uint32_vec = {std::numeric_limits<uint32_t>::min(), 1,
|
|
100,
|
|
std::numeric_limits<uint32_t>::max()};
|
|
test_doc_->set("uint32_vec", uint32_vec);
|
|
|
|
// Test uint64_t vector
|
|
std::vector<uint64_t> uint64_vec = {std::numeric_limits<uint64_t>::min(), 1,
|
|
100,
|
|
std::numeric_limits<uint64_t>::max()};
|
|
test_doc_->set("uint64_vec", uint64_vec);
|
|
|
|
// Test float vector
|
|
std::vector<float> float_vec = {std::numeric_limits<float>::min(), -1.0f,
|
|
0.0f, 1.0f,
|
|
std::numeric_limits<float>::max()};
|
|
test_doc_->set("float_vec", float_vec);
|
|
|
|
// Test double vector
|
|
std::vector<double> double_vec = {std::numeric_limits<double>::min(), -1.0,
|
|
0.0, 1.0,
|
|
std::numeric_limits<double>::max()};
|
|
test_doc_->set("double_vec", double_vec);
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
ASSERT_FALSE(serialized.empty());
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
ASSERT_NE(deserialized_doc, nullptr);
|
|
|
|
EXPECT_EQ(deserialized_doc->get<std::vector<int8_t>>("int8_vec").value(),
|
|
int8_vec);
|
|
EXPECT_EQ(deserialized_doc->get<std::vector<int16_t>>("int16_vec").value(),
|
|
int16_vec);
|
|
EXPECT_EQ(deserialized_doc->get<std::vector<int32_t>>("int32_vec").value(),
|
|
int32_vec);
|
|
EXPECT_EQ(deserialized_doc->get<std::vector<int64_t>>("int64_vec").value(),
|
|
int64_vec);
|
|
EXPECT_EQ(deserialized_doc->get<std::vector<uint32_t>>("uint32_vec").value(),
|
|
uint32_vec);
|
|
EXPECT_EQ(deserialized_doc->get<std::vector<uint64_t>>("uint64_vec").value(),
|
|
uint64_vec);
|
|
|
|
|
|
// Floating point numbers use approximate comparison
|
|
auto deserialized_float_vec =
|
|
deserialized_doc->get<std::vector<float>>("float_vec").value();
|
|
|
|
ASSERT_EQ(deserialized_float_vec.size(), float_vec.size());
|
|
for (size_t i = 0; i < float_vec.size(); ++i) {
|
|
EXPECT_FLOAT_EQ(deserialized_float_vec[i], float_vec[i])
|
|
<< "Mismatch at index " << i;
|
|
}
|
|
|
|
auto deserialized_double_vec =
|
|
deserialized_doc->get<std::vector<double>>("double_vec").value();
|
|
ASSERT_EQ(deserialized_double_vec.size(), double_vec.size());
|
|
for (size_t i = 0; i < double_vec.size(); ++i) {
|
|
EXPECT_DOUBLE_EQ(deserialized_double_vec[i], double_vec[i])
|
|
<< "Mismatch at index " << i;
|
|
}
|
|
}
|
|
|
|
// Test string vector types
|
|
TEST_F(DocDetailedTest, StringVectorSerializationDeserialization) {
|
|
std::vector<std::string> string_vec;
|
|
string_vec.push_back(""); // Empty string
|
|
string_vec.push_back("normal string");
|
|
string_vec.push_back(std::string(1000, 'x')); // Long string
|
|
string_vec.push_back("Special character test");
|
|
string_vec.push_back(
|
|
std::string("binary\0data", 11)); // Contains binary data
|
|
|
|
test_doc_->set("string_vec", string_vec);
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
ASSERT_FALSE(serialized.empty());
|
|
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
ASSERT_NE(deserialized_doc, nullptr);
|
|
|
|
auto deserialized_vec =
|
|
deserialized_doc->get<std::vector<std::string>>("string_vec").value();
|
|
ASSERT_EQ(deserialized_vec.size(), string_vec.size());
|
|
for (size_t i = 0; i < string_vec.size(); ++i) {
|
|
EXPECT_EQ(deserialized_vec[i], string_vec[i]) << "Mismatch at index " << i;
|
|
}
|
|
}
|
|
|
|
// Test sparse vector types
|
|
TEST_F(DocDetailedTest, SparseVectorSerializationDeserialization) {
|
|
// Test float type sparse vector
|
|
std::pair<std::vector<uint32_t>, std::vector<float>> sparse_float_vec;
|
|
sparse_float_vec.first = {0, 100, 1000, 10000};
|
|
sparse_float_vec.second = {0.1f, 100.5f, -200.7f,
|
|
std::numeric_limits<float>::max()};
|
|
|
|
test_doc_->set("sparse_float_vec", sparse_float_vec);
|
|
|
|
// Test ailego::Float16 type sparse vector
|
|
std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>>
|
|
sparse_float16_vec;
|
|
sparse_float16_vec.first = {1, 50, 500};
|
|
sparse_float16_vec.second = {ailego::Float16(0.5f), ailego::Float16(-10.25f),
|
|
ailego::Float16(1000.0f)};
|
|
|
|
test_doc_->set("sparse_float16_vec", sparse_float16_vec);
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
ASSERT_FALSE(serialized.empty());
|
|
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
ASSERT_NE(deserialized_doc, nullptr);
|
|
|
|
// Verify float sparse vector
|
|
auto deserialized_float_vec =
|
|
deserialized_doc
|
|
->get<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
|
"sparse_float_vec")
|
|
.value();
|
|
|
|
EXPECT_EQ(deserialized_float_vec.first, sparse_float_vec.first);
|
|
ASSERT_EQ(deserialized_float_vec.second.size(),
|
|
sparse_float_vec.second.size());
|
|
for (size_t i = 0; i < sparse_float_vec.second.size(); ++i) {
|
|
EXPECT_FLOAT_EQ(deserialized_float_vec.second[i],
|
|
sparse_float_vec.second[i])
|
|
<< "Mismatch at index " << i;
|
|
}
|
|
|
|
// Verify float16 sparse vector
|
|
auto deserialized_float16_vec =
|
|
deserialized_doc
|
|
->get<std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>>>(
|
|
"sparse_float16_vec")
|
|
.value();
|
|
|
|
EXPECT_EQ(deserialized_float16_vec.first, sparse_float16_vec.first);
|
|
EXPECT_EQ(deserialized_float16_vec.second, sparse_float16_vec.second);
|
|
}
|
|
|
|
// Test case with many fields
|
|
TEST_F(DocDetailedTest, ManyFieldsSerializationDeserialization) {
|
|
const int field_count = 1000;
|
|
for (int i = 0; i < field_count; ++i) {
|
|
test_doc_->set("field_" + std::to_string(i), i);
|
|
}
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
ASSERT_FALSE(serialized.empty());
|
|
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
ASSERT_NE(deserialized_doc, nullptr);
|
|
|
|
for (int i = 0; i < field_count; ++i) {
|
|
std::string field_name = "field_" + std::to_string(i);
|
|
EXPECT_EQ(deserialized_doc->get<int32_t>(field_name).value(), i);
|
|
}
|
|
}
|
|
|
|
// Test empty document
|
|
TEST_F(DocDetailedTest, EmptyDocSerializationDeserialization) {
|
|
Doc::Ptr empty_doc = std::make_shared<Doc>();
|
|
empty_doc->set_pk(""); // Empty primary key
|
|
|
|
auto serialized = empty_doc->serialize();
|
|
ASSERT_FALSE(serialized.empty());
|
|
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
ASSERT_NE(deserialized_doc, nullptr);
|
|
EXPECT_EQ(deserialized_doc->pk(), "");
|
|
}
|
|
|
|
// Test large document
|
|
TEST_F(DocDetailedTest, LargeDocSerializationDeserialization) {
|
|
// Create a document with a large amount of data
|
|
std::string large_string(100000, 'A');
|
|
test_doc_->set("large_string", large_string);
|
|
|
|
std::vector<int32_t> large_vector(50000);
|
|
std::iota(large_vector.begin(), large_vector.end(), 0);
|
|
test_doc_->set("large_vector", large_vector);
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
EXPECT_GT(serialized.size(), 100000); // Should be a large document
|
|
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
ASSERT_NE(deserialized_doc, nullptr);
|
|
|
|
EXPECT_EQ(deserialized_doc->get<std::string>("large_string").value(),
|
|
large_string);
|
|
EXPECT_EQ(deserialized_doc->get<std::vector<int32_t>>("large_vector").value(),
|
|
large_vector);
|
|
}
|
|
|
|
// Test memory usage calculation
|
|
TEST_F(DocDetailedTest, MemoryUsageCalculation) {
|
|
size_t initial_usage = test_doc_->memory_usage();
|
|
|
|
// Add some fields
|
|
test_doc_->set("small_string", std::string("small"));
|
|
test_doc_->set("int_field", int32_t(42));
|
|
test_doc_->set("float_field", 3.14f);
|
|
|
|
size_t usage_with_fields = test_doc_->memory_usage();
|
|
EXPECT_GT(usage_with_fields, initial_usage);
|
|
|
|
// Add a large field
|
|
std::string large_string(10000, 'B');
|
|
test_doc_->set("large_string", large_string);
|
|
|
|
size_t usage_with_large_field = test_doc_->memory_usage();
|
|
EXPECT_GT(usage_with_large_field, usage_with_fields);
|
|
}
|
|
|
|
// Test detailed string representation
|
|
TEST_F(DocDetailedTest, DetailStringRepresentation) {
|
|
test_doc_->set("test_bool", true);
|
|
test_doc_->set("test_int", int32_t(-42));
|
|
test_doc_->set("test_string", std::string("hello"));
|
|
|
|
std::vector<float> float_vec = {1.1f, 2.2f, 3.3f};
|
|
test_doc_->set("test_float_vec", float_vec);
|
|
|
|
std::string detail_str = test_doc_->to_detail_string();
|
|
EXPECT_FALSE(detail_str.empty());
|
|
EXPECT_NE(detail_str.find("test_pk"), std::string::npos);
|
|
EXPECT_NE(detail_str.find("test_bool"), std::string::npos);
|
|
EXPECT_NE(detail_str.find("test_int"), std::string::npos);
|
|
EXPECT_NE(detail_str.find("test_string"), std::string::npos);
|
|
EXPECT_NE(detail_str.find("test_float_vec"), std::string::npos);
|
|
}
|
|
|
|
// Test operator types
|
|
TEST_F(DocDetailedTest, OperatorTypes) {
|
|
test_doc_->set_operator(Operator::INSERT);
|
|
EXPECT_EQ(test_doc_->get_operator(), Operator::INSERT);
|
|
|
|
test_doc_->set_operator(Operator::DELETE);
|
|
EXPECT_EQ(test_doc_->get_operator(), Operator::DELETE);
|
|
|
|
test_doc_->set_operator(Operator::UPDATE);
|
|
EXPECT_EQ(test_doc_->get_operator(), Operator::UPDATE);
|
|
}
|
|
|
|
// Test document ID and score
|
|
TEST_F(DocDetailedTest, DocIdAndScore) {
|
|
test_doc_->set_doc_id(0);
|
|
EXPECT_EQ(test_doc_->doc_id(), 0);
|
|
|
|
test_doc_->set_doc_id(std::numeric_limits<uint64_t>::max());
|
|
EXPECT_EQ(test_doc_->doc_id(), std::numeric_limits<uint64_t>::max());
|
|
|
|
test_doc_->set_score(0.0f);
|
|
EXPECT_FLOAT_EQ(test_doc_->score(), 0.0f);
|
|
|
|
test_doc_->set_score(1.0f);
|
|
EXPECT_FLOAT_EQ(test_doc_->score(), 1.0f);
|
|
|
|
test_doc_->set_score(-1.0f);
|
|
EXPECT_FLOAT_EQ(test_doc_->score(), -1.0f);
|
|
|
|
test_doc_->set_score(std::numeric_limits<float>::max());
|
|
EXPECT_FLOAT_EQ(test_doc_->score(), std::numeric_limits<float>::max());
|
|
}
|
|
|
|
// Test primary key
|
|
TEST_F(DocDetailedTest, PrimaryKey) {
|
|
test_doc_->set_pk("");
|
|
EXPECT_EQ(test_doc_->pk(), "");
|
|
|
|
std::string long_pk(10000, 'X');
|
|
test_doc_->set_pk(long_pk);
|
|
EXPECT_EQ(test_doc_->pk(), long_pk);
|
|
|
|
test_doc_->set_pk("normal_pk");
|
|
EXPECT_EQ(test_doc_->pk(), "normal_pk");
|
|
}
|
|
|
|
// Test duplicate field names (should overwrite old values)
|
|
TEST_F(DocDetailedTest, DuplicateFieldNames) {
|
|
test_doc_->set("duplicate_field", int32_t(1));
|
|
test_doc_->set("duplicate_field", int32_t(2)); // Overwrite old value
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
|
|
EXPECT_EQ(deserialized_doc->get<int32_t>("duplicate_field").value(), 2);
|
|
}
|
|
|
|
// Test combination of various data types
|
|
TEST_F(DocDetailedTest, MixedDataTypes) {
|
|
test_doc_->set("bool_field", true);
|
|
test_doc_->set("int_field", int32_t(-1000));
|
|
test_doc_->set("uint_field", uint32_t(2000));
|
|
test_doc_->set("float_field", 3.14159f);
|
|
test_doc_->set("double_field", 2.718281828459045);
|
|
test_doc_->set("string_field", std::string("Hello, World!"));
|
|
|
|
std::vector<int32_t> int_vec = {1, 2, 3, 4, 5};
|
|
test_doc_->set("int_vec", int_vec);
|
|
|
|
std::vector<float> float_vec = {1.1f, 2.2f, 3.3f};
|
|
test_doc_->set("float_vec", float_vec);
|
|
|
|
std::vector<std::string> string_vec = {"apple", "banana", "cherry"};
|
|
test_doc_->set("string_vec", string_vec);
|
|
|
|
std::pair<std::vector<uint32_t>, std::vector<float>> sparse_vec;
|
|
sparse_vec.first = {1, 10, 100};
|
|
sparse_vec.second = {0.1f, 1.0f, 10.0f};
|
|
test_doc_->set("sparse_vec", sparse_vec);
|
|
|
|
auto serialized = test_doc_->serialize();
|
|
auto deserialized_doc =
|
|
Doc::deserialize(serialized.data(), serialized.size());
|
|
|
|
EXPECT_EQ(deserialized_doc->get<bool>("bool_field").value(), true);
|
|
EXPECT_EQ(deserialized_doc->get<int32_t>("int_field").value(), -1000);
|
|
EXPECT_EQ(deserialized_doc->get<uint32_t>("uint_field").value(), 2000);
|
|
EXPECT_FLOAT_EQ(deserialized_doc->get<float>("float_field").value(),
|
|
3.14159f);
|
|
EXPECT_DOUBLE_EQ(deserialized_doc->get<double>("double_field").value(),
|
|
2.718281828459045);
|
|
EXPECT_EQ(deserialized_doc->get<std::string>("string_field").value(),
|
|
"Hello, World!");
|
|
EXPECT_EQ(deserialized_doc->get<std::vector<int32_t>>("int_vec").value(),
|
|
int_vec);
|
|
EXPECT_EQ(deserialized_doc->get<std::vector<float>>("float_vec").value(),
|
|
float_vec);
|
|
EXPECT_EQ(
|
|
deserialized_doc->get<std::vector<std::string>>("string_vec").value(),
|
|
string_vec);
|
|
|
|
auto deserialized_sparse =
|
|
deserialized_doc
|
|
->get<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
|
"sparse_vec")
|
|
.value();
|
|
EXPECT_EQ(deserialized_sparse.first, sparse_vec.first);
|
|
EXPECT_EQ(deserialized_sparse.second, sparse_vec.second);
|
|
}
|
|
|
|
// Test doc validation and sanitization
|
|
TEST_F(DocDetailedTest, ValidateAndSanitization) {
|
|
// nullable=false: a doc with a null field is rejected
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
|
|
doc = test::TestHelper::CreateDocNull(1, *schema);
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok());
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// nullable=true: a doc with a null field is accepted
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(true);
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
|
|
doc = test::TestHelper::CreateDocNull(1, *schema);
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
}
|
|
|
|
// doc has a field that is not declared in the schema
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
|
|
doc.set("another_field", 1);
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok());
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// scalar field value type does not match the schema
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
|
|
doc.set("int32", std::string("1"));
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok());
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// dense vector field element type does not match the schema
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
|
|
std::string field = "dense_fp32";
|
|
auto field_schema = schema->get_field(field);
|
|
ASSERT_NE(field_schema, nullptr);
|
|
|
|
doc.set(field, std::vector<int16_t>(field_schema->dimension(), 1));
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok());
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// dense vector dimension does not match the schema
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
|
|
std::string field = "dense_fp32";
|
|
auto field_schema = schema->get_field(field);
|
|
ASSERT_NE(field_schema, nullptr);
|
|
|
|
doc.set(field, std::vector<float>(field_schema->dimension() - 1, 1.0));
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok());
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
|
|
doc.set(field, std::vector<float>());
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok());
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// sparse vector field value type does not match the schema
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
|
|
std::string field = "sparse_fp32";
|
|
auto field_schema = schema->get_field(field);
|
|
ASSERT_NE(field_schema, nullptr);
|
|
|
|
doc.set(field, std::vector<int16_t>(field_schema->dimension(), 1));
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok());
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// sparse vector indices and values have different lengths
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
|
|
std::string field = "sparse_fp32";
|
|
auto field_schema = schema->get_field(field);
|
|
ASSERT_NE(field_schema, nullptr);
|
|
|
|
std::vector<uint32_t> indices;
|
|
std::vector<float> values;
|
|
for (uint32_t i = 0; i < 100; i++) {
|
|
indices.push_back(i);
|
|
values.push_back(float(0.1));
|
|
}
|
|
values.push_back(float(0.1));
|
|
std::pair<std::vector<uint32_t>, std::vector<float>> sparse_float_vec{
|
|
indices, values};
|
|
doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
|
field, sparse_float_vec);
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok());
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// sparse vector indices are sorted in place; duplicates are rejected
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
|
|
// unsorted indices are accepted and sorted in place
|
|
std::pair<std::vector<uint32_t>, std::vector<float>> unsorted{
|
|
{42u, 7u, 1000u, 3u, 128u, 17u, 99u},
|
|
{0.7f, 0.1f, 0.9f, 0.2f, 0.5f, 0.3f, 0.6f}};
|
|
doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>("sparse_fp32",
|
|
unsorted);
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok()) << s.message();
|
|
const auto sorted_opt =
|
|
doc.get<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
|
"sparse_fp32");
|
|
ASSERT_TRUE(sorted_opt.has_value());
|
|
const std::vector<uint32_t> expected_sorted_indices{3u, 7u, 17u, 42u,
|
|
99u, 128u, 1000u};
|
|
ASSERT_EQ(sorted_opt->first, expected_sorted_indices);
|
|
ASSERT_EQ(sorted_opt->second.size(), expected_sorted_indices.size());
|
|
|
|
// sorted indices with a duplicate are rejected
|
|
std::pair<std::vector<uint32_t>, std::vector<float>> dup{
|
|
{3u, 7u, 17u, 42u, 42u, 99u, 128u},
|
|
{0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f}};
|
|
doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>("sparse_fp32",
|
|
dup);
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok());
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// validate rejects: null schema, missing pk, undefined field type
|
|
{
|
|
Doc doc;
|
|
// null schema
|
|
auto s = doc.validate_and_sanitize(nullptr);
|
|
ASSERT_EQ(s.code(), StatusCode::INTERNAL_ERROR);
|
|
|
|
// doc has no pk field
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
|
|
// schema contains a field with an undefined data type
|
|
schema->add_field(
|
|
std::make_shared<FieldSchema>("undefined", DataType::UNDEFINED, true));
|
|
s = doc.validate_and_sanitize(schema);
|
|
ASSERT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// validate accepts every supported field data type
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
schema->add_field(
|
|
std::make_shared<FieldSchema>("binary", DataType::BINARY, false));
|
|
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"array_binary", DataType::ARRAY_BINARY, false));
|
|
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"vector_binary32", DataType::VECTOR_BINARY32, 128, false,
|
|
std::make_shared<FlatIndexParams>(MetricType::IP)));
|
|
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"vector_binary64", DataType::VECTOR_BINARY64, 128, false,
|
|
std::make_shared<FlatIndexParams>(MetricType::IP)));
|
|
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"vector_int8", DataType::VECTOR_INT8, 128, false,
|
|
std::make_shared<FlatIndexParams>(MetricType::IP)));
|
|
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"vector_int8", DataType::VECTOR_INT8, 128, false,
|
|
std::make_shared<FlatIndexParams>(MetricType::IP)));
|
|
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"vector_int16", DataType::VECTOR_INT16, 128, false,
|
|
std::make_shared<FlatIndexParams>(MetricType::IP)));
|
|
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"dense_fp16", DataType::VECTOR_FP16, 128, false,
|
|
std::make_shared<FlatIndexParams>(MetricType::IP)));
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"dense_fp64", DataType::VECTOR_FP64, 128, false,
|
|
std::make_shared<FlatIndexParams>(MetricType::IP)));
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"sparse_fp16", DataType::SPARSE_VECTOR_FP16, 128, false,
|
|
std::make_shared<FlatIndexParams>(MetricType::IP)));
|
|
schema->add_field(std::make_shared<FieldSchema>(
|
|
"sparse_fp32", DataType::SPARSE_VECTOR_FP32, 128, false,
|
|
std::make_shared<FlatIndexParams>(MetricType::IP)));
|
|
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema);
|
|
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok());
|
|
}
|
|
|
|
// pk with characters inside the allowed set is accepted
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
std::vector<std::string> valid_names = {
|
|
// Min length = 1
|
|
"a",
|
|
"Z",
|
|
"0",
|
|
"_",
|
|
"-",
|
|
"!",
|
|
"@",
|
|
"#",
|
|
"$",
|
|
"%",
|
|
"+",
|
|
"=",
|
|
".",
|
|
|
|
// Mixed
|
|
"a1_",
|
|
"user.name",
|
|
"test@example",
|
|
"log_2025!@#",
|
|
"metric+=value",
|
|
"score%change",
|
|
|
|
"user.name", // '.' allowed
|
|
"test@example", // '@' allowed
|
|
"log_2025!@#", // !@# allowed
|
|
"metric+=value", // + = allowed
|
|
"score%change", // % allowed
|
|
"file-name_v1.2", // -, _, . allowed
|
|
"a-b_c.d!@#$%+=.", // all specials in one
|
|
|
|
// Max length = 64
|
|
std::string(64, 'a'),
|
|
std::string(63, 'a') + "_",
|
|
"_" + std::string(62, 'x') + ".",
|
|
"!" + std::string(62, '0') + "@",
|
|
};
|
|
for (auto pk : valid_names) {
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema, pk);
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_TRUE(s.ok()) << "expected valid pk: " << pk
|
|
<< ", got: " << s.message();
|
|
}
|
|
}
|
|
|
|
// pk that is too long or uses disallowed characters is rejected
|
|
{
|
|
auto schema = test::TestHelper::CreateNormalSchema(false);
|
|
std::vector<std::string> invalid_names = {
|
|
// Too long (>64)
|
|
std::string(65, 'a'), std::string(64, 'a') + "_",
|
|
|
|
// Illegal characters
|
|
"a b", // space
|
|
"a&b", // & not in set
|
|
"a*b", // *
|
|
"a(b)", // ( )
|
|
"a:b", // :
|
|
"a;b", // ;
|
|
"a/b", // /
|
|
"a\\b", // backslash
|
|
"a\"b", // "
|
|
"a'b", // '
|
|
"a<b",
|
|
"a>b", // < >
|
|
"a?b", // ?
|
|
"a~b", // ~
|
|
"a`b", // `
|
|
"a[b",
|
|
"a]b", // [ ]
|
|
"a{b",
|
|
"a}b", // { }
|
|
"a|b", // |
|
|
"a^b", // ^
|
|
"a,b", // ,
|
|
"用户", // non-ASCII (Chinese)
|
|
"αβγ", // Greek
|
|
"résumé", // accented chars (é not in [a-zA-Z])
|
|
};
|
|
for (auto pk : invalid_names) {
|
|
auto doc = test::TestHelper::CreateDoc(1, *schema, pk);
|
|
auto s = doc.validate_and_sanitize(schema);
|
|
ASSERT_FALSE(s.ok()) << "expected invalid pk: " << pk;
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST_F(DocDetailedTest, GetValueTypeNameCoverage) {
|
|
Doc::Value bool_val = true;
|
|
EXPECT_EQ(get_value_type_name(bool_val, false), "BOOL");
|
|
|
|
Doc::Value int32_val = int32_t(42);
|
|
EXPECT_EQ(get_value_type_name(int32_val, false), "INT32");
|
|
|
|
Doc::Value uint32_val = uint32_t(42);
|
|
EXPECT_EQ(get_value_type_name(uint32_val, false), "UINT32");
|
|
|
|
Doc::Value int64_val = int64_t(42);
|
|
EXPECT_EQ(get_value_type_name(int64_val, false), "INT64");
|
|
|
|
Doc::Value uint64_val = uint64_t(42);
|
|
EXPECT_EQ(get_value_type_name(uint64_val, false), "UINT64");
|
|
|
|
Doc::Value float_val = 3.14f;
|
|
EXPECT_EQ(get_value_type_name(float_val, false), "FLOAT");
|
|
|
|
Doc::Value double_val = 3.14;
|
|
EXPECT_EQ(get_value_type_name(double_val, false), "DOUBLE");
|
|
|
|
Doc::Value string_val = std::string("test");
|
|
EXPECT_EQ(get_value_type_name(string_val, false), "STRING");
|
|
|
|
Doc::Value vector_bool_val = std::vector<bool>{true, false};
|
|
EXPECT_EQ(get_value_type_name(vector_bool_val, false), "ARRAY_BOOL");
|
|
|
|
Doc::Value vector_int8_val = std::vector<int8_t>{1, 2, 3};
|
|
EXPECT_EQ(get_value_type_name(vector_int8_val, true), "VECTOR_INT8");
|
|
|
|
Doc::Value vector_int16_val = std::vector<int16_t>{10, 20, 30};
|
|
EXPECT_EQ(get_value_type_name(vector_int16_val, true), "VECTOR_INT16");
|
|
|
|
Doc::Value vector_int32_val = std::vector<int32_t>{100, 200, 300};
|
|
EXPECT_EQ(get_value_type_name(vector_int32_val, true), "VECTOR_INT32");
|
|
|
|
Doc::Value vector_int64_val = std::vector<int64_t>{1000, 2000, 3000};
|
|
EXPECT_EQ(get_value_type_name(vector_int64_val, true), "VECTOR_INT64");
|
|
|
|
Doc::Value vector_uint32_val = std::vector<uint32_t>{10, 20, 30};
|
|
EXPECT_EQ(get_value_type_name(vector_uint32_val, true), "VECTOR_UINT32");
|
|
|
|
Doc::Value vector_uint64_val = std::vector<uint64_t>{100, 200, 300};
|
|
EXPECT_EQ(get_value_type_name(vector_uint64_val, true), "VECTOR_UINT64");
|
|
|
|
Doc::Value vector_float_val = std::vector<float>{1.1f, 2.2f, 3.3f};
|
|
EXPECT_EQ(get_value_type_name(vector_float_val, true), "VECTOR_FP32");
|
|
|
|
Doc::Value vector_double_val = std::vector<double>{1.1, 2.2, 3.3};
|
|
EXPECT_EQ(get_value_type_name(vector_double_val, true), "VECTOR_FP64");
|
|
|
|
Doc::Value vector_float16_val = std::vector<ailego::Float16>{
|
|
ailego::Float16(1.1f), ailego::Float16(2.2f), ailego::Float16(3.3f)};
|
|
EXPECT_EQ(get_value_type_name(vector_float16_val, true), "VECTOR_FP16");
|
|
|
|
Doc::Value vector_string_val = std::vector<std::string>{"a", "b", "c"};
|
|
EXPECT_EQ(get_value_type_name(vector_string_val, false), "ARRAY_STRING");
|
|
|
|
Doc::Value sparse_fp32_val =
|
|
std::pair<std::vector<uint32_t>, std::vector<float>>(
|
|
std::vector<uint32_t>{1, 2, 3}, std::vector<float>{1.1f, 2.2f, 3.3f});
|
|
EXPECT_EQ(get_value_type_name(sparse_fp32_val, true), "SPARSE_VECTOR_FP32");
|
|
|
|
Doc::Value sparse_fp16_val =
|
|
std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>>(
|
|
std::vector<uint32_t>{1, 2, 3},
|
|
std::vector<ailego::Float16>{ailego::Float16(1.1f),
|
|
ailego::Float16(2.2f),
|
|
ailego::Float16(3.3f)});
|
|
EXPECT_EQ(get_value_type_name(sparse_fp16_val, true), "SPARSE_VECTOR_FP16");
|
|
|
|
// Test monostate (null) value
|
|
Doc::Value null_val = std::monostate{};
|
|
EXPECT_EQ(get_value_type_name(null_val, false), "EMPTY");
|
|
}
|
|
|
|
TEST_F(DocDetailedTest, SerializeValueCoverage) {
|
|
Doc doc;
|
|
|
|
doc.set<bool>("bool_field", true);
|
|
doc.set<int32_t>("int32_field", 42);
|
|
doc.set<uint32_t>("uint32_field", 42);
|
|
doc.set<int64_t>("int64_field", 42);
|
|
doc.set<uint64_t>("uint64_field", 42);
|
|
doc.set<float>("float_field", 3.14f);
|
|
doc.set<double>("double_field", 3.14);
|
|
doc.set<std::string>("string_field", "test");
|
|
|
|
std::vector<bool> bool_vec = {true, false};
|
|
doc.set<std::vector<bool>>("vector_bool_field", bool_vec);
|
|
|
|
std::vector<int8_t> int8_vec = {1, 2, 3};
|
|
doc.set<std::vector<int8_t>>("vector_int8_field", int8_vec);
|
|
|
|
std::vector<int16_t> int16_vec = {10, 20, 30};
|
|
doc.set<std::vector<int16_t>>("vector_int16_field", int16_vec);
|
|
|
|
std::vector<int32_t> int32_vec = {100, 200, 300};
|
|
doc.set<std::vector<int32_t>>("vector_int32_field", int32_vec);
|
|
|
|
std::vector<int64_t> int64_vec = {1000, 2000, 3000};
|
|
doc.set<std::vector<int64_t>>("vector_int64_field", int64_vec);
|
|
|
|
std::vector<uint32_t> uint32_vec = {10, 20, 30};
|
|
doc.set<std::vector<uint32_t>>("vector_uint32_field", uint32_vec);
|
|
|
|
std::vector<uint64_t> uint64_vec = {100, 200, 300};
|
|
doc.set<std::vector<uint64_t>>("vector_uint64_field", uint64_vec);
|
|
|
|
std::vector<float> float_vec = {1.1f, 2.2f, 3.3f};
|
|
doc.set<std::vector<float>>("vector_float_field", float_vec);
|
|
|
|
std::vector<double> double_vec = {1.1, 2.2, 3.3};
|
|
doc.set<std::vector<double>>("vector_double_field", double_vec);
|
|
|
|
std::vector<ailego::Float16> float16_vec = {
|
|
ailego::Float16(1.1f), ailego::Float16(2.2f), ailego::Float16(3.3f)};
|
|
doc.set<std::vector<ailego::Float16>>("vector_float16_field", float16_vec);
|
|
|
|
std::vector<std::string> string_vec = {"a", "b", "c"};
|
|
doc.set<std::vector<std::string>>("vector_string_field", string_vec);
|
|
|
|
std::pair<std::vector<uint32_t>, std::vector<float>> sparse_fp32(
|
|
std::vector<uint32_t>{1, 2, 3}, std::vector<float>{1.1f, 2.2f, 3.3f});
|
|
doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
|
"sparse_fp32_field", sparse_fp32);
|
|
|
|
std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>> sparse_fp16(
|
|
std::vector<uint32_t>{1, 2, 3},
|
|
std::vector<ailego::Float16>{ailego::Float16(1.1f), ailego::Float16(2.2f),
|
|
ailego::Float16(3.3f)});
|
|
doc.set<std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>>>(
|
|
"sparse_fp16_field", sparse_fp16);
|
|
|
|
// Test null value
|
|
doc.set_null("null_field");
|
|
|
|
// for code coverage
|
|
EXPECT_GT(doc.to_detail_string().size(), doc.to_string().size());
|
|
|
|
auto buffer = doc.serialize();
|
|
EXPECT_FALSE(buffer.empty());
|
|
|
|
auto deserialized_doc = Doc::deserialize(buffer.data(), buffer.size());
|
|
EXPECT_NE(deserialized_doc, nullptr);
|
|
|
|
EXPECT_EQ(deserialized_doc->get<bool>("bool_field"), true);
|
|
EXPECT_EQ(deserialized_doc->get<int32_t>("int32_field"), 42);
|
|
EXPECT_EQ(deserialized_doc->get<uint32_t>("uint32_field"), 42u);
|
|
EXPECT_EQ(deserialized_doc->get<int64_t>("int64_field"), 42);
|
|
EXPECT_EQ(deserialized_doc->get<uint64_t>("uint64_field"), 42u);
|
|
EXPECT_FLOAT_EQ(deserialized_doc->get<float>("float_field").value(), 3.14f);
|
|
EXPECT_DOUBLE_EQ(deserialized_doc->get<double>("double_field").value(), 3.14);
|
|
EXPECT_EQ(deserialized_doc->get<std::string>("string_field"), "test");
|
|
|
|
// Test null value deserialization
|
|
EXPECT_TRUE(deserialized_doc->is_null("null_field"));
|
|
EXPECT_FALSE(deserialized_doc->has_value("null_field"));
|
|
EXPECT_TRUE(deserialized_doc->has("null_field"));
|
|
}
|
|
|
|
TEST_F(DocDetailedTest, ToDetailStringCoverage) {
|
|
Doc doc;
|
|
doc.set_pk("test_pk");
|
|
doc.set_doc_id(1);
|
|
doc.set_score(0.95f);
|
|
|
|
doc.set<bool>("bool_field", true);
|
|
doc.set<int32_t>("int32_field", 42);
|
|
doc.set<uint32_t>("uint32_field", 42);
|
|
doc.set<int64_t>("int64_field", 42);
|
|
doc.set<uint64_t>("uint64_field", 42);
|
|
doc.set<float>("float_field", 3.14f);
|
|
doc.set<double>("double_field", 3.14);
|
|
doc.set<std::string>("string_field", "test");
|
|
|
|
std::vector<bool> bool_vec = {true, false};
|
|
doc.set<std::vector<bool>>("vector_bool_field", bool_vec);
|
|
|
|
std::vector<int8_t> int8_vec = {1, 2};
|
|
doc.set<std::vector<int8_t>>("vector_int8_field", int8_vec);
|
|
|
|
std::vector<int16_t> int16_vec = {10, 20};
|
|
doc.set<std::vector<int16_t>>("vector_int16_field", int16_vec);
|
|
|
|
std::vector<int32_t> int32_vec = {100, 200};
|
|
doc.set<std::vector<int32_t>>("vector_int32_field", int32_vec);
|
|
|
|
std::vector<int64_t> int64_vec = {1000, 2000};
|
|
doc.set<std::vector<int64_t>>("vector_int64_field", int64_vec);
|
|
|
|
std::vector<uint32_t> uint32_vec = {10, 20};
|
|
doc.set<std::vector<uint32_t>>("vector_uint32_field", uint32_vec);
|
|
|
|
std::vector<uint64_t> uint64_vec = {100, 200};
|
|
doc.set<std::vector<uint64_t>>("vector_uint64_field", uint64_vec);
|
|
|
|
std::vector<float> float_vec = {1.1f, 2.2f};
|
|
doc.set<std::vector<float>>("vector_float_field", float_vec);
|
|
|
|
std::vector<double> double_vec = {1.1, 2.2};
|
|
doc.set<std::vector<double>>("vector_double_field", double_vec);
|
|
|
|
std::vector<ailego::Float16> float16_vec = {ailego::Float16(1.1f),
|
|
ailego::Float16(2.2f)};
|
|
doc.set<std::vector<ailego::Float16>>("vector_float16_field", float16_vec);
|
|
|
|
std::vector<std::string> string_vec = {"a", "b"};
|
|
doc.set<std::vector<std::string>>("vector_string_field", string_vec);
|
|
|
|
std::pair<std::vector<uint32_t>, std::vector<float>> sparse_fp32(
|
|
std::vector<uint32_t>{1, 2}, std::vector<float>{1.1f, 2.2f});
|
|
doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
|
"sparse_fp32_field", sparse_fp32);
|
|
|
|
std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>> sparse_fp16(
|
|
std::vector<uint32_t>{1, 2},
|
|
std::vector<ailego::Float16>{ailego::Float16(1.1f),
|
|
ailego::Float16(2.2f)});
|
|
doc.set<std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>>>(
|
|
"sparse_fp16_field", sparse_fp16);
|
|
|
|
// Test null value in detail string
|
|
doc.set_null("null_field");
|
|
|
|
std::string detail_str = doc.to_detail_string();
|
|
EXPECT_FALSE(detail_str.empty());
|
|
EXPECT_NE(detail_str.find("bool_field"), std::string::npos);
|
|
EXPECT_NE(detail_str.find("int32_field"), std::string::npos);
|
|
EXPECT_NE(detail_str.find("vector_float_field"), std::string::npos);
|
|
EXPECT_NE(detail_str.find("null"),
|
|
std::string::npos); // Should contain "null" for null field
|
|
}
|
|
|
|
TEST_F(DocDetailedTest, EqualityOperatorCoverage) {
|
|
Doc doc1, doc2;
|
|
doc1.set_pk("test_pk");
|
|
doc2.set_pk("test_pk");
|
|
|
|
doc1.set_doc_id(1);
|
|
doc2.set_doc_id(1);
|
|
|
|
doc1.set<bool>("bool_field", true);
|
|
doc2.set<bool>("bool_field", true);
|
|
|
|
doc1.set<int32_t>("int32_field", 42);
|
|
doc2.set<int32_t>("int32_field", 42);
|
|
|
|
doc1.set<uint32_t>("uint32_field", 42);
|
|
doc2.set<uint32_t>("uint32_field", 42);
|
|
|
|
doc1.set<int64_t>("int64_field", 42);
|
|
doc2.set<int64_t>("int64_field", 42);
|
|
|
|
doc1.set<uint64_t>("uint64_field", 42);
|
|
doc2.set<uint64_t>("uint64_field", 42);
|
|
|
|
doc1.set<float>("float_field", 3.14f);
|
|
doc2.set<float>("float_field", 3.14f);
|
|
|
|
doc1.set<double>("double_field", 3.14);
|
|
doc2.set<double>("double_field", 3.14);
|
|
|
|
doc1.set<std::string>("string_field", "test");
|
|
doc2.set<std::string>("string_field", "test");
|
|
|
|
std::vector<bool> bool_vec = {true, false};
|
|
doc1.set<std::vector<bool>>("vector_bool_field", bool_vec);
|
|
doc2.set<std::vector<bool>>("vector_bool_field", bool_vec);
|
|
|
|
std::vector<int8_t> int8_vec = {1, 2};
|
|
doc1.set<std::vector<int8_t>>("vector_int8_field", int8_vec);
|
|
doc2.set<std::vector<int8_t>>("vector_int8_field", int8_vec);
|
|
|
|
std::vector<int16_t> int16_vec = {10, 20};
|
|
doc1.set<std::vector<int16_t>>("vector_int16_field", int16_vec);
|
|
doc2.set<std::vector<int16_t>>("vector_int16_field", int16_vec);
|
|
|
|
std::vector<int32_t> int32_vec = {100, 200};
|
|
doc1.set<std::vector<int32_t>>("vector_int32_field", int32_vec);
|
|
doc2.set<std::vector<int32_t>>("vector_int32_field", int32_vec);
|
|
|
|
std::vector<int64_t> int64_vec = {1000, 2000};
|
|
doc1.set<std::vector<int64_t>>("vector_int64_field", int64_vec);
|
|
doc2.set<std::vector<int64_t>>("vector_int64_field", int64_vec);
|
|
|
|
std::vector<uint32_t> uint32_vec = {10, 20};
|
|
doc1.set<std::vector<uint32_t>>("vector_uint32_field", uint32_vec);
|
|
doc2.set<std::vector<uint32_t>>("vector_uint32_field", uint32_vec);
|
|
|
|
std::vector<uint64_t> uint64_vec = {100, 200};
|
|
doc1.set<std::vector<uint64_t>>("vector_uint64_field", uint64_vec);
|
|
doc2.set<std::vector<uint64_t>>("vector_uint64_field", uint64_vec);
|
|
|
|
std::vector<float> float_vec = {1.1f, 2.2f};
|
|
doc1.set<std::vector<float>>("vector_float_field", float_vec);
|
|
doc2.set<std::vector<float>>("vector_float_field", float_vec);
|
|
|
|
std::vector<double> double_vec = {1.1, 2.2};
|
|
doc1.set<std::vector<double>>("vector_double_field", double_vec);
|
|
doc2.set<std::vector<double>>("vector_double_field", double_vec);
|
|
|
|
std::vector<ailego::Float16> float16_vec = {ailego::Float16(1.1f),
|
|
ailego::Float16(2.2f)};
|
|
doc1.set<std::vector<ailego::Float16>>("vector_float16_field", float16_vec);
|
|
doc2.set<std::vector<ailego::Float16>>("vector_float16_field", float16_vec);
|
|
|
|
std::vector<std::string> string_vec = {"a", "b"};
|
|
doc1.set<std::vector<std::string>>("vector_string_field", string_vec);
|
|
doc2.set<std::vector<std::string>>("vector_string_field", string_vec);
|
|
|
|
std::pair<std::vector<uint32_t>, std::vector<float>> sparse_fp32(
|
|
std::vector<uint32_t>{1, 2}, std::vector<float>{1.1f, 2.2f});
|
|
doc1.set<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
|
"sparse_fp32_field", sparse_fp32);
|
|
doc2.set<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
|
"sparse_fp32_field", sparse_fp32);
|
|
|
|
std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>> sparse_fp16(
|
|
std::vector<uint32_t>{1, 2},
|
|
std::vector<ailego::Float16>{ailego::Float16(1.1f),
|
|
ailego::Float16(2.2f)});
|
|
doc1.set<std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>>>(
|
|
"sparse_fp16_field", sparse_fp16);
|
|
doc2.set<std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>>>(
|
|
"sparse_fp16_field", sparse_fp16);
|
|
|
|
// Test equality with null values
|
|
doc1.set_null("null_field");
|
|
doc2.set_null("null_field");
|
|
|
|
EXPECT_TRUE(doc1 == doc2);
|
|
|
|
doc2.set<int32_t>("int32_field", 43);
|
|
EXPECT_FALSE(doc1 == doc2);
|
|
|
|
doc1.set_pk("test_pk1");
|
|
EXPECT_FALSE(doc1 == doc2);
|
|
|
|
doc1.set_pk("test_pk");
|
|
doc1.set<uint32_t>("int32_field", 42);
|
|
EXPECT_FALSE(doc1 == doc2);
|
|
|
|
doc1.set<int32_t>("int32_field", 42);
|
|
doc1.set<int32_t>("rename_int32_field", 42);
|
|
EXPECT_FALSE(doc1 == doc2);
|
|
|
|
// Test inequality with different null values
|
|
Doc doc3, doc4;
|
|
doc3.set_pk("test");
|
|
doc4.set_pk("test");
|
|
doc3.set_null("null_field");
|
|
doc4.set<int32_t>("null_field", 42);
|
|
EXPECT_FALSE(doc3 == doc4);
|
|
}
|
|
|
|
|
|
TEST(SearchQuery, ValidateAndSanitize) {
|
|
// scalar-only query (no query vector): field schema is null
|
|
{
|
|
SearchQuery query;
|
|
query.topk_ = 10;
|
|
query.target_.field_name_ = "field_name";
|
|
auto s = query.validate(nullptr, nullptr);
|
|
EXPECT_TRUE(s.ok());
|
|
}
|
|
|
|
// vector query requires a non-null field schema
|
|
{
|
|
SearchQuery query;
|
|
query.topk_ = 10;
|
|
query.target_.field_name_ = "field_name";
|
|
std::vector<float> query_vector = {1.0f, 2.0f, 3.0f, 4.0f};
|
|
std::string query_vector_str =
|
|
std::string(reinterpret_cast<char *>(query_vector.data()),
|
|
query_vector.size() * sizeof(float));
|
|
query.target_.set_vector(query_vector_str);
|
|
auto s = query.validate(nullptr, nullptr);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// output_fields count exceeds the allowed maximum
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 10;
|
|
query.output_fields_ = std::vector<std::string>(1025);
|
|
FieldSchema schema = FieldSchema("field_name", DataType::INT32);
|
|
auto s = query.validate(&schema, nullptr);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// dense vector query dimension must match the field schema
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 100;
|
|
std::vector<float> query_vector = {1.0f, 2.0f, 3.0f, 4.0f};
|
|
std::string query_vector_str =
|
|
std::string(reinterpret_cast<char *>(query_vector.data()),
|
|
query_vector.size() * sizeof(float));
|
|
query.target_.set_vector(query_vector_str);
|
|
FieldSchema schema =
|
|
FieldSchema("field_name", DataType::VECTOR_FP32, 4, true);
|
|
auto s = query.validate(&schema, nullptr);
|
|
EXPECT_TRUE(s.ok());
|
|
|
|
query.target_.set_vector(query_vector_str.substr(0, 3));
|
|
s = query.validate(&schema, nullptr);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// sparse query indices count must not exceed the allowed maximum
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 100;
|
|
std::vector<uint32_t> query_indices(16385);
|
|
std::vector<float> query_values(16385);
|
|
query.target_.set_sparse_vector(
|
|
std::string(reinterpret_cast<char *>(query_indices.data()),
|
|
query_indices.size() * sizeof(uint32_t)),
|
|
std::string(reinterpret_cast<char *>(query_values.data()),
|
|
query_values.size() * sizeof(float)));
|
|
FieldSchema schema =
|
|
FieldSchema("field_name", DataType::SPARSE_VECTOR_FP32);
|
|
auto s = query.validate(&schema, nullptr);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
|
|
// one valid index and matching value: accepted
|
|
uint32_t one_index = 0;
|
|
float one_value = 0.0f;
|
|
query.target_.set_sparse_vector(
|
|
std::string(reinterpret_cast<char *>(&one_index), sizeof(uint32_t)),
|
|
std::string(reinterpret_cast<char *>(&one_value), sizeof(float)));
|
|
s = query.validate(&schema, nullptr);
|
|
EXPECT_TRUE(s.ok());
|
|
}
|
|
|
|
// sparse: validate sets need_sanitize for unsorted, sanitize sorts and
|
|
// detects duplicates
|
|
{
|
|
auto pack_idx = [](const std::vector<uint32_t> &v) {
|
|
return std::string(reinterpret_cast<const char *>(v.data()),
|
|
v.size() * sizeof(uint32_t));
|
|
};
|
|
auto pack_val = [](const std::vector<float> &v) {
|
|
return std::string(reinterpret_cast<const char *>(v.data()),
|
|
v.size() * sizeof(float));
|
|
};
|
|
auto decode_idx = [](const std::string &buf) {
|
|
const auto *p = reinterpret_cast<const uint32_t *>(buf.data());
|
|
return std::vector<uint32_t>(p, p + buf.size() / sizeof(uint32_t));
|
|
};
|
|
auto decode_val = [](const std::string &buf) {
|
|
const auto *p = reinterpret_cast<const float *>(buf.data());
|
|
return std::vector<float>(p, p + buf.size() / sizeof(float));
|
|
};
|
|
FieldSchema schema =
|
|
FieldSchema("field_name", DataType::SPARSE_VECTOR_FP32);
|
|
|
|
// unsorted indices: validate sets need_sanitize, sanitize sorts in place
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 100;
|
|
query.target_.set_sparse_vector(pack_idx({42u, 7u, 128u, 3u, 99u}),
|
|
pack_val({0.1f, 0.2f, 0.3f, 0.4f, 0.5f}));
|
|
bool need_sanitize = false;
|
|
auto s = query.validate(&schema, &need_sanitize);
|
|
EXPECT_TRUE(s.ok()) << s.message();
|
|
EXPECT_TRUE(need_sanitize);
|
|
|
|
VectorClause vc = *query.target_.get_vector_clause();
|
|
s = sanitize_sparse_vector(vc, &schema);
|
|
EXPECT_TRUE(s.ok()) << s.message();
|
|
EXPECT_EQ(decode_idx(vc.sparse_indices_),
|
|
(std::vector<uint32_t>{3u, 7u, 42u, 99u, 128u}));
|
|
EXPECT_EQ(decode_val(vc.sparse_values_),
|
|
(std::vector<float>{0.4f, 0.2f, 0.1f, 0.5f, 0.3f}));
|
|
}
|
|
|
|
// duplicates (sorted): validate detects duplicates directly
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 100;
|
|
query.target_.set_sparse_vector(pack_idx({3u, 7u, 42u, 42u, 99u}),
|
|
pack_val({0.1f, 0.2f, 0.3f, 0.4f, 0.5f}));
|
|
bool need_sanitize = false;
|
|
auto s = query.validate(&schema, &need_sanitize);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
EXPECT_FALSE(need_sanitize);
|
|
}
|
|
|
|
// duplicates (unsorted): sanitize sorts then reports duplicates
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 100;
|
|
query.target_.set_sparse_vector(pack_idx({42u, 3u, 7u, 42u, 99u}),
|
|
pack_val({0.1f, 0.2f, 0.3f, 0.4f, 0.5f}));
|
|
bool need_sanitize = false;
|
|
auto s = query.validate(&schema, &need_sanitize);
|
|
EXPECT_TRUE(s.ok());
|
|
EXPECT_TRUE(need_sanitize);
|
|
|
|
VectorClause vc = *query.target_.get_vector_clause();
|
|
s = sanitize_sparse_vector(vc, &schema);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// sorted without duplicates: need_sanitize is false
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 100;
|
|
query.target_.set_sparse_vector(pack_idx({1u, 2u, 3u, 4u}),
|
|
pack_val({0.1f, 0.2f, 0.3f, 0.4f}));
|
|
bool need_sanitize = false;
|
|
auto s = query.validate(&schema, &need_sanitize);
|
|
EXPECT_TRUE(s.ok()) << s.message();
|
|
EXPECT_FALSE(need_sanitize);
|
|
}
|
|
|
|
// mismatched counts are rejected by validate
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 100;
|
|
query.target_.set_sparse_vector(pack_idx({1u, 2u, 3u, 4u}),
|
|
pack_val({0.1f, 0.2f, 0.3f}));
|
|
auto s = query.validate(&schema, nullptr);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
}
|
|
|
|
// query_params type must match the field's index type
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "embedding";
|
|
query.topk_ = 10;
|
|
std::vector<float> query_vector(128, 1.0f);
|
|
query.target_.set_vector(
|
|
std::string(reinterpret_cast<char *>(query_vector.data()),
|
|
query_vector.size() * sizeof(float)));
|
|
FieldSchema schema =
|
|
FieldSchema("embedding", DataType::VECTOR_FP32, 128, false,
|
|
std::make_shared<HnswIndexParams>(MetricType::L2));
|
|
|
|
query.target_.query_params_ = std::make_shared<HnswQueryParams>(150);
|
|
auto s = query.validate(&schema, nullptr);
|
|
EXPECT_TRUE(s.ok());
|
|
|
|
query.target_.query_params_ = std::make_shared<IVFQueryParams>(50);
|
|
s = query.validate(&schema, nullptr);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
|
|
query.target_.query_params_ = nullptr;
|
|
s = query.validate(&schema, nullptr);
|
|
EXPECT_TRUE(s.ok());
|
|
}
|
|
|
|
// FTS clause validation
|
|
{
|
|
auto fts_params = std::make_shared<FtsIndexParams>();
|
|
FieldSchema fts_schema("content", DataType::STRING, false, fts_params);
|
|
|
|
// FTS query with proper FTS field schema -> OK
|
|
SearchQuery fts_only;
|
|
fts_only.target_.field_name_ = "content";
|
|
fts_only.topk_ = 10;
|
|
FtsClause fts_test;
|
|
fts_test.query_string_ = "test";
|
|
fts_only.target_.clause_ = fts_test;
|
|
auto s = fts_only.validate(&fts_schema, nullptr);
|
|
EXPECT_TRUE(s.ok());
|
|
|
|
// FTS query with nullptr schema -> fail (field not found)
|
|
s = fts_only.validate(nullptr, nullptr);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
|
|
// FTS query with vector field schema -> fail (type mismatch)
|
|
FieldSchema vec_schema("embedding", DataType::VECTOR_FP32, 128, false,
|
|
std::make_shared<HnswIndexParams>(MetricType::L2));
|
|
s = fts_only.validate(&vec_schema, nullptr);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// VectorViewClause: validate handles VectorViewClause the same as
|
|
// VectorClause
|
|
{
|
|
FieldSchema schema =
|
|
FieldSchema("field_name", DataType::VECTOR_FP32, 4, true);
|
|
std::vector<float> query_vector = {1.0f, 2.0f, 3.0f, 4.0f};
|
|
std::string vec_data(reinterpret_cast<char *>(query_vector.data()),
|
|
query_vector.size() * sizeof(float));
|
|
|
|
// Dense VectorViewClause: valid dimension
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 10;
|
|
query.target_.clause_ =
|
|
VectorViewClause{vec_data, std::string_view{}, std::string_view{}};
|
|
auto s = query.validate(&schema, nullptr);
|
|
EXPECT_TRUE(s.ok()) << s.message();
|
|
}
|
|
|
|
// Dense VectorViewClause: wrong dimension
|
|
{
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 10;
|
|
std::string short_vec = vec_data.substr(0, sizeof(float) * 2);
|
|
query.target_.clause_ =
|
|
VectorViewClause{short_vec, std::string_view{}, std::string_view{}};
|
|
auto s = query.validate(&schema, nullptr);
|
|
EXPECT_FALSE(s.ok());
|
|
EXPECT_EQ(s.code(), StatusCode::INVALID_ARGUMENT);
|
|
}
|
|
|
|
// Sparse VectorViewClause: unsorted triggers need_sanitize
|
|
{
|
|
FieldSchema sparse_schema(
|
|
"field_name", DataType::SPARSE_VECTOR_FP32, false,
|
|
std::make_shared<HnswIndexParams>(MetricType::IP));
|
|
std::vector<uint32_t> idx_vec = {3u, 1u, 2u};
|
|
std::vector<float> val_vec = {0.3f, 0.1f, 0.2f};
|
|
std::string idx_data(reinterpret_cast<const char *>(idx_vec.data()),
|
|
idx_vec.size() * sizeof(uint32_t));
|
|
std::string val_data(reinterpret_cast<const char *>(val_vec.data()),
|
|
val_vec.size() * sizeof(float));
|
|
SearchQuery query;
|
|
query.target_.field_name_ = "field_name";
|
|
query.topk_ = 10;
|
|
query.target_.clause_ =
|
|
VectorViewClause{std::string_view{}, idx_data, val_data};
|
|
bool need_sanitize = false;
|
|
auto s = query.validate(&sparse_schema, &need_sanitize);
|
|
EXPECT_TRUE(s.ok()) << s.message();
|
|
EXPECT_TRUE(need_sanitize);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test null value
|
|
TEST_F(DocDetailedTest, NullValue) {
|
|
Doc doc;
|
|
|
|
// Test setting null value
|
|
doc.set_null("null_field");
|
|
EXPECT_TRUE(doc.is_null("null_field"));
|
|
EXPECT_FALSE(doc.has_value("null_field"));
|
|
EXPECT_TRUE(doc.has("null_field"));
|
|
|
|
// Test get_field with null field
|
|
auto result = doc.get_field<int32_t>("null_field");
|
|
EXPECT_EQ(result.status(), Doc::FieldGetStatus::IS_NULL);
|
|
EXPECT_FALSE(result.ok());
|
|
|
|
// Test get with null field
|
|
auto opt_result = doc.get<int32_t>("null_field");
|
|
EXPECT_FALSE(opt_result.has_value());
|
|
|
|
// Test overwriting null with actual value
|
|
doc.set<int32_t>("null_field", 42);
|
|
EXPECT_FALSE(doc.is_null("null_field"));
|
|
EXPECT_TRUE(doc.has_value("null_field"));
|
|
EXPECT_TRUE(doc.has("null_field"));
|
|
EXPECT_EQ(doc.get<int32_t>("null_field").value(), 42);
|
|
|
|
// Test overwriting value with null
|
|
doc.set_null("null_field");
|
|
EXPECT_TRUE(doc.is_null("null_field"));
|
|
EXPECT_FALSE(doc.has_value("null_field"));
|
|
EXPECT_TRUE(doc.has("null_field"));
|
|
|
|
// Test serialization/deserialization of null values
|
|
auto buffer = doc.serialize();
|
|
auto deserialized_doc = Doc::deserialize(buffer.data(), buffer.size());
|
|
EXPECT_NE(deserialized_doc, nullptr);
|
|
EXPECT_TRUE(deserialized_doc->is_null("null_field"));
|
|
EXPECT_FALSE(deserialized_doc->has_value("null_field"));
|
|
EXPECT_TRUE(deserialized_doc->has("null_field"));
|
|
}
|
|
|
|
// Test field existence checks
|
|
TEST_F(DocDetailedTest, FieldExistenceChecks) {
|
|
Doc doc;
|
|
|
|
// Test non-existent field
|
|
EXPECT_FALSE(doc.has("nonexistent"));
|
|
EXPECT_FALSE(doc.has_value("nonexistent"));
|
|
EXPECT_FALSE(doc.is_null("nonexistent"));
|
|
|
|
// Test get_field with non-existent field
|
|
auto result = doc.get_field<int32_t>("nonexistent");
|
|
EXPECT_EQ(result.status(), Doc::FieldGetStatus::NOT_FOUND);
|
|
EXPECT_FALSE(result.ok());
|
|
|
|
// Test get with non-existent field
|
|
auto opt_result = doc.get<int32_t>("nonexistent");
|
|
EXPECT_FALSE(opt_result.has_value());
|
|
|
|
// Add a field and test again
|
|
doc.set<int32_t>("existent", 123);
|
|
EXPECT_TRUE(doc.has("existent"));
|
|
EXPECT_TRUE(doc.has_value("existent"));
|
|
EXPECT_FALSE(doc.is_null("existent"));
|
|
|
|
// Test type mismatch
|
|
auto type_mismatch_result = doc.get_field<std::string>("existent");
|
|
EXPECT_EQ(type_mismatch_result.status(), Doc::FieldGetStatus::TYPE_MISMATCH);
|
|
EXPECT_FALSE(type_mismatch_result.ok());
|
|
|
|
auto type_mismatch_opt = doc.get<std::string>("existent");
|
|
EXPECT_FALSE(type_mismatch_opt.has_value());
|
|
} |