wehub/alibaba--zvec
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:47:42 +08:00
commit be3ef883e1
1214 changed files with 431743 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/db/sqlengine/fts_recall_test.cc
+824
View File
@@ -0,0 +1,824 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License
#include <cstdint>
#include <memory>
#include <set>
#include <string>
#include <vector>
#include <gtest/gtest.h>
#include "db/common/file_helper.h"
#include "db/index/common/version_manager.h"
#include "db/index/segment/segment.h"
#include "db/sqlengine/sqlengine.h"
#include "zvec/db/doc.h"
#include "zvec/db/index_params.h"
#include "zvec/db/query_params.h"
#include "zvec/db/schema.h"
#include "zvec/db/type.h"
namespace zvec::sqlengine {
// ============================================================
// FTS Recall Test fixture (real Segment + SQLEngine::execute via SearchQuery)
// ============================================================
class FtsRecallTest : public ::testing::Test {
protected:
static void SetUpTestSuite() {
FileHelper::RemoveDirectory(seg_path_);
FileHelper::CreateDirectory(seg_path_);
build_schema();
auto segment = create_segment();
ASSERT_NE(segment, nullptr);
insert_docs(segment);
segments_.push_back(segment);
engine_ = SQLEngine::create(std::make_shared<Profiler>());
}
static void TearDownTestSuite() {
segments_.clear();
engine_.reset();
schema_.reset();
FileHelper::RemoveDirectory(seg_path_);
}
// Helper: execute FTS query_string search via SearchQuery
Result<DocPtrList> fts_search(const std::string &query_string,
int topk = 10) {
SearchQuery vq;
vq.topk_ = topk;
vq.target_.field_name_ = "content";
FtsClause fts;
fts.query_string_ = query_string;
vq.target_.clause_ = fts;
return engine_->execute(schema_, vq, segments_);
}
// Helper: execute FTS match_string search via SearchQuery
Result<DocPtrList> fts_match(const std::string &match_string,
const std::string &default_op = "",
int topk = 10) {
SearchQuery vq;
vq.topk_ = topk;
vq.target_.field_name_ = "content";
FtsClause fts;
fts.match_string_ = match_string;
vq.target_.clause_ = fts;
if (!default_op.empty()) {
auto fts_qp = std::make_shared<zvec::FtsQueryParams>();
fts_qp->set_default_operator(default_op);
vq.target_.query_params_ = fts_qp;
}
return engine_->execute(schema_, vq, segments_);
}
// Helper: execute FTS query_string with default_operator via SearchQuery
Result<DocPtrList> fts_query_with_op(const std::string &query_string,
const std::string &default_op,
int topk = 10) {
SearchQuery vq;
vq.topk_ = topk;
vq.target_.field_name_ = "content";
FtsClause fts;
fts.query_string_ = query_string;
vq.target_.clause_ = fts;
auto fts_qp = std::make_shared<zvec::FtsQueryParams>();
fts_qp->set_default_operator(default_op);
vq.target_.query_params_ = fts_qp;
return engine_->execute(schema_, vq, segments_);
}
// Helper: execute FTS query_string with WHERE filter via SearchQuery
Result<DocPtrList> fts_search_with_filter(const std::string &query_string,
const std::string &filter,
int topk = 10) {
SearchQuery vq;
vq.topk_ = topk;
vq.target_.field_name_ = "content";
vq.filter_ = filter;
FtsClause fts;
fts.query_string_ = query_string;
vq.target_.clause_ = fts;
return engine_->execute(schema_, vq, segments_);
}
private:
static void build_schema() {
auto fts_params = std::make_shared<FtsIndexParams>(
"whitespace", std::vector<std::string>{"lowercase"}, "");
auto invert_params = std::make_shared<InvertIndexParams>(true);
schema_ = std::make_shared<CollectionSchema>(
"fts_recall_test",
std::vector<FieldSchema::Ptr>{
std::make_shared<FieldSchema>("content", DataType::STRING, false,
fts_params),
std::make_shared<FieldSchema>("tag", DataType::INT32, false,
invert_params),
// Dummy vector field required for filter parsing path in
// execute
std::make_shared<FieldSchema>(
"vec", DataType::VECTOR_FP32, 4, false,
std::make_shared<FlatIndexParams>(MetricType::L2)),
});
}
static Segment::Ptr create_segment() {
auto segment_meta = std::make_shared<SegmentMeta>();
segment_meta->set_id(0);
auto id_map = IDMap::CreateAndOpen("fts_recall_test", seg_path_ + "/id_map",
true, false);
auto delete_store = std::make_shared<DeleteStore>("fts_recall_test");
Version v1;
v1.set_schema(*schema_);
std::string v_path = seg_path_ + "/manifest";
FileHelper::CreateDirectory(v_path);
auto vm = VersionManager::Create(v_path, v1);
if (!vm.has_value()) {
return nullptr;
}
BlockMeta mem_block;
mem_block.id_ = 0;
mem_block.type_ = BlockType::SCALAR;
mem_block.min_doc_id_ = 0;
mem_block.max_doc_id_ = 0;
mem_block.doc_count_ = 0;
segment_meta->set_writing_forward_block(mem_block);
SegmentOptions options;
options.read_only_ = false;
options.enable_mmap_ = true;
options.max_buffer_size_ = 256 * 1024;
auto result = Segment::CreateAndOpen(seg_path_, *schema_, 0, 0, id_map,
delete_store, vm.value(), options);
if (!result) {
return nullptr;
}
return result.value();
}
static void insert_docs(const Segment::Ptr &segment) {
// doc_id 0: "apple banana cherry" tag=1
// doc_id 1: "banana date elderberry" tag=2
// doc_id 2: "cherry fig grape" tag=1
// doc_id 3: "apple fig honeydew" tag=2
// doc_id 4: "date grape kiwi" tag=1
// doc_id 5: "apple apple apple" tag=2
// doc_id 6: "mango papaya starfruit" tag=1
// doc_id 7: "banana banana grape" tag=2
struct Entry {
std::string content;
int32_t tag;
};
std::vector<Entry> entries = {
{"apple banana cherry", 1}, {"banana date elderberry", 2},
{"cherry fig grape", 1}, {"apple fig honeydew", 2},
{"date grape kiwi", 1}, {"apple apple apple", 2},
{"mango papaya starfruit", 1}, {"banana banana grape", 2},
};
for (size_t i = 0; i < entries.size(); ++i) {
Doc doc;
doc.set_pk("pk_" + std::to_string(i));
doc.set_doc_id(i);
doc.set<std::string>("content", entries[i].content);
doc.set<int32_t>("tag", entries[i].tag);
auto status = segment->Insert(doc);
ASSERT_TRUE(status.ok())
<< "Insert doc " << i << " failed: " << status.c_str();
}
}
protected:
static inline std::string seg_path_ = "./fts_recall_test_collection";
static inline CollectionSchema::Ptr schema_;
static inline std::vector<Segment::Ptr> segments_;
static inline SQLEngine::Ptr engine_;
};
// ============================================================
// Basic FTS search tests
// ============================================================
// "apple" matches docs 0, 3, 5
TEST_F(FtsRecallTest, BasicSingleTerm) {
auto result = fts_search("apple");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 3u);
}
// BM25 ordering: doc 5 ("apple apple apple") should have highest score
TEST_F(FtsRecallTest, BM25ScoreOrdering) {
auto result = fts_search("apple");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
ASSERT_GE(result->size(), 2u);
// Results should be sorted by score descending
for (size_t i = 0; i + 1 < result->size(); ++i) {
EXPECT_GE((*result)[i]->score(), (*result)[i + 1]->score())
<< "Results not sorted descending at index " << i;
}
// Doc 5 has highest TF for "apple"
EXPECT_EQ((*result)[0]->pk(), "pk_5");
}
// "kiwi" only in doc 4
TEST_F(FtsRecallTest, SingleMatch) {
auto result = fts_search("kiwi");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
ASSERT_EQ(result->size(), 1u);
EXPECT_EQ((*result)[0]->pk(), "pk_4");
}
// Nonexistent term
TEST_F(FtsRecallTest, NoMatch) {
auto result = fts_search("zzznomatch");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 0u);
}
// Topk limit: "banana" in docs 0, 1, 7 (3 matches), topk=2
TEST_F(FtsRecallTest, TopkLimit) {
auto result = fts_search("banana", /*topk=*/2);
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_LE(result->size(), 2u);
}
// Multi-term implicit OR: "apple banana" matches union of {0,3,5} and {0,1,7}
TEST_F(FtsRecallTest, MultiTermImplicitOr) {
auto result = fts_search("apple banana");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
// Union: {0,1,3,5,7} = 5 docs
EXPECT_EQ(result->size(), 5u);
}
// "starfruit" only in doc 6
TEST_F(FtsRecallTest, RareTerm) {
auto result = fts_search("starfruit");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
ASSERT_EQ(result->size(), 1u);
EXPECT_EQ((*result)[0]->pk(), "pk_6");
}
// "grape" in docs 2, 4, 7
TEST_F(FtsRecallTest, CommonTerm) {
auto result = fts_search("grape");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 3u);
}
// ============================================================
// Explicit AND
// ============================================================
// "apple AND banana" -> intersection of {0,3,5} and {0,1,7} = {0}
TEST_F(FtsRecallTest, ExplicitAnd) {
auto result = fts_search("apple AND banana");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 1u);
EXPECT_EQ((*result)[0]->pk(), "pk_0");
}
// "cherry AND fig" -> {0,2} AND {2,3} = {2}
TEST_F(FtsRecallTest, ExplicitAnd2) {
auto result = fts_search("cherry AND fig");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 1u);
EXPECT_EQ((*result)[0]->pk(), "pk_2");
}
// ============================================================
// Binary NOT (AND-NOT)
// ============================================================
// "apple NOT banana" -> {0,3,5} minus {0,1,7} = {3,5}
TEST_F(FtsRecallTest, BinaryNot) {
auto result = fts_search("apple NOT banana");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 2u);
std::set<std::string> pks;
for (auto &doc : *result) {
pks.insert(doc->pk());
}
EXPECT_TRUE(pks.count("pk_3"));
EXPECT_TRUE(pks.count("pk_5"));
}
// "banana NOT grape" -> {0,1,7} minus {2,4,7} = {0,1}
TEST_F(FtsRecallTest, BinaryNot2) {
auto result = fts_search("banana NOT grape");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 2u);
std::set<std::string> pks;
for (auto &doc : *result) {
pks.insert(doc->pk());
}
EXPECT_TRUE(pks.count("pk_0"));
EXPECT_TRUE(pks.count("pk_1"));
}
// ============================================================
// Error cases
// ============================================================
// Leading NOT should fail parse
TEST_F(FtsRecallTest, LeadingNotIsRejected) {
auto result = fts_search("NOT apple");
EXPECT_FALSE(result.has_value());
}
// Both query_string_ and match_string_ empty
TEST_F(FtsRecallTest, BothEmptyReturnsError) {
SearchQuery vq;
vq.topk_ = 10;
vq.target_.field_name_ = "content";
vq.target_.clause_ = FtsClause{}; // both fields empty
auto result = engine_->execute(schema_, vq, segments_);
EXPECT_FALSE(result.has_value());
}
// Both query_string_ and match_string_ set
TEST_F(FtsRecallTest, BothSetReturnsError) {
SearchQuery vq;
vq.topk_ = 10;
vq.target_.field_name_ = "content";
FtsClause fts;
fts.query_string_ = "apple";
fts.match_string_ = "banana";
vq.target_.clause_ = fts;
auto result = engine_->execute(schema_, vq, segments_);
EXPECT_FALSE(result.has_value());
}
// ============================================================
// match_string tests
// ============================================================
// match_string "starfruit" -> doc 6
TEST_F(FtsRecallTest, MatchStringRareTerm) {
auto result = fts_match("starfruit");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
ASSERT_EQ(result->size(), 1u);
EXPECT_EQ((*result)[0]->pk(), "pk_6");
}
// match_string "grape" -> docs 2, 4, 7
TEST_F(FtsRecallTest, MatchStringCommonTerm) {
auto result = fts_match("grape");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 3u);
}
// match_string "apple banana" -> OR -> union {0,1,3,5,7}
TEST_F(FtsRecallTest, MatchStringMultipleTokens) {
auto result = fts_match("apple banana");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 5u);
}
// match_string analysing to zero tokens → empty result, not an error.
TEST_F(FtsRecallTest, MatchStringEmptyTokensReturnsNoResults) {
auto result = fts_match(" \t ");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_TRUE(result->empty());
}
// ============================================================
// default_operator tests
// ============================================================
// AND default for match_string: "apple banana" -> intersection = {0}
TEST_F(FtsRecallTest, DefaultOperatorAnd_MatchString) {
auto result = fts_match("apple banana", "AND");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 1u);
EXPECT_EQ((*result)[0]->pk(), "pk_0");
}
// OR default for match_string (backward compat)
TEST_F(FtsRecallTest, DefaultOperatorOr_MatchString) {
auto result = fts_match("apple banana", "OR");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 5u);
}
// AND default for query_string: "apple banana" -> AND
TEST_F(FtsRecallTest, DefaultOperatorAnd_QueryString) {
auto result = fts_query_with_op("apple banana", "AND");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 1u);
EXPECT_EQ((*result)[0]->pk(), "pk_0");
}
// Explicit OR in query not overridden by default_operator=AND
// "apple OR grape" with AND default -> OR still applies
TEST_F(FtsRecallTest, DefaultOperatorAnd_DoesNotOverrideExplicitOr) {
auto result = fts_query_with_op("apple OR grape", "AND");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
// apple: {0,3,5}, grape: {2,4,7} -> union = 6
EXPECT_EQ(result->size(), 6u);
}
// Empty default_operator keeps historical OR for match_string
TEST_F(FtsRecallTest, DefaultOperatorEmpty_BackwardCompatibleOr) {
auto result = fts_match("apple banana"); // no default_op arg
ASSERT_TRUE(result.has_value()) << result.error().c_str();
// OR semantics: union of apple{0,3,5} and banana{0,1,7} = 5
EXPECT_EQ(result->size(), 5u);
}
// Lowercase "and" must be accepted
TEST_F(FtsRecallTest, DefaultOperatorAndLowercase_Accepted) {
auto result = fts_match("apple banana", "and");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 1u);
}
// Mixed-case "And" / "oR" are accepted via case-insensitive normalisation.
TEST_F(FtsRecallTest, DefaultOperatorMixedCase_Accepted) {
{
// "And" -> AND semantics: intersection of apple{0,3,5} and banana{0,1,7}
auto result = fts_match("apple banana", "And");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 1u);
}
{
// "oR" -> OR semantics: union = 5 docs
auto result = fts_match("apple banana", "oR");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_EQ(result->size(), 5u);
}
}
// Invalid default_operator value should be rejected (was previously silently
// downgraded to OR).
TEST_F(FtsRecallTest, DefaultOperatorInvalid_Rejected) {
auto result = fts_match("apple banana", "xor");
EXPECT_FALSE(result.has_value());
}
// ============================================================
// Error cases (additional)
// ============================================================
// Empty field_name should fail
TEST_F(FtsRecallTest, EmptyFieldNameReturnsError) {
SearchQuery vq;
vq.topk_ = 10;
vq.target_.field_name_ = "";
FtsClause fts;
fts.query_string_ = "apple";
vq.target_.clause_ = fts;
auto result = engine_->execute(schema_, vq, segments_);
EXPECT_FALSE(result.has_value());
}
// Empty query_string (with field_name set) should fail
TEST_F(FtsRecallTest, EmptyQueryStringReturnsError) {
SearchQuery vq;
vq.topk_ = 10;
vq.target_.field_name_ = "content";
// Both query_string_ and match_string_ empty -> error
vq.target_.clause_ = FtsClause{};
auto result = engine_->execute(schema_, vq, segments_);
EXPECT_FALSE(result.has_value());
}
// ============================================================
// FTS search with WHERE filter
// ============================================================
// "apple" (docs 0,3,5) + tag = 1 (docs 0,2,4,6) -> intersection = {0}
TEST_F(FtsRecallTest, FtsSearchWithFilter_ScoreTag) {
auto result = fts_search_with_filter("apple", "tag = 1");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
// Filter should reduce results to doc 0 only
EXPECT_LE(result->size(), 3u);
// Verify that at least doc 0 (which satisfies both FTS and filter) is present
bool found_pk0 = false;
for (auto &doc : *result) {
if (doc->pk() == "pk_0") {
found_pk0 = true;
}
}
EXPECT_TRUE(found_pk0);
}
// "banana" (docs 0,1,7) + tag = 2 (docs 1,3,5,7) + topk=1
TEST_F(FtsRecallTest, FtsSearchWithFilter_TopkRespected) {
auto result = fts_search_with_filter("banana", "tag = 2", /*topk=*/1);
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_LE(result->size(), 1u);
}
// "apple" matches docs 0,3,5, but no doc has tag=999.
TEST_F(FtsRecallTest, FtsSearchWithFilter_ZeroMatchesReturnsEmpty) {
auto result = fts_search_with_filter("apple", "tag = 999");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_TRUE(result->empty());
}
// An FTS field can only be used as a query target, not as a filter condition.
// Putting the FTS field ("content") in the WHERE filter must be rejected.
TEST_F(FtsRecallTest, FtsFieldNotAllowedInFilter) {
auto result = fts_search_with_filter("apple", "content = 'apple'");
ASSERT_FALSE(result.has_value());
}
// ============================================================
// Repeated-term linearity: the AST rewriter collapses a repeated term into a
// single TermNode whose boost equals the occurrence count. With linear boost
// the per-document score must be exactly N× the single-term score, matching
// the pre-rewrite "N independent scorers summed" semantics.
// ============================================================
TEST_F(FtsRecallTest, MatchStringRepeatedTermLinearBoost) {
auto baseline = fts_match("apple");
auto repeated = fts_match("apple apple");
ASSERT_TRUE(baseline.has_value()) << baseline.error().c_str();
ASSERT_TRUE(repeated.has_value()) << repeated.error().c_str();
ASSERT_EQ(baseline->size(), repeated->size());
// Same doc set, same ordering — only the absolute scores differ.
for (size_t i = 0; i < baseline->size(); ++i) {
EXPECT_EQ((*baseline)[i]->pk(), (*repeated)[i]->pk()) << "rank " << i;
EXPECT_FLOAT_EQ((*baseline)[i]->score() * 2.0f, (*repeated)[i]->score())
<< "rank " << i << " pk=" << (*repeated)[i]->pk();
}
}
// Unary `-` prefix inside an OR was previously executed via build_or_iterator
// wrapping the disjunction in a must_not Conjunction. After the rewriter
// canonicalizes OR-with-must_not into AND(positive..., -negative...), the
// must_not iterator path lives only in build_and_iterator. End-to-end the
// match set must be unchanged: apple{0,3,5} − banana{0,1,7} = {3, 5}.
TEST_F(FtsRecallTest, QueryStringUnaryMinusExcludesMatchingDocs) {
auto result = fts_search("apple -banana");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
std::set<std::string> pks;
for (const auto &d : *result) {
pks.insert(d->pk());
}
EXPECT_EQ(pks, std::set<std::string>({"pk_3", "pk_5"}));
}
// `apple -apple` is a self-contradiction; the rewriter detects the must vs
// must_not conflict after canonicalization and rewrites the whole subtree
// to EmptyNode, so the query short-circuits to zero docs.
TEST_F(FtsRecallTest, QueryStringSelfContradictionReturnsNoResults) {
auto result = fts_search("apple -apple");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_TRUE(result->empty());
}
TEST_F(FtsRecallTest, MatchStringRepeatedTermPreservesUnion) {
// "apple apple banana" — apple repeated, banana once. Doc set must equal
// "apple banana" (union), and apple-only docs should score 2× their
// single-term score plus zero for banana.
auto plain_union = fts_match("apple banana");
auto repeated_union = fts_match("apple apple banana");
ASSERT_TRUE(plain_union.has_value()) << plain_union.error().c_str();
ASSERT_TRUE(repeated_union.has_value()) << repeated_union.error().c_str();
EXPECT_EQ(plain_union->size(), repeated_union->size());
std::set<std::string> plain_pks;
std::set<std::string> repeated_pks;
for (const auto &d : *plain_union) {
plain_pks.insert(d->pk());
}
for (const auto &d : *repeated_union) {
repeated_pks.insert(d->pk());
}
EXPECT_EQ(plain_pks, repeated_pks);
}
// ============================================================
// FTS delete / upsert end-to-end tests (per-test fixture)
// ============================================================
class FtsRecallDeleteTest : public ::testing::Test {
protected:
void SetUp() override {
seg_path_ = "./fts_recall_delete_test_" +
std::to_string(reinterpret_cast<uintptr_t>(this));
FileHelper::RemoveDirectory(seg_path_);
FileHelper::CreateDirectory(seg_path_);
auto fts_params = std::make_shared<FtsIndexParams>(
"whitespace", std::vector<std::string>{"lowercase"}, "");
auto invert_params = std::make_shared<InvertIndexParams>(true);
schema_ = std::make_shared<CollectionSchema>(
"fts_delete_test",
std::vector<FieldSchema::Ptr>{
std::make_shared<FieldSchema>("content", DataType::STRING, false,
fts_params),
std::make_shared<FieldSchema>("tag", DataType::INT32, false,
invert_params),
std::make_shared<FieldSchema>(
"vec", DataType::VECTOR_FP32, 4, false,
std::make_shared<FlatIndexParams>(MetricType::L2)),
});
auto segment_meta = std::make_shared<SegmentMeta>();
segment_meta->set_id(0);
auto id_map = IDMap::CreateAndOpen("fts_delete_test", seg_path_ + "/id_map",
true, false);
auto delete_store = std::make_shared<DeleteStore>("fts_delete_test");
Version v1;
v1.set_schema(*schema_);
std::string v_path = seg_path_ + "/manifest";
FileHelper::CreateDirectory(v_path);
auto vm = VersionManager::Create(v_path, v1);
ASSERT_TRUE(vm.has_value());
BlockMeta mem_block;
mem_block.id_ = 0;
mem_block.type_ = BlockType::SCALAR;
mem_block.min_doc_id_ = 0;
mem_block.max_doc_id_ = 0;
mem_block.doc_count_ = 0;
segment_meta->set_writing_forward_block(mem_block);
SegmentOptions options;
options.read_only_ = false;
options.enable_mmap_ = true;
options.max_buffer_size_ = 256 * 1024;
auto result = Segment::CreateAndOpen(seg_path_, *schema_, 0, 0, id_map,
delete_store, vm.value(), options);
ASSERT_TRUE(result.has_value());
segment_ = result.value();
segments_.push_back(segment_);
engine_ = SQLEngine::create(std::make_shared<Profiler>());
insert_docs();
}
void TearDown() override {
segments_.clear();
segment_.reset();
engine_.reset();
schema_.reset();
FileHelper::RemoveDirectory(seg_path_);
}
void insert_docs() {
// doc_id 0: "apple banana cherry" tag=1
// doc_id 1: "banana date elderberry" tag=2
// doc_id 2: "cherry fig grape" tag=1
// doc_id 3: "apple fig honeydew" tag=2
// doc_id 4: "date grape kiwi" tag=1
struct Entry {
std::string content;
int32_t tag;
};
std::vector<Entry> entries = {
{"apple banana cherry", 1}, {"banana date elderberry", 2},
{"cherry fig grape", 1}, {"apple fig honeydew", 2},
{"date grape kiwi", 1},
};
for (size_t i = 0; i < entries.size(); ++i) {
Doc doc;
doc.set_pk("pk_" + std::to_string(i));
doc.set_doc_id(i);
doc.set<std::string>("content", entries[i].content);
doc.set<int32_t>("tag", entries[i].tag);
auto status = segment_->Insert(doc);
ASSERT_TRUE(status.ok())
<< "Insert doc " << i << " failed: " << status.c_str();
}
}
Result<DocPtrList> fts_search(const std::string &query_string,
int topk = 10) {
SearchQuery vq;
vq.topk_ = topk;
vq.target_.field_name_ = "content";
FtsClause fts;
fts.query_string_ = query_string;
vq.target_.clause_ = fts;
return engine_->execute(schema_, vq, segments_);
}
std::set<std::string> collect_pks(const DocPtrList &docs) {
std::set<std::string> pks;
for (const auto &d : docs) {
pks.insert(d->pk());
}
return pks;
}
std::string seg_path_;
CollectionSchema::Ptr schema_;
Segment::Ptr segment_;
std::vector<Segment::Ptr> segments_;
SQLEngine::Ptr engine_;
};
// Delete doc 0 ("apple banana cherry"), then search "apple":
// before: {0, 3}, after: {3} only.
TEST_F(FtsRecallDeleteTest, DeletedDocExcludedFromSearch) {
auto before = fts_search("apple");
ASSERT_TRUE(before.has_value()) << before.error().c_str();
EXPECT_TRUE(collect_pks(*before).count("pk_0"));
auto s = segment_->Delete("pk_0");
ASSERT_TRUE(s.ok()) << s.c_str();
auto after = fts_search("apple");
ASSERT_TRUE(after.has_value()) << after.error().c_str();
auto pks = collect_pks(*after);
EXPECT_FALSE(pks.count("pk_0"));
EXPECT_TRUE(pks.count("pk_3"));
}
// Delete all docs matching "banana" (0, 1), verify "banana" returns empty.
TEST_F(FtsRecallDeleteTest, DeleteAllMatchingDocsReturnsEmpty) {
auto s1 = segment_->Delete("pk_0");
ASSERT_TRUE(s1.ok()) << s1.c_str();
auto s2 = segment_->Delete("pk_1");
ASSERT_TRUE(s2.ok()) << s2.c_str();
auto result = fts_search("banana");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_TRUE(result->empty());
}
// Upsert doc 0 with new content, verify old content no longer matches
// and new content is searchable.
TEST_F(FtsRecallDeleteTest, UpsertUpdatesSearchableContent) {
// Before: "apple" matches {0, 3}
auto before = fts_search("apple");
ASSERT_TRUE(before.has_value()) << before.error().c_str();
EXPECT_EQ(before->size(), 2u);
// Upsert pk_0 with completely different content
Doc updated;
updated.set_pk("pk_0");
updated.set<std::string>("content", "mango pineapple watermelon");
updated.set<int32_t>("tag", 1);
auto s = segment_->Upsert(updated);
ASSERT_TRUE(s.ok()) << s.c_str();
// "apple" should now only match doc 3
auto after_apple = fts_search("apple");
ASSERT_TRUE(after_apple.has_value()) << after_apple.error().c_str();
ASSERT_EQ(after_apple->size(), 1u);
EXPECT_EQ((*after_apple)[0]->pk(), "pk_3");
// "pineapple" should match the upserted doc
auto after_new = fts_search("pineapple");
ASSERT_TRUE(after_new.has_value()) << after_new.error().c_str();
ASSERT_EQ(after_new->size(), 1u);
EXPECT_EQ((*after_new)[0]->pk(), "pk_0");
}
// Delete a doc, then search with AND: "cherry AND fig" was {2},
// delete doc 2 → empty.
TEST_F(FtsRecallDeleteTest, DeleteAffectsConjunctionQuery) {
auto before = fts_search("cherry AND fig");
ASSERT_TRUE(before.has_value()) << before.error().c_str();
ASSERT_EQ(before->size(), 1u);
EXPECT_EQ((*before)[0]->pk(), "pk_2");
auto s = segment_->Delete("pk_2");
ASSERT_TRUE(s.ok()) << s.c_str();
auto after = fts_search("cherry AND fig");
ASSERT_TRUE(after.has_value()) << after.error().c_str();
EXPECT_TRUE(after->empty());
}
// Delete a doc, flush, then verify deleted doc stays excluded.
TEST_F(FtsRecallDeleteTest, DeletePersistsAcrossFlush) {
auto s = segment_->Delete("pk_4");
ASSERT_TRUE(s.ok()) << s.c_str();
auto flush_s = segment_->flush();
ASSERT_TRUE(flush_s.ok()) << flush_s.c_str();
auto result = fts_search("kiwi");
ASSERT_TRUE(result.has_value()) << result.error().c_str();
EXPECT_TRUE(result->empty());
}
} // namespace zvec::sqlengine