chore: import upstream snapshot with attribution

tests/core/algorithm/vamana/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_vamana core_knn_hnsw core_knn_flat
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm/vamana
+    )
+endforeach()

tests/core/algorithm/vamana/vamana_streamer_test.cc

@@ -0,0 +1,892 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "vamana_streamer.h"
+#include <sys/stat.h>
+#include <sys/types.h>
+#ifndef _MSC_VER
+#include <fcntl.h>
+#include <unistd.h>
+#endif
+#include <future>
+#include <iostream>
+#include <memory>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/vector.h>
+#include "tests/test_util.h"
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+using namespace std;
+using namespace testing;
+using namespace zvec::ailego;
+
+namespace zvec {
+namespace core {
+
+constexpr size_t kDim = 16;
+
+class VamanaStreamerTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+
+  IndexStreamer::Pointer CreateVamanaStreamer(
+      const ailego::Params &extra_params = ailego::Params());
+
+  static std::string dir_;
+  static shared_ptr<IndexMeta> index_meta_ptr_;
+};
+
+std::string VamanaStreamerTest::dir_("vamana_streamer_test_dir/");
+shared_ptr<IndexMeta> VamanaStreamerTest::index_meta_ptr_;
+
+void VamanaStreamerTest::SetUp(void) {
+  index_meta_ptr_.reset(new (nothrow)
+                            IndexMeta(IndexMeta::DataType::DT_FP32, kDim));
+  index_meta_ptr_->set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+void VamanaStreamerTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+IndexStreamer::Pointer VamanaStreamerTest::CreateVamanaStreamer(
+    const ailego::Params &extra_params) {
+  auto streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  if (!streamer) return nullptr;
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 32U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 100U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 500U);
+  params.merge(extra_params);
+
+  if (streamer->init(*index_meta_ptr_, params) != 0) {
+    return nullptr;
+  }
+  return streamer;
+}
+
+TEST_F(VamanaStreamerTest, TestAddVector) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestAddVector", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  for (size_t i = 0; i < 1000UL; i++) {
+    NumericalVector<float> vec(kDim);
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+
+  streamer->flush(0UL);
+  streamer.reset();
+}
+
+TEST_F(VamanaStreamerTest, TestLinearSearch) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t cnt = 5000UL;
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  NumericalVector<float> vec(kDim);
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+
+  size_t topk = 3;
+  for (size_t i = 0; i < cnt; i += 1) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ctx->set_topk(1U);
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(1UL, result1.size());
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i) + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+}
+
+TEST_F(VamanaStreamerTest, TestKnnSearch) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  ailego::Params stg_params;
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearch.index", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  NumericalVector<float> vec(kDim);
+  size_t cnt = 5000U;
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+
+  auto linearCtx = streamer->create_context();
+  auto knnCtx = streamer->create_context();
+  size_t topk = 100;
+  linearCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+  int totalHits = 0;
+  int totalCnts = 0;
+  int topk1Hits = 0;
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i) + 0.1f;
+    }
+    ASSERT_EQ(0, streamer->search_impl(vec.data(), qmeta, knnCtx));
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, linearCtx));
+
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+    topk1Hits += i == knnResult[0].key();
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      totalCnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linearResult[j].key() == knnResult[k].key()) {
+          totalHits++;
+          break;
+        }
+      }
+    }
+  }
+  float recall = totalHits * 1.0f / totalCnts;
+  float topk1Recall = topk1Hits * 1.0f / cnt;
+  EXPECT_GT(recall, 0.90f);
+  EXPECT_GT(topk1Recall, 0.95f);
+}
+
+TEST_F(VamanaStreamerTest, TestOpenClose) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  constexpr size_t dim_large = 128;
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim_large);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 32U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 100U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+
+  streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+  ASSERT_EQ(0, streamer->init(meta, params));
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestOpenClose.index", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t testCnt = 200;
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim_large);
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  for (size_t i = 0; i < testCnt; i++) {
+    std::vector<float> vec(dim_large);
+    for (size_t d = 0; d < dim_large; ++d) {
+      vec[d] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+
+  ASSERT_EQ(0, streamer->flush(0UL));
+  ASSERT_EQ(0, streamer->close());
+
+  // Re-open and verify data
+  ASSERT_EQ(0, streamer->open(storage));
+  auto provider = streamer->create_provider();
+  auto iter = provider->create_iterator();
+  ASSERT_TRUE(!!iter);
+  size_t total = 0;
+  while (iter->is_valid()) {
+    float *data = (float *)iter->data();
+    for (size_t d = 0; d < dim_large; ++d) {
+      ASSERT_FLOAT_EQ(static_cast<float>(iter->key()), data[d]);
+    }
+    total++;
+    iter->next();
+  }
+  ASSERT_EQ(testCnt, total);
+}
+
+TEST_F(VamanaStreamerTest, TestKnnMultiThread) {
+  constexpr size_t dim = 32;
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 64U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 500U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 200U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 1000U);
+  params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 30U * 1024U * 1024U);
+  params.set(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, true);
+
+  auto streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+  ASSERT_EQ(0, streamer->init(meta, params));
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestKnnMultiThread", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto addVector = [&streamer, dim](int baseKey, size_t addCnt) {
+    NumericalVector<float> vec(dim);
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    size_t succAdd = 0;
+    auto ctx = streamer->create_context();
+    for (size_t i = 0; i < addCnt; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i + baseKey);
+      }
+      succAdd += !streamer->add_impl(baseKey + i, vec.data(), qmeta, ctx);
+    }
+    streamer->flush(0UL);
+    return succAdd;
+  };
+  auto t1 = std::async(std::launch::async, addVector, 0, 1000);
+  auto t2 = std::async(std::launch::async, addVector, 1000, 1000);
+  auto t3 = std::async(std::launch::async, addVector, 2000, 1000);
+  ASSERT_EQ(1000U, t1.get());
+  ASSERT_EQ(1000U, t2.get());
+  ASSERT_EQ(1000U, t3.get());
+  streamer->close();
+
+  // Verify data
+  ASSERT_EQ(0, streamer->open(storage));
+  auto provider = streamer->create_provider();
+  auto iter = provider->create_iterator();
+  ASSERT_TRUE(!!iter);
+  size_t total = 0;
+  uint64_t minKey = 10000;
+  uint64_t maxKey = 0;
+  while (iter->is_valid()) {
+    float *data = (float *)iter->data();
+    for (size_t d = 0; d < dim; ++d) {
+      ASSERT_FLOAT_EQ(static_cast<float>(iter->key()), data[d]);
+    }
+    total++;
+    minKey = std::min(minKey, iter->key());
+    maxKey = std::max(maxKey, iter->key());
+    iter->next();
+  }
+  ASSERT_EQ(3000, total);
+  ASSERT_EQ(0, minKey);
+  ASSERT_EQ(2999, maxKey);
+
+  // Multi-thread search
+  size_t topk = 100;
+  size_t cnt = 3000;
+  auto knnSearch = [&]() {
+    NumericalVector<float> vec(dim);
+    auto linearCtx = streamer->create_context();
+    auto knnCtx = streamer->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    linearCtx->set_topk(topk);
+    knnCtx->set_topk(topk);
+    size_t totalCnts = 0;
+    size_t totalHits = 0;
+    for (size_t i = 0; i < cnt; i += 1) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i) + 0.1f;
+      }
+      ASSERT_EQ(0, streamer->search_impl(vec.data(), qmeta, knnCtx));
+      ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, linearCtx));
+      auto &knnResult = knnCtx->result();
+      ASSERT_EQ(topk, knnResult.size());
+      auto &linearResult = linearCtx->result();
+      ASSERT_EQ(topk, linearResult.size());
+      ASSERT_EQ(i, linearResult[0].key());
+      for (size_t k = 0; k < topk; ++k) {
+        totalCnts++;
+        for (size_t j = 0; j < topk; ++j) {
+          if (linearResult[j].key() == knnResult[k].key()) {
+            totalHits++;
+            break;
+          }
+        }
+      }
+    }
+    ASSERT_TRUE((totalHits * 1.0f / totalCnts) > 0.80f);
+  };
+  auto s1 = std::async(std::launch::async, knnSearch);
+  auto s2 = std::async(std::launch::async, knnSearch);
+  auto s3 = std::async(std::launch::async, knnSearch);
+  s1.wait();
+  s2.wait();
+  s3.wait();
+}
+
+TEST_F(VamanaStreamerTest, TestContiguousMemory) {
+  ailego::Params extra;
+  extra.set(PARAM_VAMANA_STREAMER_USE_CONTIGUOUS_MEMORY, true);
+  extra.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 2000U);
+  auto streamer = CreateVamanaStreamer(extra);
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestContiguous.index", true));
+
+  // First build with default mmap mode
+  {
+    auto builder_streamer = CreateVamanaStreamer();
+    ASSERT_NE(nullptr, builder_streamer);
+    ASSERT_EQ(0, builder_streamer->open(storage));
+    auto ctx = builder_streamer->create_context();
+    ASSERT_TRUE(!!ctx);
+
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+    NumericalVector<float> vec(kDim);
+    size_t cnt = 3000UL;
+    for (size_t i = 0; i < cnt; i++) {
+      for (size_t j = 0; j < kDim; ++j) {
+        vec[j] = static_cast<float>(i);
+      }
+      ASSERT_EQ(0, builder_streamer->add_impl(i, vec.data(), qmeta, ctx));
+    }
+    ASSERT_EQ(0, builder_streamer->flush(0UL));
+    ASSERT_EQ(0, builder_streamer->close());
+  }
+
+  // Re-open with contiguous memory mode for search
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t cnt = 3000UL;
+  size_t topk = 50;
+  NumericalVector<float> vec(kDim);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  auto linearCtx = streamer->create_context();
+  auto knnCtx = streamer->create_context();
+  linearCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+  int totalHits = 0;
+  int totalCnts = 0;
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i) + 0.1f;
+    }
+    ASSERT_EQ(0, streamer->search_impl(vec.data(), qmeta, knnCtx));
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, linearCtx));
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+    for (size_t k = 0; k < topk; ++k) {
+      totalCnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linearResult[j].key() == knnResult[k].key()) {
+          totalHits++;
+          break;
+        }
+      }
+    }
+  }
+  float recall = totalHits * 1.0f / totalCnts;
+  EXPECT_GT(recall, 0.90f);
+}
+
+TEST_F(VamanaStreamerTest, TestContiguousMultiThreadSearch) {
+  constexpr size_t dim = 32;
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  // Build with mmap mode
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestContiguousMT", true));
+
+  {
+    ailego::Params build_params;
+    build_params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 64U);
+    build_params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 128U);
+    build_params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+    build_params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+    build_params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 30U * 1024U * 1024U);
+    build_params.set(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, true);
+
+    auto builder = IndexFactory::CreateStreamer("VamanaStreamer");
+    ASSERT_NE(nullptr, builder);
+    ASSERT_EQ(0, builder->init(meta, build_params));
+    ASSERT_EQ(0, builder->open(storage));
+
+    auto ctx = builder->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    NumericalVector<float> vec(dim);
+    for (size_t i = 0; i < 3000; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i);
+      }
+      ASSERT_EQ(0, builder->add_impl(i, vec.data(), qmeta, ctx));
+    }
+    ASSERT_EQ(0, builder->flush(0UL));
+    ASSERT_EQ(0, builder->close());
+  }
+
+  // Re-open with contiguous memory
+  ailego::Params search_params;
+  search_params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 64U);
+  search_params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 128U);
+  search_params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  search_params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  search_params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 30U * 1024U * 1024U);
+  search_params.set(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, true);
+  search_params.set(PARAM_VAMANA_STREAMER_USE_CONTIGUOUS_MEMORY, true);
+
+  auto searcher = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, searcher);
+  ASSERT_EQ(0, searcher->init(meta, search_params));
+  ASSERT_EQ(0, searcher->open(storage));
+
+  size_t topk = 50;
+  size_t cnt = 3000;
+  auto knnSearch = [&]() {
+    NumericalVector<float> vec(dim);
+    auto linearCtx = searcher->create_context();
+    auto knnCtx = searcher->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    linearCtx->set_topk(topk);
+    knnCtx->set_topk(topk);
+    size_t totalCnts = 0;
+    size_t totalHits = 0;
+    for (size_t i = 0; i < cnt; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i) + 0.1f;
+      }
+      ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
+      ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
+      auto &knnResult = knnCtx->result();
+      ASSERT_EQ(topk, knnResult.size());
+      auto &linearResult = linearCtx->result();
+      ASSERT_EQ(topk, linearResult.size());
+      ASSERT_EQ(i, linearResult[0].key());
+      for (size_t k = 0; k < topk; ++k) {
+        totalCnts++;
+        for (size_t j = 0; j < topk; ++j) {
+          if (linearResult[j].key() == knnResult[k].key()) {
+            totalHits++;
+            break;
+          }
+        }
+      }
+    }
+    ASSERT_TRUE((totalHits * 1.0f / totalCnts) > 0.80f);
+  };
+  auto s1 = std::async(std::launch::async, knnSearch);
+  auto s2 = std::async(std::launch::async, knnSearch);
+  auto s3 = std::async(std::launch::async, knnSearch);
+  s1.wait();
+  s2.wait();
+  s3.wait();
+}
+
+TEST_F(VamanaStreamerTest, TestProvider) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestProvider", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t cnt = 500;
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  NumericalVector<float> vec(kDim);
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+  ASSERT_EQ(0, streamer->flush(0UL));
+
+  auto provider = streamer->create_provider();
+  ASSERT_NE(nullptr, provider);
+  auto iter = provider->create_iterator();
+  ASSERT_TRUE(!!iter);
+  size_t total = 0;
+  while (iter->is_valid()) {
+    ASSERT_NE(nullptr, iter->data());
+    float *data = (float *)iter->data();
+    for (size_t d = 0; d < kDim; ++d) {
+      ASSERT_FLOAT_EQ(static_cast<float>(iter->key()), data[d]);
+    }
+    total++;
+    iter->next();
+  }
+  ASSERT_EQ(cnt, total);
+}
+
+TEST_F(VamanaStreamerTest, TestAddAndSearch) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearch.index", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  NumericalVector<float> vec(kDim);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  // Add and search interleaved
+  for (size_t batch = 0; batch < 5; batch++) {
+    size_t base = batch * 200;
+    for (size_t i = 0; i < 200; i++) {
+      for (size_t j = 0; j < kDim; ++j) {
+        vec[j] = static_cast<float>(base + i);
+      }
+      ASSERT_EQ(0, streamer->add_impl(base + i, vec.data(), qmeta, ctx));
+    }
+
+    // Search for recently added vectors
+    size_t current_cnt = (batch + 1) * 200;
+    size_t topk = std::min(current_cnt, (size_t)10);
+    auto searchCtx = streamer->create_context();
+    searchCtx->set_topk(topk);
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(base);
+    }
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, searchCtx));
+    auto &result = searchCtx->result();
+    ASSERT_EQ(topk, result.size());
+    ASSERT_EQ(base, result[0].key());
+  }
+}
+
+TEST_F(VamanaStreamerTest, TestKnnConcurrentAddAndSearch) {
+  constexpr size_t dim = 32;
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 64U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 128U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 500U);
+  params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 30U * 1024U * 1024U);
+  params.set(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, true);
+
+  auto streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+  ASSERT_EQ(0, streamer->init(meta, params));
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestConcurrentAddSearch", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  // First add some base data
+  {
+    auto ctx = streamer->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    NumericalVector<float> vec(dim);
+    for (size_t i = 0; i < 2000; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i);
+      }
+      ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+    }
+  }
+
+  std::atomic<bool> stop_search{false};
+
+  // Concurrent add
+  auto addFuture = std::async(std::launch::async, [&]() {
+    auto ctx = streamer->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    NumericalVector<float> vec(dim);
+    for (size_t i = 2000; i < 3000; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i);
+      }
+      streamer->add_impl(i, vec.data(), qmeta, ctx);
+    }
+    stop_search.store(true);
+  });
+
+  // Concurrent search
+  auto searchFuture = std::async(std::launch::async, [&]() {
+    auto ctx = streamer->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    NumericalVector<float> vec(dim);
+    ctx->set_topk(10);
+    while (!stop_search.load()) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = 100.1f;
+      }
+      int ret = streamer->search_impl(vec.data(), qmeta, ctx);
+      ASSERT_EQ(0, ret);
+      auto &result = ctx->result();
+      ASSERT_GT(result.size(), 0UL);
+    }
+  });
+
+  addFuture.wait();
+  searchFuture.wait();
+}
+
+// Test concurrent build (parallel add_impl) which was crashing due to
+// unprotected node_chunks_ / node_chunk_bases_ access during chunk allocation.
+TEST_F(VamanaStreamerTest, TestConcurrentBuild) {
+  constexpr size_t dim = kDim;
+  constexpr size_t total_vectors = 5000;
+  constexpr size_t thread_count = 4;
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 32U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 100U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 500U);
+  params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 50U * 1024U * 1024U);
+
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  auto streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+  ASSERT_EQ(0, streamer->init(meta, params));
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestConcurrentBuild", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  // Parallel insertion from multiple threads (mimics local_builder behavior)
+  std::atomic<int> error_count{0};
+  std::vector<std::future<void>> futures;
+
+  for (size_t t = 0; t < thread_count; ++t) {
+    futures.push_back(std::async(std::launch::async, [&, t]() {
+      auto ctx = streamer->create_context();
+      ASSERT_TRUE(!!ctx);
+      IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+      NumericalVector<float> vec(dim);
+
+      for (size_t i = t; i < total_vectors; i += thread_count) {
+        for (size_t j = 0; j < dim; ++j) {
+          vec[j] = static_cast<float>(i) + static_cast<float>(j) * 0.01f;
+        }
+        int ret = streamer->add_impl(i, vec.data(), qmeta, ctx);
+        if (ret != 0) {
+          error_count.fetch_add(1);
+          return;
+        }
+      }
+    }));
+  }
+
+  for (auto &f : futures) {
+    f.wait();
+  }
+  ASSERT_EQ(0, error_count.load());
+
+  // Verify search still works correctly after concurrent build
+  auto search_ctx = streamer->create_context();
+  ASSERT_TRUE(!!search_ctx);
+  search_ctx->set_topk(1);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 0.0f;
+  }
+  ASSERT_EQ(0, streamer->search_impl(vec.data(), qmeta, search_ctx));
+  auto &result = search_ctx->result();
+  ASSERT_GT(result.size(), 0UL);
+}
+
+// Test Vamana + INT8 quantization + rotation end-to-end
+TEST_F(VamanaStreamerTest, TestInt8WithRotate) {
+  constexpr size_t kTestDim = 128;
+  constexpr size_t kCnt = 2000U;
+  constexpr size_t kTopk = 10;
+
+  IndexStreamer::Pointer streamer =
+      IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+
+  Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 32U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 100U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 500U);
+
+  IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, kTestDim);
+  index_meta_raw.set_metric("SquaredEuclidean", 0, Params());
+
+  // Create INT8 converter with rotation enabled
+  Params converter_params;
+  converter_params.set("integer_streaming.converter.enable_rotate", true);
+  auto converter = IndexFactory::CreateConverter("Int8StreamingConverter");
+  ASSERT_NE(nullptr, converter);
+  ASSERT_EQ(0, converter->init(index_meta_raw, converter_params));
+
+  IndexMeta index_meta = converter->meta();
+
+  auto reformer = IndexFactory::CreateReformer(index_meta.reformer_name());
+  ASSERT_NE(nullptr, reformer);
+  ASSERT_EQ(0, reformer->init(index_meta.reformer_params()));
+
+  Params stg_params;
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestInt8WithRotate.index", true));
+  ASSERT_EQ(0, streamer->init(index_meta, params));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  // Add 2000 vectors
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kTestDim);
+
+  std::mt19937 gen(42);
+  std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
+  for (size_t i = 0; i < kCnt; i++) {
+    NumericalVector<float> vec(kTestDim);
+    for (size_t j = 0; j < kTestDim; ++j) vec[j] = dist(gen);
+
+    std::string new_vec;
+    IndexQueryMeta new_meta;
+    ASSERT_EQ(0, reformer->convert(vec.data(), qmeta, &new_vec, &new_meta));
+    ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx));
+  }
+
+  streamer->flush(0UL);
+  streamer.reset();
+  storage.reset();
+
+  // Reopen: reformer should auto-detect rotator from storage
+  auto storage2 = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage2);
+  ASSERT_EQ(0, storage2->init(stg_params));
+  ASSERT_EQ(0, storage2->open(dir_ + "TestInt8WithRotate.index", false));
+
+  auto streamer2 = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer2);
+  ASSERT_EQ(0, streamer2->init(index_meta, params));
+  ASSERT_EQ(0, streamer2->open(storage2));
+
+  auto reformer2 = IndexFactory::CreateReformer(index_meta.reformer_name());
+  ASSERT_NE(nullptr, reformer2);
+  ASSERT_EQ(0, reformer2->init(index_meta.reformer_params()));
+  ASSERT_EQ(0, reformer2->load(storage2));
+
+  // Search: verify knn results are non-empty
+  auto knnCtx = streamer2->create_context();
+  knnCtx->set_topk(kTopk);
+  auto linearCtx = streamer2->create_context();
+  linearCtx->set_topk(kTopk);
+
+  NumericalVector<float> query(kTestDim);
+  for (size_t j = 0; j < kTestDim; ++j) query[j] = dist(gen);
+
+  std::string new_query;
+  IndexQueryMeta new_qmeta;
+  ASSERT_EQ(0,
+            reformer2->transform(query.data(), qmeta, &new_query, &new_qmeta));
+  ASSERT_EQ(0, streamer2->search_impl(new_query.data(), new_qmeta, knnCtx));
+  ASSERT_EQ(0,
+            streamer2->search_bf_impl(new_query.data(), new_qmeta, linearCtx));
+
+  EXPECT_EQ(kTopk, knnCtx->result().size());
+  EXPECT_EQ(kTopk, linearCtx->result().size());
+}
+
+}  // namespace core
+}  // namespace zvec
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif