chore: import upstream snapshot with attribution

tests/core/CMakeLists.txt

@@ -0,0 +1,9 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
+
+cc_directories(algorithm)
+cc_directories(framework)
+cc_directories(metric)
+cc_directories(utility)
+cc_directories(interface)
+cc_directories(quantizer)

tests/core/algorithm/CMakeLists.txt

@@ -0,0 +1,21 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
+
+cc_directories(cluster)
+cc_directories(flat)
+cc_directories(flat_sparse)
+cc_directories(ivf)
+cc_directories(hnsw)
+cc_directories(hnsw_sparse)
+cc_directories(vamana)
+
+if(DISKANN_SUPPORTED)
+  message(STATUS "build diskann tests")
+  cc_directory(diskann)
+else()
+  message(STATUS "skip diskann tests (unsupported platform)")
+endif()
+
+if(RABITQ_SUPPORTED)
+  cc_directories(hnsw_rabitq)
+endif()

tests/core/algorithm/cluster/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_knn_cluster
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
+    )
+endforeach()

tests/core/algorithm/cluster/kmeans_cluster_test.cc

@@ -0,0 +1,106 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <random>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/params.h>
+#include "zvec/core/framework/index_framework.h"
+#include "zvec/core/framework/index_meta.h"
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+
+TEST(KmeansCluster, General) {
+  // Prepare index data
+  const uint32_t count = 5000u;
+  const uint32_t dimension = 33u;
+
+  IndexMeta index_meta;
+  index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
+  index_meta.set_metric("SquaredEuclidean", 0, zvec::ailego::Params());
+
+  std::shared_ptr<CompactIndexFeatures> features(
+      new CompactIndexFeatures(index_meta));
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(0.0, 5.0);
+
+  for (uint32_t i = 0; i < count; ++i) {
+    std::vector<float> vec(dimension);
+    for (size_t j = 0; j < dimension; ++j) {
+      vec[j] = dist(gen);
+    }
+    features->emplace(vec.data());
+  }
+
+  // Create a Kmeans cluster
+  // IndexCluster::Pointer cluster = std::make_shared<KmeansCluster>();
+  IndexCluster::Pointer cluster = IndexFactory::CreateCluster("KmeansCluster");
+  ASSERT_TRUE(!!cluster);
+
+  zvec::ailego::Params params;
+  params.set("zvec.general.cluster.count", 1);
+  params.set("zvec.kmeans.cluster.count", 56);
+
+  ASSERT_EQ(0, cluster->init(index_meta, params));
+  ASSERT_EQ(0, cluster->mount(features));
+  cluster->suggest(64u);
+
+  auto threads = std::make_shared<SingleQueueIndexThreads>();
+
+  std::cout << "---------- FIRST ----------\n";
+  std::vector<IndexCluster::Centroid> centroids;
+  std::vector<uint32_t> labels;
+  ASSERT_NE(0, cluster->classify(threads, centroids));
+  ASSERT_NE(0, cluster->label(threads, centroids, &labels));
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  std::cout << "---------- SECOND ----------\n";
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  std::cout << "---------- THIRD ----------\n";
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  ASSERT_EQ(0, cluster->classify(threads, centroids));
+  ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
+}

tests/core/algorithm/cluster/multi_chunk_cluster_test.cc

@@ -0,0 +1,221 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cluster/multi_chunk_cluster.h"
+#include <cmath>
+#include <random>
+#include <ailego/algorithm/kmeans.h>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/params.h>
+#include "zvec/core/framework/index_framework.h"
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace zvec::ailego;
+TEST(MultiChunkCluster, General) {
+  const uint32_t count = 10000u;
+  const uint32_t dimension = 960u;
+  const uint32_t chunk_count = 480u;
+  const uint32_t cluster_count = 256u;
+  // const uint32_t thread_count = 4;
+  const uint32_t thread_count = 16;
+
+  IndexMeta index_meta;
+
+  index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
+  index_meta.set_metric("SquaredEuclidean", 0, Params());
+
+  std::shared_ptr<CompactIndexFeatures> features(
+      new CompactIndexFeatures(index_meta));
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(0.0, 5.0);
+
+  for (uint32_t i = 0; i < count; ++i) {
+    std::vector<float> vec(dimension);
+    for (size_t j = 0; j < dimension; ++j) {
+      vec[j] = dist(gen);
+    }
+    features->emplace(vec.data());
+  }
+
+  // Create a Kmeans cluster
+  MultiChunkCluster cluster = MultiChunkCluster();
+
+  Params params;
+  params.set("zvec.cluster.multi_chunk_cluster.count", cluster_count);
+  params.set("zvec.cluster.multi_chunk_cluster.chunk_count", chunk_count);
+  params.set("zvec.cluster.multi_chunk_cluster.thread_count", thread_count);
+
+  ASSERT_EQ(0, cluster.init(index_meta, params));
+  ASSERT_EQ(0, cluster.mount(features));
+
+  IndexCluster::CentroidList centroids;
+  std::vector<uint32_t> labels;
+
+  ASSERT_EQ(0, cluster.cluster(nullptr, centroids));
+
+  for (size_t chunk = 0; chunk < chunk_count; ++chunk) {
+    for (size_t cluster = 0; cluster < cluster_count; ++cluster) {
+      size_t idx = chunk * cluster_count + cluster;
+      const auto &cent = centroids[idx];
+      const auto &vec = cent.vector<float>();
+
+      std::cout << "chunk: " << chunk << ", cluster: " << cluster
+                << ", dim: " << vec.size() << ", count: " << cent.follows()
+                << " (" << cent.score() << ") { " << vec[0] << "," << vec[1]
+                << " }" << std::endl;
+      ASSERT_EQ(0u, cent.similars().size());
+    }
+  }
+
+  ASSERT_EQ(0, cluster.label(nullptr, centroids, &labels));
+}
+
+TEST(MultiChunkCluster, TestChunk) {
+  const uint32_t count = 10000u;
+  const uint32_t dimension = 95;
+  const uint32_t chunk_count = 20u;
+  const uint32_t cluster_count = 256u;
+  // const uint32_t thread_count = 4;
+  const uint32_t thread_count = 16;
+
+  IndexMeta index_meta;
+
+  index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
+  index_meta.set_metric("SquaredEuclidean", 0, Params());
+
+  std::shared_ptr<CompactIndexFeatures> features(
+      new CompactIndexFeatures(index_meta));
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(0.0, 5.0);
+
+  for (uint32_t i = 0; i < count; ++i) {
+    std::vector<float> vec(dimension);
+    for (size_t j = 0; j < dimension; ++j) {
+      vec[j] = dist(gen);
+    }
+    features->emplace(vec.data());
+  }
+
+  // Create a Kmeans cluster
+  MultiChunkCluster cluster = MultiChunkCluster();
+
+  Params params;
+  params.set("zvec.cluster.multi_chunk_cluster.count", cluster_count);
+  params.set("zvec.cluster.multi_chunk_cluster.chunk_count", chunk_count);
+  params.set("zvec.cluster.multi_chunk_cluster.thread_count", thread_count);
+
+  ASSERT_EQ(0, cluster.init(index_meta, params));
+  ASSERT_EQ(0, cluster.mount(features));
+
+  IndexCluster::CentroidList centroids;
+  std::vector<uint32_t> labels;
+
+  ASSERT_EQ(0, cluster.cluster(nullptr, centroids));
+
+  for (size_t chunk = 0; chunk < chunk_count; ++chunk) {
+    for (size_t cluster = 0; cluster < cluster_count; ++cluster) {
+      size_t idx = chunk * cluster_count + cluster;
+      const auto &cent = centroids[idx];
+      const auto &vec = cent.vector<float>();
+
+      std::cout << "chunk: " << chunk << ", cluster: " << cluster
+                << ", dim: " << vec.size() << ", count: " << cent.follows()
+                << " (" << cent.score() << ") { " << vec[0] << "," << vec[1]
+                << " }" << std::endl;
+      ASSERT_EQ(0u, cent.similars().size());
+    }
+  }
+
+  ASSERT_EQ(0, cluster.label(nullptr, centroids, &labels));
+}
+
+TEST(MultiChunkCluster, General_InnerProduct) {
+  const uint32_t count = 50000u;
+  const uint32_t dimension = 96u;
+  const uint32_t chunk_count = 12u;
+  const uint32_t cluster_count = 16u;
+  const uint32_t chain_length = 0;
+  const uint32_t thread_count = 16;
+
+  IndexMeta index_meta;
+
+  index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
+  index_meta.set_metric("InnerProduct", 0, Params());
+
+  std::shared_ptr<CompactIndexFeatures> features(
+      new CompactIndexFeatures(index_meta));
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(-1.0, 1.0);
+
+  // do normalize
+  for (uint32_t i = 0; i < count; ++i) {
+    std::vector<float> vec(dimension);
+
+    float norm = 0;
+    for (size_t j = 0; j < dimension; ++j) {
+      vec[j] = dist(gen);
+
+      norm += vec[j] * vec[j];
+    }
+    norm = sqrt(norm);
+
+    for (size_t j = 0; j < dimension; ++j) {
+      vec[j] /= norm;
+    }
+
+    features->emplace(vec.data());
+  }
+
+  // Create a Kmeans cluster
+  MultiChunkCluster cluster = MultiChunkCluster();
+
+  Params params;
+  params.set("zvec.cluster.multi_chunk_cluster.count", cluster_count);
+  params.set("zvec.cluster.multi_chunk_cluster.chunk_count", chunk_count);
+  params.set("zvec.cluster.multi_chunk_cluster.thread_count", thread_count);
+  params.set("zvec.cluster.multi_chunk_cluster.markov_chain_length",
+             chain_length);
+
+  ASSERT_EQ(0, cluster.init(index_meta, params));
+  ASSERT_EQ(0, cluster.mount(features));
+
+  IndexCluster::CentroidList centroids;
+  std::vector<uint32_t> labels;
+
+  ASSERT_EQ(0, cluster.cluster(nullptr, centroids));
+
+  for (size_t chunk = 0; chunk < chunk_count; ++chunk) {
+    for (size_t cluster = 0; cluster < cluster_count; ++cluster) {
+      size_t idx = chunk * cluster_count + cluster;
+      const auto &cent = centroids[idx];
+      const auto &vec = cent.vector<float>();
+
+      std::cout << "chunk: " << chunk << ", cluster: " << cluster
+                << ", dim: " << vec.size() << ", count: " << cent.follows()
+                << " (" << cent.score() << ") { " << vec[0] << ", " << vec[1]
+                << ", " << vec[2] << ", ... , " << vec[vec.size() - 2] << ", "
+                << vec[vec.size() - 1] << " }" << std::endl;
+      ASSERT_EQ(0u, cent.similars().size());
+    }
+  }
+
+  ASSERT_EQ(0, cluster.label(nullptr, centroids, &labels));
+}

tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc

@@ -0,0 +1,502 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <random>
+#include <ailego/algorithm/kmeans.h>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/params.h>
+#include "zvec/core/framework/index_framework.h"
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace zvec::ailego;
+
+TEST(OptKmeansCluster, General) {
+  // Prepare index data
+  const uint32_t count = 5000u;
+  const uint32_t dimension = 33u;
+
+  IndexMeta index_meta;
+  index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
+
+  std::shared_ptr<CompactIndexFeatures> features(
+      new CompactIndexFeatures(index_meta));
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(0.0, 5.0);
+
+  for (uint32_t i = 0; i < count; ++i) {
+    std::vector<float> vec(dimension);
+    for (size_t j = 0; j < dimension; ++j) {
+      vec[j] = dist(gen);
+    }
+    features->emplace(vec.data());
+  }
+
+  // Create a Kmeans cluster
+  IndexCluster::Pointer cluster =
+      IndexFactory::CreateCluster("OptKmeansCluster");
+  ASSERT_TRUE(!!cluster);
+
+  Params params;
+  params.set("zvec.general.cluster.count", 1);
+  params.set("zvec.optkmeans.cluster.count", 56);
+
+  ASSERT_EQ(0, cluster->init(index_meta, params));
+  ASSERT_EQ(0, cluster->mount(features));
+  cluster->suggest(64u);
+
+  auto threads = std::make_shared<SingleQueueIndexThreads>();
+
+  std::cout << "---------- FIRST ----------\n";
+  std::vector<IndexCluster::Centroid> centroids;
+  std::vector<uint32_t> labels;
+  ASSERT_NE(0, cluster->classify(threads, centroids));
+  ASSERT_NE(0, cluster->label(threads, centroids, &labels));
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  std::cout << "---------- SECOND ----------\n";
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  std::cout << "---------- THIRD ----------\n";
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  ASSERT_EQ(0, cluster->classify(threads, centroids));
+  ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
+}
+
+// TEST(OptKmeansCluster, NoEmptyCentroids) {
+//   // Prepare index data
+//   const uint32_t count = 500u;
+//   const uint32_t dimension = 8u;
+
+//   IndexMeta index_meta;
+//   index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
+//   index_meta.set_metric("SquaredEuclidean", 0, Params());
+
+//   std::shared_ptr<CompactIndexFeatures> features(
+//       new CompactIndexFeatures(index_meta));
+
+//   std::random_device rd;
+//   std::mt19937 gen(rd());
+//   std::uniform_real_distribution<float> dist(0.0, 5.0);
+
+//   for (uint32_t i = 0; i < count; ++i) {
+//     std::vector<float> vec(dimension);
+//     for (size_t j = 0; j < dimension; ++j) {
+//       vec[j] = dist(gen);
+//     }
+//     features->emplace(vec.data());
+//   }
+
+//   // Create a Kmeans cluster
+//   IndexCluster::Pointer cluster =
+//       IndexFactory::CreateCluster("OptKmeansCluster");
+//   ASSERT_TRUE(!!cluster);
+
+//   Params params;
+//   ASSERT_EQ(0, cluster->init(index_meta, params));
+//   ASSERT_EQ(0, cluster->mount(features));
+//   cluster->suggest(20u);
+
+//   auto threads = std::make_shared<SingleQueueIndexThreads>();
+//   std::vector<IndexCluster::Centroid> centroids;
+//   for (uint32_t i = 0; i < 3; ++i) {
+//     std::vector<float> vec(dimension);
+//     for (size_t j = 0; j < dimension; ++j) {
+//       vec[j] = NAN;
+//     }
+//     centroids.emplace_back(vec.data(), vec.size() * sizeof(float));
+//   }
+//   ASSERT_EQ(0, cluster->cluster(threads, centroids));
+//   ASSERT_EQ(3u, centroids.size());
+
+//   for (uint32_t i = 0; i < 3; ++i) {
+//     std::vector<float> vec(dimension);
+//     for (size_t j = 0; j < dimension; ++j) {
+//       vec[j] = dist(gen);
+//     }
+//     centroids.emplace_back(vec.data(), vec.size() * sizeof(float));
+//   }
+//   ASSERT_EQ(0, cluster->cluster(threads, centroids));
+//   ASSERT_EQ(6u, centroids.size());
+
+//   for (uint32_t i = 0; i < 3; ++i) {
+//     std::vector<float> vec(dimension);
+//     for (size_t j = 0; j < dimension; ++j) {
+//       vec[j] = NAN;
+//     }
+//     centroids.emplace_back(vec.data(), vec.size() * sizeof(float));
+//   }
+//   ASSERT_EQ(0, cluster->cluster(threads, centroids));
+//   ASSERT_EQ(9u, centroids.size());
+
+//   for (uint32_t i = 0; i < 3; ++i) {
+//     std::vector<float> vec(dimension);
+//     for (size_t j = 0; j < dimension; ++j) {
+//       vec[j] = dist(gen);
+//     }
+//     centroids.emplace_back(vec.data(), vec.size() * sizeof(float));
+//   }
+//   ASSERT_EQ(0, cluster->cluster(threads, centroids));
+//   ASSERT_EQ(12u, centroids.size());
+
+//   for (const auto &it : centroids) {
+//     const auto &vec = it.vector<float>();
+
+//     std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ",
+//     "
+//               << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() -
+//               2]
+//               << ", " << vec[vec.size() - 1] << " }" << std::endl;
+//   }
+
+//   params.set("zvec.optkmeans.cluster.purge_empty", true);
+//   cluster->update(params);
+
+//   ASSERT_EQ(12u, centroids.size());
+//   ASSERT_EQ(0, cluster->cluster(threads, centroids));
+//   ASSERT_EQ(7u, centroids.size());
+//   for (const auto &it : centroids) {
+//     const auto &vec = it.vector<float>();
+
+//     std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ",
+//     "
+//               << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() -
+//               2]
+//               << ", " << vec[vec.size() - 1] << " }" << std::endl;
+//   }
+// }
+
+TEST(OptKmeansCluster, IN4General) {
+  // Prepare index data
+  const uint32_t count = 5000u;
+  const uint32_t dimension = 64u;
+  const uint32_t dimension_wrong = 66u;
+
+  IndexMeta index_meta;
+  index_meta.set_meta(IndexMeta::DataType::DT_INT4, dimension);
+  index_meta.set_metric("SquaredEuclidean", 0, Params());
+
+  IndexMeta index_meta_wrong;
+  index_meta_wrong.set_meta(IndexMeta::DataType::DT_INT4, dimension_wrong);
+  index_meta_wrong.set_metric("SquaredEuclidean", 0, Params());
+
+  std::shared_ptr<CompactIndexFeatures> features(
+      new CompactIndexFeatures(index_meta));
+
+  std::shared_ptr<CompactIndexFeatures> features_wrong(
+      new CompactIndexFeatures(index_meta_wrong));
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_int_distribution<unsigned short> dist(0, UINT8_MAX);
+
+  for (uint32_t i = 0; i < count; ++i) {
+    std::vector<uint8_t> vec(dimension / 2);
+    std::vector<uint8_t> vec_wrong(dimension_wrong / 2);
+    for (size_t j = 0; j < dimension / 2; ++j) {
+      vec[j] = dist(gen);
+    }
+    for (size_t j = 0; j < dimension_wrong / 2; ++j) {
+      vec_wrong[j] = dist(gen);
+    }
+    features->emplace(vec.data());
+    features_wrong->emplace(vec_wrong.data());
+  }
+
+  // Create a OptKmeans cluster
+  IndexCluster::Pointer cluster =
+      IndexFactory::CreateCluster("OptKmeansCluster");
+  ASSERT_TRUE(!!cluster);
+
+  Params params;
+  ASSERT_EQ(0, cluster->init(index_meta_wrong, params));
+  ASSERT_NE(0, cluster->mount(features_wrong));
+
+  params.set("zvec.general.cluster.count", 1);
+  params.set("zvec.optkmeans.cluster.count", 56);
+
+  ASSERT_EQ(0, cluster->init(index_meta, params));
+  ASSERT_EQ(0, cluster->mount(features));
+  cluster->suggest(64u);
+
+  auto threads = std::make_shared<SingleQueueIndexThreads>();
+
+  std::cout << "---------- FIRST ----------\n";
+  std::vector<IndexCluster::Centroid> centroids;
+  std::vector<uint32_t> labels;
+  ASSERT_NE(0, cluster->classify(threads, centroids));
+  ASSERT_NE(0, cluster->label(threads, centroids, &labels));
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  std::cout << "---------- SECOND ----------\n";
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  std::cout << "---------- THIRD ----------\n";
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  ASSERT_EQ(0, cluster->classify(threads, centroids));
+  ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
+}
+
+
+TEST(OptKmeansCluster, IN4Correctness) {
+  // Prepare index data
+  const uint32_t count = 5000u;
+  const uint32_t dimension = 64u;
+
+  IndexMeta index_meta1;
+  index_meta1.set_meta(IndexMeta::DataType::DT_INT8, dimension);
+  index_meta1.set_metric("SquaredEuclidean", 0, Params());
+
+  IndexMeta index_meta2;
+  index_meta2.set_meta(IndexMeta::DataType::DT_INT4, dimension);
+  index_meta2.set_metric("SquaredEuclidean", 0, Params());
+
+  std::shared_ptr<CompactIndexFeatures> features1(
+      new CompactIndexFeatures(index_meta1));
+
+  std::shared_ptr<CompactIndexFeatures> features2(
+      new CompactIndexFeatures(index_meta2));
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_int_distribution<int> dist(-8, 7);
+
+  // Generate features
+  for (size_t i = 0; i < count; ++i) {
+    NumericalVector<int8_t> vec1(dimension);
+    NibbleVector<int32_t> vec2(dimension);
+
+    for (size_t j = 0; j < dimension; ++j) {
+      int8_t val = (int8_t)dist(gen);
+      vec1[j] = val;
+      vec2.set(j, val);
+    }
+    features1->emplace(vec1.data());
+    features2->emplace(vec2.data());
+  }
+
+  // Create a OptKmeans cluster of int8, and cluster only once
+  IndexCluster::Pointer cluster_once =
+      IndexFactory::CreateCluster("OptKmeansCluster");
+  ASSERT_TRUE(!!cluster_once);
+
+  Params params_once;
+  params_once.set("zvec.general.cluster.count", 65);
+  params_once.set("zvec.optkmeans.cluster.count", 63);
+  params_once.set("zvec.optkmeans.cluster.max_iterations", 1);
+  // Use KMC2 to init centroids
+  params_once.set("zvec.optkmeans.cluster.markov_chain_length", 20);
+
+  ASSERT_EQ(0, cluster_once->init(index_meta1, params_once));
+  ASSERT_EQ(0, cluster_once->mount(features1));
+  cluster_once->suggest(63);
+
+  auto threads = std::make_shared<SingleQueueIndexThreads>();
+
+  // Cluster once and get centroids
+  std::vector<IndexCluster::Centroid> centroids1;
+  ASSERT_EQ(0, cluster_once->cluster(threads, centroids1));
+
+  // Use centroids_one as init centroids to both int8 and int4 cluster
+  // Create a int8 cluster
+  IndexCluster::Pointer cluster_int8 =
+      IndexFactory::CreateCluster("OptKmeansCluster");
+  ASSERT_TRUE(!!cluster_int8);
+
+  Params params_int8;
+  params_int8.set("zvec.general.cluster.count", 65);
+  params_int8.set("zvec.optkmeans.cluster.count", 63);
+
+  ASSERT_EQ(0, cluster_int8->init(index_meta1, params_int8));
+  ASSERT_EQ(0, cluster_int8->mount(features1));
+  cluster_int8->suggest(63u);
+
+  // Create a int4 cluster
+  IndexCluster::Pointer cluster_int4 =
+      IndexFactory::CreateCluster("OptKmeansCluster");
+  ASSERT_TRUE(!!cluster_int4);
+
+  Params params_int4;
+  params_int4.set("zvec.general.cluster.count", 65);
+  params_int4.set("zvec.optkmeans.cluster.count", 63);
+
+  ASSERT_EQ(0, cluster_int4->init(index_meta2, params_int4));
+  ASSERT_EQ(0, cluster_int4->mount(features2));
+  cluster_int4->suggest(63u);
+
+  std::vector<IndexCluster::Centroid> centroids2;
+
+  // Use centroids of int8 to init centroids of int4
+  for (size_t i = 0; i < centroids1.size(); ++i) {
+    NibbleVector<int8_t> nvec;
+    nvec.assign(reinterpret_cast<const int8_t *>(centroids1[i].feature()),
+                dimension);
+    IndexCluster::Centroid curr_centroid;
+    curr_centroid.set_score(centroids1[i].score());
+    curr_centroid.set_follows(centroids1[i].follows());
+    curr_centroid.set_feature(nvec.data(), nvec.dimension() >> 1);
+    centroids2.push_back(curr_centroid);
+  }
+
+  ASSERT_EQ(0, cluster_int8->cluster(threads, centroids1));
+  ASSERT_EQ(0, cluster_int4->cluster(threads, centroids2));
+
+  EXPECT_EQ(centroids1.size(), centroids2.size());
+  for (size_t i = 0; i < centroids1.size(); ++i) {
+    EXPECT_EQ(centroids1[i].follows(), centroids2[i].follows());
+    EXPECT_DOUBLE_EQ(centroids1[i].score(), centroids2[i].score());
+  }
+}
+
+TEST(OptKmeansCluster, InnerProduct) {
+  // Prepare index data
+  const uint32_t count = 5000u;
+  const uint32_t dimension = 33u;
+
+  IndexMeta index_meta;
+  index_meta.set_meta(IndexMeta::DataType::DT_FP32, dimension);
+  index_meta.set_metric("InnerProduct", 0, Params());
+
+  std::shared_ptr<CompactIndexFeatures> features(
+      new CompactIndexFeatures(index_meta));
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(-1.0, 1.0);
+
+  for (uint32_t i = 0; i < count; ++i) {
+    std::vector<float> vec(dimension);
+    for (size_t j = 0; j < dimension; ++j) {
+      vec[j] = dist(gen);
+    }
+    features->emplace(vec.data());
+  }
+
+  // Create a Kmeans cluster
+  IndexCluster::Pointer cluster =
+      IndexFactory::CreateCluster("OptKmeansCluster");
+  ASSERT_TRUE(!!cluster);
+
+  Params params;
+  params.set("zvec.general.cluster.count", 1);
+  params.set("zvec.optkmeans.cluster.count", 56);
+
+  ASSERT_EQ(0, cluster->init(index_meta, params));
+  ASSERT_EQ(0, cluster->mount(features));
+  cluster->suggest(64u);
+
+  auto threads = std::make_shared<SingleQueueIndexThreads>();
+
+  std::cout << "---------- FIRST ----------\n";
+  std::vector<IndexCluster::Centroid> centroids;
+  std::vector<uint32_t> labels;
+  ASSERT_NE(0, cluster->classify(threads, centroids));
+  ASSERT_NE(0, cluster->label(threads, centroids, &labels));
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  std::cout << "---------- SECOND ----------\n";
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  std::cout << "---------- THIRD ----------\n";
+  ASSERT_EQ(0, cluster->cluster(threads, centroids));
+
+  for (const auto &it : centroids) {
+    const auto &vec = it.vector<float>();
+
+    std::cout << it.follows() << " (" << it.score() << ") { " << vec[0] << ", "
+              << vec[1] << ", " << vec[2] << ", ... , " << vec[vec.size() - 2]
+              << ", " << vec[vec.size() - 1] << " }" << std::endl;
+    ASSERT_EQ(0u, it.similars().size());
+  }
+
+  ASSERT_EQ(0, cluster->classify(threads, centroids));
+  ASSERT_EQ(0, cluster->label(threads, centroids, &labels));
+}

tests/core/algorithm/diskann/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_cluster core_plugin core_knn_diskann
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm/diskann
+    )
+endforeach()

tests/core/algorithm/diskann/diskann_builder_test.cc

@@ -0,0 +1,169 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "diskann_builder.h"
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <chrono>
+#include <future>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/vector.h>
+#include <zvec/core/framework/index_framework.h>
+#include "diskann_holder.h"
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace std;
+
+constexpr size_t static dim = 64;
+
+class DiskAnnBuilderTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+
+  static std::string _dir;
+  static shared_ptr<IndexMeta> _index_meta_ptr;
+};
+
+std::string DiskAnnBuilderTest::_dir("DiskAnnBuilderTest");
+shared_ptr<IndexMeta> DiskAnnBuilderTest::_index_meta_ptr;
+
+void DiskAnnBuilderTest::SetUp(void) {
+  LoggerBroker::SetLevel(Logger::LEVEL_INFO);
+
+  _index_meta_ptr.reset(new (nothrow)
+                            IndexMeta(IndexMeta::DataType::DT_FP32, dim));
+  _index_meta_ptr->set_metric("SquaredEuclidean", 0, Params());
+}
+
+void DiskAnnBuilderTest::TearDown(void) {
+  char cmdBuf[100];
+  snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str());
+  system(cmdBuf);
+}
+
+TEST_F(DiskAnnBuilderTest, TestGeneral) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 10000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+
+  params.set("zvec.diskann.builder.max_degree", 32);
+  params.set("zvec.diskann.builder.list_size", 50);
+  params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
+  params.set("zvec.diskann.builder.threads", 4);
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->train(holder));
+
+  ASSERT_EQ(0, builder->build(holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "/TestGeneral";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(doc_cnt, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_GT(stats.trained_costtime(), 0UL);
+  ASSERT_GT(stats.built_costtime(), 0UL);
+}
+
+// Regression test: building a small DiskAnn index must complete quickly.
+// A lost-wakeup bug in the condition-variable progress loops previously caused
+// 15–30 second stalls during train/build on small datasets because
+// notify_one() was either missing or racing against a wrong predicate.
+TEST_F(DiskAnnBuilderTest, SmallDatasetBuildTime) {
+  constexpr size_t kSmallDim = 4;
+  constexpr size_t kSmallDocCnt = 12;
+
+  auto meta = make_shared<IndexMeta>(IndexMeta::DataType::DT_FP32, kSmallDim);
+  meta->set_metric("SquaredEuclidean", 0, Params());
+
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder = make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+      kSmallDim);
+  for (size_t i = 0; i < kSmallDocCnt; ++i) {
+    NumericalVector<float> vec(kSmallDim, static_cast<float>(i));
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+  params.set("zvec.diskann.builder.max_degree", 32);
+  params.set("zvec.diskann.builder.list_size", 50);
+  params.set("zvec.diskann.builder.max_pq_chunk_num", 2);
+  params.set("zvec.diskann.builder.threads", 4);
+
+  ASSERT_EQ(0, builder->init(*meta, params));
+
+  auto t0 = std::chrono::steady_clock::now();
+  ASSERT_EQ(0, builder->train(holder));
+  ASSERT_EQ(0, builder->build(holder));
+  auto t1 = std::chrono::steady_clock::now();
+
+  auto elapsed_ms =
+      std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
+  // Before the fix, this took 15–30 seconds. After the fix, it should
+  // complete in well under 5 seconds even on slow CI machines.
+  EXPECT_LT(elapsed_ms, 5000)
+      << "DiskAnn build with " << kSmallDocCnt << " vectors took " << elapsed_ms
+      << " ms — likely a lost-wakeup regression in progress loops.";
+}
+
+// DiskAnn is now exposed implicitly: no caller ever invokes a
+// ``LoadDiskAnnPlugin`` / ``IsLibAioAvailable`` API (those were removed from
+// the public surface together with ``zvec.load_diskann_plugin()`` in Python).
+// The only contract this test validates is the UX guarantee: once the DiskAnn
+// module has been linked into the hosting binary (here, directly into the
+// test via the ``core_knn_diskann`` target), its factory entries are
+// registered automatically and the global ``IndexFactory`` can hand out a
+// ``DiskAnnBuilder`` without any explicit setup step. On hosts missing
+// libaio, DiskAnn would fail at the index-creation layer with a clear error
+// while other index types (HNSW/IVF/Flat/Vamana) remain unaffected; that
+// runtime branch lives in ``DiskAnnIndex::CreateAndInitStreamer`` and is
+// covered by the higher-level interface tests.
+TEST_F(DiskAnnBuilderTest, TestImplicitFactoryRegistration) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
+  ASSERT_NE(builder, nullptr)
+      << "DiskAnnBuilder factory entry missing: DiskAnn must be available "
+         "without any manual plugin load step.";
+
+  IndexStreamer::Pointer streamer =
+      IndexFactory::CreateStreamer("DiskAnnStreamer");
+  ASSERT_NE(streamer, nullptr)
+      << "DiskAnnStreamer factory entry missing: DiskAnn must be available "
+         "without any manual plugin load step.";
+}

tests/core/algorithm/diskann/diskann_searcher_test.cc

@@ -0,0 +1,816 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "diskann_searcher.h"
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <ailego/math/distance.h>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/vector.h>
+#include <zvec/core/framework/index_framework.h>
+#include "diskann_holder.h"
+#include "diskann_params.h"
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace std;
+
+constexpr size_t static dim = 64;
+
+class DiskAnnSearcherTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+
+  static std::string _dir;
+  static shared_ptr<IndexMeta> _index_meta_ptr;
+};
+
+std::string DiskAnnSearcherTest::_dir("DiskAnnSearcherTest/");
+shared_ptr<IndexMeta> DiskAnnSearcherTest::_index_meta_ptr;
+
+void DiskAnnSearcherTest::SetUp(void) {
+  LoggerBroker::SetLevel(Logger::LEVEL_INFO);
+
+  _index_meta_ptr.reset(new (nothrow)
+                            IndexMeta(IndexMeta::DataType::DT_FP32, dim));
+  _index_meta_ptr->set_metric("SquaredEuclidean", 0, Params());
+}
+
+void DiskAnnSearcherTest::TearDown(void) {
+  char cmdBuf[100];
+  snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str());
+  system(cmdBuf);
+}
+
+TEST_F(DiskAnnSearcherTest, TestGeneral) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 10000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+
+  params.set("zvec.diskann.builder.max_degree", 32);
+  params.set("zvec.diskann.builder.list_size", 300);
+  params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
+  params.set("zvec.diskann.builder.threads", 4);
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->train(holder));
+
+  ASSERT_EQ(0, builder->build(holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "/TestGeneral";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(doc_cnt, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_GT(stats.trained_costtime(), 0UL);
+  ASSERT_GT(stats.built_costtime(), 0UL);
+
+  // test searcher
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("DiskAnnSearcher");
+  ASSERT_TRUE(searcher != nullptr);
+
+  Params search_params;
+  search_params.set("zvec.diskann.searcher.list_size", 500);
+
+  ASSERT_EQ(0, searcher->init(search_params));
+
+  auto storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_EQ(0, storage->open(path, false));
+  ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
+  auto ctx = searcher->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  auto linearCtx = searcher->create_context();
+  auto linearByPKeysCtx = searcher->create_context();
+  auto knnCtx = searcher->create_context();
+
+  ASSERT_TRUE(!!linearCtx);
+  ASSERT_TRUE(!!linearByPKeysCtx);
+  ASSERT_TRUE(!!knnCtx);
+
+  NumericalVector<float> vec(dim);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+  size_t topk = 200;
+  uint64_t knnTotalTime = 0;
+  uint64_t linearTotalTime = 0;
+  int totalHits = 0;
+  int totalCnts = 0;
+  int topk1Hits = 0;
+  linearCtx->set_topk(topk);
+  linearByPKeysCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+
+  // do linear search test
+  {
+    float query[dim];
+    for (size_t i = 0; i < dim; ++i) {
+      query[i] = 3.1f;
+    }
+    ASSERT_EQ(0, searcher->search_bf_impl(query, qmeta, linearCtx));
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(3UL, linearResult[0].key());
+    ASSERT_EQ(4UL, linearResult[1].key());
+    ASSERT_EQ(2UL, linearResult[2].key());
+    ASSERT_EQ(5UL, linearResult[3].key());
+    ASSERT_EQ(1UL, linearResult[4].key());
+    ASSERT_EQ(6UL, linearResult[5].key());
+    ASSERT_EQ(0UL, linearResult[6].key());
+    ASSERT_EQ(7UL, linearResult[7].key());
+    for (size_t i = 8; i < topk; ++i) {
+      ASSERT_EQ(i, linearResult[i].key());
+    }
+  }
+
+  // do linear search by p_keys test
+  std::vector<std::vector<uint64_t>> p_keys;
+  p_keys.resize(1);
+  p_keys[0] = {8, 9, 10, 11, 3, 2, 1, 0};
+  {
+    float query[dim];
+    for (size_t i = 0; i < dim; ++i) {
+      query[i] = 3.1f;
+    }
+
+    ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(query, p_keys, qmeta,
+                                                    linearByPKeysCtx));
+    auto &linearByPKeysResult = linearByPKeysCtx->result();
+    ASSERT_EQ(8, linearByPKeysResult.size());
+    ASSERT_EQ(3UL, linearByPKeysResult[0].key());
+    ASSERT_EQ(2UL, linearByPKeysResult[1].key());
+    ASSERT_EQ(1UL, linearByPKeysResult[2].key());
+    ASSERT_EQ(0UL, linearByPKeysResult[3].key());
+    ASSERT_EQ(8UL, linearByPKeysResult[4].key());
+    ASSERT_EQ(9UL, linearByPKeysResult[5].key());
+    ASSERT_EQ(10UL, linearByPKeysResult[6].key());
+    ASSERT_EQ(11UL, linearByPKeysResult[7].key());
+  }
+
+  size_t step = 500;
+  for (size_t i = 0; i < doc_cnt; i += step) {
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    auto t1 = Realtime::MicroSeconds();
+    ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
+    auto t2 = Realtime::MicroSeconds();
+
+    ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
+    auto t3 = Realtime::MicroSeconds();
+    knnTotalTime += t2 - t1;
+    linearTotalTime += t3 - t2;
+
+    auto &knnResult = knnCtx->result();
+    // TODO: check
+    topk1Hits += i == knnResult[0].key();
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      totalCnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linearResult[j].key() == knnResult[k].key()) {
+          totalHits++;
+          break;
+        }
+      }
+    }
+  }
+
+  float recall = totalHits * step * step * 1.0f / totalCnts;
+  float topk1Recall = topk1Hits * step * 1.0f / doc_cnt;
+  float cost = linearTotalTime * 1.0f / knnTotalTime;
+
+  EXPECT_GT(recall, 0.90f);
+  EXPECT_GT(topk1Recall, 0.80f);
+  EXPECT_GT(cost, 2.0f);
+}
+
+TEST_F(DiskAnnSearcherTest, TestNodeCache) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 10000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+
+  params.set("zvec.diskann.builder.max_degree", 32);
+  params.set("zvec.diskann.builder.list_size", 300);
+  params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
+  params.set("zvec.diskann.builder.threads", 4);
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->train(holder));
+
+  ASSERT_EQ(0, builder->build(holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "/TestNodeCache";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(doc_cnt, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_GT(stats.trained_costtime(), 0UL);
+  ASSERT_GT(stats.built_costtime(), 0UL);
+
+  // test searcher
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("DiskAnnSearcher");
+  ASSERT_TRUE(searcher != nullptr);
+
+  Params search_params;
+  search_params.set("zvec.diskann.searcher.cache_node_num", 32);
+  search_params.set("zvec.diskann.searcher.list_size", 500);
+
+  ASSERT_EQ(0, searcher->init(search_params));
+
+  auto storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_EQ(0, storage->open(path, false));
+  ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
+  auto ctx = searcher->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  auto linearCtx = searcher->create_context();
+  auto linearByPKeysCtx = searcher->create_context();
+  auto knnCtx = searcher->create_context();
+
+  ASSERT_TRUE(!!linearCtx);
+  ASSERT_TRUE(!!linearByPKeysCtx);
+  ASSERT_TRUE(!!knnCtx);
+
+  NumericalVector<float> vec(dim);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+  size_t topk = 200;
+  uint64_t knnTotalTime = 0;
+  uint64_t linearTotalTime = 0;
+  int totalHits = 0;
+  int totalCnts = 0;
+  int topk1Hits = 0;
+  linearCtx->set_topk(topk);
+  linearByPKeysCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+
+  size_t step = 500;
+  for (size_t i = 0; i < doc_cnt; i += step) {
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    auto t1 = Realtime::MicroSeconds();
+    ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
+    auto t2 = Realtime::MicroSeconds();
+
+    ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
+    auto t3 = Realtime::MicroSeconds();
+    knnTotalTime += t2 - t1;
+    linearTotalTime += t3 - t2;
+
+    auto &knnResult = knnCtx->result();
+    // TODO: check
+    topk1Hits += i == knnResult[0].key();
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      totalCnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linearResult[j].key() == knnResult[k].key()) {
+          totalHits++;
+          break;
+        }
+      }
+    }
+  }
+
+  float recall = totalHits * step * step * 1.0f / totalCnts;
+  float topk1Recall = topk1Hits * step * 1.0f / doc_cnt;
+  float cost = linearTotalTime * 1.0f / knnTotalTime;
+
+  EXPECT_GT(recall, 0.90f);
+  EXPECT_GT(topk1Recall, 0.80f);
+  EXPECT_GT(cost, 2.0f);
+}
+
+TEST_F(DiskAnnSearcherTest, TestFilter) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 10000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+
+  params.set("zvec.diskann.builder.max_degree", 32);
+  params.set("zvec.diskann.builder.list_size", 300);
+  params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
+  params.set("zvec.diskann.builder.threads", 4);
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->train(holder));
+
+  ASSERT_EQ(0, builder->build(holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "/TestFilter";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(doc_cnt, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_GT(stats.trained_costtime(), 0UL);
+  ASSERT_GT(stats.built_costtime(), 0UL);
+
+  // test searcher
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("DiskAnnSearcher");
+  ASSERT_TRUE(searcher != nullptr);
+
+  Params search_params;
+  search_params.set("zvec.diskann.searcher.cache_node_num", 32);
+  search_params.set("zvec.diskann.searcher.list_size", 500);
+
+  ASSERT_EQ(0, searcher->init(search_params));
+
+  auto storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_EQ(0, storage->open(path, false));
+  ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
+  auto ctx = searcher->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  auto linearCtx = searcher->create_context();
+  auto linearByPKeysCtx = searcher->create_context();
+  auto knnCtx = searcher->create_context();
+
+  ASSERT_TRUE(!!linearCtx);
+  ASSERT_TRUE(!!linearByPKeysCtx);
+  ASSERT_TRUE(!!knnCtx);
+
+  NumericalVector<float> vec(dim);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+
+  size_t topk = 200;
+  linearCtx->set_topk(topk);
+  linearByPKeysCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+
+  size_t key = 50;
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = key + 0.1f;
+  }
+
+  // no filter
+  {
+    ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
+
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+    ASSERT_EQ(50UL, knnResult[0].key());
+    ASSERT_EQ(51UL, knnResult[1].key());
+    ASSERT_EQ(49UL, knnResult[2].key());
+
+    ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(50UL, linearResult[0].key());
+    ASSERT_EQ(51UL, linearResult[1].key());
+    ASSERT_EQ(49UL, linearResult[2].key());
+  }
+
+  // with filter
+  {
+    auto filterFunc = [](uint64_t key) {
+      if (key == 50UL || key == 51UL || key == 49UL) {
+        return true;
+      }
+      return false;
+    };
+
+
+    knnCtx->set_filter(filterFunc);
+    ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
+
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+    ASSERT_EQ(52UL, knnResult[0].key());
+    ASSERT_EQ(48UL, knnResult[1].key());
+    ASSERT_EQ(53UL, knnResult[2].key());
+
+    linearCtx->set_filter(filterFunc);
+    ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(52UL, linearResult[0].key());
+    ASSERT_EQ(48UL, linearResult[1].key());
+    ASSERT_EQ(53UL, linearResult[2].key());
+  }
+}
+
+TEST_F(DiskAnnSearcherTest, TestGroup) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 10000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i / 10.0;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+
+  params.set("zvec.diskann.builder.max_degree", 32);
+  params.set("zvec.diskann.builder.list_size", 300);
+  params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
+  params.set("zvec.diskann.builder.threads", 4);
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->train(holder));
+
+  ASSERT_EQ(0, builder->build(holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "/TestGroup";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(doc_cnt, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_GT(stats.trained_costtime(), 0UL);
+  ASSERT_GT(stats.built_costtime(), 0UL);
+
+  // test searcher
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("DiskAnnSearcher");
+  ASSERT_TRUE(searcher != nullptr);
+
+  Params search_params;
+  search_params.set("zvec.diskann.searcher.list_size", 500);
+
+  ASSERT_EQ(0, searcher->init(search_params));
+
+  auto storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_EQ(0, storage->open(path, false));
+  ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
+  auto ctx = searcher->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  NumericalVector<float> vec(dim);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+  size_t group_topk = 20;
+  uint64_t total_time = 0;
+
+  auto groupbyFunc = [](uint64_t key) {
+    uint32_t group_id = key / 10 % 10;
+
+    // std::cout << "key: " << key << ", group id: " << group_id << std::endl;
+
+    return std::string("g_") + std::to_string(group_id);
+  };
+
+  size_t group_num = 5;
+
+  ctx->set_group_params(group_num, group_topk);
+  ctx->set_group_by(groupbyFunc);
+
+  size_t query_value = doc_cnt / 2;
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = query_value / 10 + 0.1f;
+  }
+
+  auto t1 = Realtime::MicroSeconds();
+  ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, ctx));
+  auto t2 = Realtime::MicroSeconds();
+
+  total_time += t2 - t1;
+
+  auto &group_result = ctx->group_result();
+
+  for (uint32_t i = 0; i < group_result.size(); ++i) {
+    const std::string &group_id = group_result[i].group_id();
+    auto &result = group_result[i].docs();
+
+    ASSERT_GT(result.size(), 0);
+    std::cout << "Group ID: " << group_id << std::endl;
+
+    for (uint32_t j = 0; j < result.size(); ++j) {
+      std::cout << "\tKey: " << result[j].key() << std::fixed
+                << std::setprecision(3) << ", Score: " << result[j].score()
+                << std::endl;
+    }
+  }
+
+#if 0
+  // do linear search by p_keys test
+  auto groupbyFuncLinear = [](uint64_t key) {
+    uint32_t group_id = key % 10;
+
+    return std::string("g_") + std::to_string(group_id);
+  };
+
+  auto linear_pk_ctx = searcher->create_context();
+
+  linear_pk_ctx->set_group_params(group_num, group_topk);
+  linear_pk_ctx->set_group_by(groupbyFuncLinear);
+
+  std::vector<std::vector<uint64_t>> p_keys;
+  p_keys.resize(1);
+  p_keys[0] = {4, 3, 2, 1, 5, 6, 7, 8, 9, 10};
+
+  ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(vec.data(), p_keys, qmeta,
+                                                  linear_pk_ctx));
+  auto &linear_by_pkeys_group_result = linear_pk_ctx->group_result();
+  ASSERT_EQ(linear_by_pkeys_group_result.size(), group_num);
+
+  for (uint32_t i = 0; i < linear_by_pkeys_group_result.size(); ++i) {
+    const std::string &group_id = linear_by_pkeys_group_result[i].group_id();
+    auto &result = linear_by_pkeys_group_result[i].docs();
+
+    ASSERT_GT(result.size(), 0);
+    std::cout << "Group ID: " << group_id << std::endl;
+
+    for (uint32_t j = 0; j < result.size(); ++j) {
+      std::cout << "\tKey: " << result[j].key() << std::fixed
+                << std::setprecision(3) << ", Score: " << result[j].score()
+                << std::endl;
+    }
+
+    ASSERT_EQ(10 - i, result[0].key());
+  }
+#endif
+}
+
+TEST_F(DiskAnnSearcherTest, TestFetchVector) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 10000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+
+  params.set("zvec.diskann.builder.max_degree", 32);
+  params.set("zvec.diskann.builder.list_size", 300);
+  params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
+  params.set("zvec.diskann.builder.threads", 4);
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->train(holder));
+
+  ASSERT_EQ(0, builder->build(holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "/TestFetchVector";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(doc_cnt, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_GT(stats.trained_costtime(), 0UL);
+  ASSERT_GT(stats.built_costtime(), 0UL);
+
+  // test searcher
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("DiskAnnSearcher");
+  ASSERT_TRUE(searcher != nullptr);
+
+  Params search_params;
+  search_params.set("zvec.diskann.searcher.list_size", 500);
+
+  ASSERT_EQ(0, searcher->init(search_params));
+
+  auto storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_EQ(0, storage->open(path, false));
+  ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
+
+  size_t query_cnt = 20U;
+  auto linearCtx = searcher->create_context();
+  auto knnCtx = searcher->create_context();
+  auto linearByPKeysCtx = searcher->create_context();
+  knnCtx->set_fetch_vector(true);
+
+  for (size_t i = 0; i < doc_cnt; i += doc_cnt / 10) {
+    std::string vec_value;
+    ASSERT_EQ(0, searcher->get_vector(i, linearCtx, vec_value));
+
+    float vector_value = *(const float *)(vec_value.data());
+    ASSERT_EQ(vector_value, i);
+  }
+
+  size_t topk = 200;
+  linearCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+  uint64_t knnTotalTime = 0;
+  uint64_t linearTotalTime = 0;
+
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+
+  NumericalVector<float> vec(dim);
+  for (size_t i = 0; i < query_cnt; i++) {
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+
+    auto t1 = Realtime::MicroSeconds();
+    ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
+    auto t2 = Realtime::MicroSeconds();
+    ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
+    auto t3 = Realtime::MicroSeconds();
+    knnTotalTime += t2 - t1;
+    linearTotalTime += t3 - t2;
+
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+
+    ASSERT_NE(knnResult[0].vector_string(), "");
+    float vector_value = *((float *)(knnResult[0].vector_string().data()));
+    ASSERT_EQ(vector_value, i);
+  }
+}
+
+TEST_F(DiskAnnSearcherTest, TestRnnSearch) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("DiskAnnBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 10000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+
+  params.set("zvec.diskann.builder.max_degree", 32);
+  params.set("zvec.diskann.builder.list_size", 300);
+  params.set("zvec.diskann.builder.max_pq_chunk_num", 32);
+  params.set("zvec.diskann.builder.threads", 4);
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->train(holder));
+
+  ASSERT_EQ(0, builder->build(holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "/TestRnnSearch";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(doc_cnt, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_GT(stats.trained_costtime(), 0UL);
+  ASSERT_GT(stats.built_costtime(), 0UL);
+
+  // test searcher
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("DiskAnnSearcher");
+  ASSERT_TRUE(searcher != nullptr);
+
+  Params search_params;
+  search_params.set("zvec.diskann.searcher.list_size", 500);
+
+  ASSERT_EQ(0, searcher->init(search_params));
+
+  auto storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_EQ(0, storage->open(path, false));
+  ASSERT_EQ(0, searcher->load(storage, IndexMetric::Pointer()));
+
+  auto ctx = searcher->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 0.0;
+  }
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+  size_t topk = 50;
+  ctx->set_topk(topk);
+  ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, ctx));
+  auto &results = ctx->result();
+  ASSERT_EQ(topk, results.size());
+
+  float radius = results[topk / 2].score();
+  ctx->set_threshold(radius);
+  ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, ctx));
+  ASSERT_GT(topk, results.size());
+  for (size_t k = 0; k < results.size(); ++k) {
+    ASSERT_GE(radius, results[k].score());
+  }
+
+  // Test Reset Threshold
+  ctx->reset_threshold();
+  ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, ctx));
+  ASSERT_EQ(topk, results.size());
+  ASSERT_LT(radius, results[topk - 1].score());
+}

tests/core/algorithm/flat/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_flat 
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
+    )
+endforeach()

tests/core/algorithm/flat/flat_builder_test.cc

@@ -0,0 +1,334 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "flat/flat_builder.h"
+#include <future>
+#include <iostream>
+#include <vector>
+#include <gtest/gtest.h>
+#include "tests/test_util.h"
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace std;
+
+static inline size_t RandomDimension(void) {
+  std::mt19937 gen((std::random_device())());
+  return (std::uniform_int_distribution<size_t>(1, 129))(gen);
+}
+
+static size_t DIMENSION = RandomDimension();
+class FlatBuilderTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+
+ public:
+  static std::string dir_;
+  static IndexMeta meta_;
+};
+
+std::string FlatBuilderTest ::dir_("flat_builder_test/");
+IndexMeta FlatBuilderTest::meta_;
+
+void FlatBuilderTest::SetUp(void) {
+  meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_COLUMN);
+}
+
+//! self-check column-major and row-major search.
+void FlatBuilderTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+void build_process(IndexBuilder::Pointer &builder,
+                   IndexHolder::Pointer holder) {
+  Params params;
+  ASSERT_EQ(0, builder->init(FlatBuilderTest::meta_, params));
+  ASSERT_EQ(0, builder->train(holder));
+  ASSERT_EQ(0, builder->build(holder));
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  std::string path = FlatBuilderTest::dir_ + "TestGeneral";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+}
+
+TEST_F(FlatBuilderTest, TestInitSuccess) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+  Params params;
+  ASSERT_EQ(0, builder->init(meta_, params));
+}
+
+TEST_F(FlatBuilderTest, TestInitFailedWithInvalidMeasure) {
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+  meta_.set_metric("invalid", 0, Params());
+  Params params;
+  int ret = builder->init(meta_, params);
+  EXPECT_EQ(IndexError_InvalidArgument, ret);
+}
+
+TEST_F(FlatBuilderTest, TestInt8InvalidColumnMajor) {
+  size_t dim = (DIMENSION + 3) / 4 * 4;
+  meta_.set_meta(IndexMeta::DataType::DT_INT8, dim + 2);
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_COLUMN);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  ASSERT_EQ(IndexMeta::MO_COLUMN, meta_.major_order());
+  Params params;
+  ASSERT_NE(0, builder->init(meta_, params));
+}
+
+TEST_F(FlatBuilderTest, TestInt8WithRandomDimension) {
+  size_t dim = DIMENSION;
+  meta_.set_meta(IndexMeta::DataType::DT_INT8, dim);
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_UNDEFINED);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  Params params;
+  ASSERT_EQ(0, builder->init(meta_, params));
+}
+
+TEST_F(FlatBuilderTest, TestBuildWithRowMajor) {
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_ROW);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+  Params params;
+  ASSERT_EQ(0, builder->init(meta_, params));
+  std::string path = dir_ + "TestGeneral";
+
+  auto holder =
+      std::make_shared<OnePassIndexHolder<IndexMeta::DT_FP32>>(DIMENSION);
+  size_t doc_cnt = 2000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  int ret = builder->train(holder);
+  EXPECT_EQ(0, ret);
+
+  ret = builder->build(holder);
+  EXPECT_EQ(0, ret);
+}
+
+TEST_F(FlatBuilderTest, TestInt8BuildWithRowMajor) {
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_meta(IndexMeta::DT_INT8, DIMENSION);
+  meta_.set_major_order(IndexMeta::MO_ROW);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+  Params params;
+  ASSERT_EQ(0, builder->init(meta_, params));
+  std::string path = dir_ + "TestGeneral";
+
+  auto holder =
+      std::make_shared<OnePassIndexHolder<IndexMeta::DT_INT8>>(DIMENSION);
+  size_t doc_cnt = 128UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<int8_t> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = (int8_t)(i % 128);
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  int ret = builder->train(holder);
+  EXPECT_EQ(0, ret);
+
+  ret = builder->build(holder);
+  EXPECT_EQ(0, ret);
+}
+
+TEST_F(FlatBuilderTest, TestBuildWithColumnMajor) {
+  meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_COLUMN);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+  Params params;
+  ASSERT_EQ(0, builder->init(meta_, params));
+  std::string path = dir_ + "TestGeneral";
+
+  auto holder =
+      std::make_shared<OnePassIndexHolder<IndexMeta::DT_FP32>>(DIMENSION);
+  size_t doc_cnt = 2000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  int ret = builder->train(holder);
+  EXPECT_EQ(0, ret);
+
+  ret = builder->build(holder);
+  EXPECT_EQ(0, ret);
+}
+
+TEST_F(FlatBuilderTest, TestInt8BuildWithColumnMajor) {
+  size_t dim = (DIMENSION + 3) / 4 * 4;
+  meta_.set_meta(IndexMeta::DataType::DT_INT8, dim);
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_COLUMN);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+  Params params;
+  ASSERT_EQ(0, builder->init(meta_, params));
+  std::string path = dir_ + "TestGeneral";
+
+  auto holder = std::make_shared<OnePassIndexHolder<IndexMeta::DT_INT8>>(dim);
+  size_t doc_cnt = 128UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<int8_t> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = (int8_t)(i % 128);
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  int ret = builder->train(holder);
+  EXPECT_EQ(0, ret);
+
+  ret = builder->build(holder);
+  EXPECT_EQ(0, ret);
+}
+
+TEST_F(FlatBuilderTest, TestWithRowMajor) {
+  meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_ROW);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+  Params params;
+  std::string path = dir_ + "TestGeneral";
+
+  auto holder =
+      std::make_shared<OnePassIndexHolder<IndexMeta::DT_FP32>>(DIMENSION);
+  size_t doc_cnt = 2000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+  build_process(builder, holder);
+
+  // cleanup and rebuild
+  ASSERT_EQ(0, builder->cleanup());
+}
+
+TEST_F(FlatBuilderTest, TestInt8WithRowMajor) {
+  meta_.set_meta(IndexMeta::DataType::DT_INT8, DIMENSION);
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_ROW);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+  Params params;
+  std::string path = dir_ + "TestGeneral";
+
+  auto holder =
+      std::make_shared<OnePassIndexHolder<IndexMeta::DT_INT8>>(DIMENSION);
+  size_t doc_cnt = 128UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<int8_t> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = (int8_t)(i % 128);
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+  build_process(builder, holder);
+
+  // cleanup and rebuild
+  ASSERT_EQ(0, builder->cleanup());
+}
+
+TEST_F(FlatBuilderTest, TestWithColumnMajor) {
+  meta_.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_COLUMN);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+  Params params;
+  std::string path = dir_ + "TestGeneral";
+
+  auto holder =
+      std::make_shared<OnePassIndexHolder<IndexMeta::DT_FP32>>(DIMENSION);
+  size_t doc_cnt = 2000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+  build_process(builder, holder);
+
+  // cleanup and rebuild
+  ASSERT_EQ(0, builder->cleanup());
+}
+
+TEST_F(FlatBuilderTest, TestInt8WithColumnMajor) {
+  size_t dim = (DIMENSION + 3) / 4 * 4;
+  meta_.set_meta(IndexMeta::DataType::DT_INT8, dim);
+  meta_.set_metric("SquaredEuclidean", 0, Params());
+  meta_.set_major_order(IndexMeta::MO_COLUMN);
+  IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder");
+  ASSERT_NE(builder, nullptr);
+  Params params;
+  std::string path = dir_ + "TestGeneral";
+
+  auto holder = std::make_shared<OnePassIndexHolder<IndexMeta::DT_INT8>>(dim);
+  size_t doc_cnt = 128UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<int8_t> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = (int8_t)(i % 128);
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+  build_process(builder, holder);
+
+  // cleanup and rebuild
+  ASSERT_EQ(0, builder->cleanup());
+}
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/algorithm/flat/flat_streamer_buffer_test.cc

@@ -0,0 +1,623 @@
+#include <future>
+#include <string>
+#include <vector>
+#include <ailego/utility/math_helper.h>
+#include <ailego/utility/memory_helper.h>
+#include <gtest/gtest.h>
+#include <zvec/core/framework/index_framework.h>
+#include <zvec/core/framework/index_streamer.h>
+#include "tests/test_util.h"
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace std;
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+constexpr size_t static dim = 16;
+
+class FlatStreamerTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+  void hybrid_scale(std::vector<float> &dense_value,
+                    std::vector<float> &sparse_value, float alpha_scale);
+
+  static std::string dir_;
+  static std::shared_ptr<IndexMeta> index_meta_ptr_;
+};
+
+std::string FlatStreamerTest::dir_("flat_streamer_buffer_test_dir/");
+std::shared_ptr<IndexMeta> FlatStreamerTest::index_meta_ptr_;
+
+void FlatStreamerTest::SetUp(void) {
+  index_meta_ptr_.reset(new (std::nothrow)
+                            IndexMeta(IndexMeta::DataType::DT_FP32, dim));
+  index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params());
+
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+void FlatStreamerTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+TEST_F(FlatStreamerTest, TestLinearSearch) {
+  MemoryLimitPool::get_instance().init(2 * 1024UL * 1024UL * 1024UL);
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearch", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t cnt = 10000UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  storage->close();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearch", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 3;
+  auto provider = read_streamer->create_provider();
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_FLOAT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  ctx->set_topk(100U);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 10.1f;
+  }
+  ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
+  auto &result = ctx->result();
+  ASSERT_EQ(100U, result.size());
+  ASSERT_EQ(10, result[0].key());
+  ASSERT_EQ(11, result[1].key());
+  ASSERT_EQ(5, result[10].key());
+  ASSERT_EQ(0, result[20].key());
+  ASSERT_EQ(30, result[30].key());
+  ASSERT_EQ(35, result[35].key());
+  ASSERT_EQ(99, result[99].key());
+
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_FLOAT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+  cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
+
+  read_streamer->close();
+  read_streamer.reset();
+}
+
+TEST_F(FlatStreamerTest, TestLinearSearchBuffer) {
+  MemoryLimitPool::get_instance().init(2 * 1024UL * 1024UL * 1024UL);
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearchBuffer", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t cnt = 10000UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  storage->close();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearchBuffer", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 3;
+  auto provider = read_streamer->create_provider();
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_FLOAT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  ctx->set_topk(100U);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 10.1f;
+  }
+  ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
+  auto &result = ctx->result();
+  ASSERT_EQ(100U, result.size());
+  ASSERT_EQ(10, result[0].key());
+  ASSERT_EQ(11, result[1].key());
+  ASSERT_EQ(5, result[10].key());
+  ASSERT_EQ(0, result[20].key());
+  ASSERT_EQ(30, result[30].key());
+  ASSERT_EQ(35, result[35].key());
+  ASSERT_EQ(99, result[99].key());
+
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_FLOAT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+  cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
+
+  read_streamer->close();
+  read_streamer.reset();
+}
+
+TEST_F(FlatStreamerTest, TestLinearSearchBufferMMap) {
+  MemoryLimitPool::get_instance().init(2 * 1024UL * 1024UL * 1024UL);
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearchBuffer", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t cnt = 10000UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  storage->close();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearchBuffer", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 3;
+  auto provider = read_streamer->create_provider();
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_FLOAT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  ctx->set_topk(100U);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 10.1f;
+  }
+  ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
+  auto &result = ctx->result();
+  ASSERT_EQ(100U, result.size());
+  ASSERT_EQ(10, result[0].key());
+  ASSERT_EQ(11, result[1].key());
+  ASSERT_EQ(5, result[10].key());
+  ASSERT_EQ(0, result[20].key());
+  ASSERT_EQ(30, result[30].key());
+  ASSERT_EQ(35, result[35].key());
+  ASSERT_EQ(99, result[99].key());
+
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_FLOAT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+  cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
+
+  read_streamer->close();
+  read_streamer.reset();
+}
+
+
+TEST_F(FlatStreamerTest, TestLinearSearchWithLRU) {
+  MemoryLimitPool::get_instance().init(100 * 1024UL * 1024UL);
+#ifdef __ANDROID__
+  GTEST_SKIP()
+      << "Skipped on Android: requires ~6GB memory/disk (emulator limit)";
+#endif
+  constexpr size_t static dim = 1600;
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  IndexMeta meta = IndexMeta(IndexMeta::DataType::DT_FP32, dim);
+  meta.set_metric("SquaredEuclidean", 0, Params());
+  ASSERT_EQ(0, write_streamer->init(meta, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearchWithLRU", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t cnt = 50000UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  storage->close();
+
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_EQ(0, read_streamer->init(meta, params));
+  auto read_storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearchWithLRU", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 3;
+  auto provider = read_streamer->create_provider();
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < 10; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+  cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
+
+  read_streamer->close();
+  read_streamer.reset();
+}
+
+TEST_F(FlatStreamerTest, TestLinearSearchMMap) {
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearchMMap", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t cnt = 10000UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  storage->close();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearchMMap", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 3;
+  auto provider = read_streamer->create_provider();
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_FLOAT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  ctx->set_topk(100U);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 10.1f;
+  }
+  ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
+  auto &result = ctx->result();
+  ASSERT_EQ(100U, result.size());
+  ASSERT_EQ(10, result[0].key());
+  ASSERT_EQ(11, result[1].key());
+  ASSERT_EQ(5, result[10].key());
+  ASSERT_EQ(0, result[20].key());
+  ASSERT_EQ(30, result[30].key());
+  ASSERT_EQ(35, result[35].key());
+  ASSERT_EQ(99, result[99].key());
+
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    for (size_t j = 0; j < dim; ++j) {
+      const float *data = (float *)provider->get_vector(result1[0].key());
+      EXPECT_FLOAT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  read_streamer->close();
+  read_streamer.reset();
+  cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
+}
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/algorithm/flat/flat_streamer_buffer_time_test.cc

@@ -0,0 +1,231 @@
+#include <future>
+#include <string>
+#include <vector>
+#include <ailego/utility/math_helper.h>
+#include <ailego/utility/memory_helper.h>
+#include <gtest/gtest.h>
+#include <zvec/ailego/utility/file_helper.h>
+#include <zvec/core/framework/index_framework.h>
+#include <zvec/core/framework/index_streamer.h>
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace std;
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+constexpr size_t static dim = 128;
+
+class FlatStreamerTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+  void hybrid_scale(std::vector<float> &dense_value,
+                    std::vector<float> &sparse_value, float alpha_scale);
+
+  static std::string dir_;
+  static std::shared_ptr<IndexMeta> index_meta_ptr_;
+};
+
+std::string FlatStreamerTest::dir_("streamer_test/");
+std::shared_ptr<IndexMeta> FlatStreamerTest::index_meta_ptr_;
+
+void FlatStreamerTest::SetUp(void) {
+  index_meta_ptr_.reset(new (std::nothrow)
+                            IndexMeta(IndexMeta::DataType::DT_FP32, dim));
+  index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params());
+
+  zvec::ailego::FileHelper::RemovePath(dir_.c_str());
+}
+
+void FlatStreamerTest::TearDown(void) {
+  zvec::ailego::FileHelper::RemovePath(dir_.c_str());
+}
+
+TEST_F(FlatStreamerTest, TestLinearSearchMMap) {
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearchMMap", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t data_cnt = 300000UL, cnt = 500UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < data_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearchMMap", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 30;
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    // auto &result1 = ctx->result();
+    // ASSERT_EQ(topk, result1.size());
+    // ASSERT_EQ(i, result1[0].key());
+
+    // for (size_t j = 0; j < dim; ++j) {
+    //   vec[j] = i + 0.1f;
+    // }
+    // ctx->set_topk(topk);
+    // ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    // auto &result2 = ctx->result();
+    // ASSERT_EQ(topk, result2.size());
+    // ASSERT_EQ(i, result2[0].key());
+    // ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    // ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+  cout << "Elapsed time: " << elapsed_time.micro_seconds() << " us" << endl;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    // auto &result1 = ctx->result();
+    // ASSERT_EQ(topk, result1.size());
+    // ASSERT_EQ(i, result1[0].key());
+
+    // for (size_t j = 0; j < dim; ++j) {
+    //   vec[j] = i + 0.1f;
+    // }
+    // ctx->set_topk(topk);
+    // ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    // auto &result2 = ctx->result();
+    // ASSERT_EQ(topk, result2.size());
+    // ASSERT_EQ(i, result2[0].key());
+    // ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    // ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+  cout << "Elapsed time: " << elapsed_time.micro_seconds() << " us" << endl;
+  read_streamer->close();
+  read_streamer.reset();
+}
+
+TEST_F(FlatStreamerTest, TestLinearSearchBuffer) {
+  MemoryLimitPool::get_instance().init(2 * 1024UL * 1024UL * 1024UL);
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearchBuffer", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t data_cnt = 300000UL, cnt = 500UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < data_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("FlatStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearchBuffer", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 30;
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    // auto &result1 = ctx->result();
+    // ASSERT_EQ(topk, result1.size());
+    // ASSERT_EQ(i, result1[0].key());
+
+    // for (size_t j = 0; j < dim; ++j) {
+    //   vec[j] = i + 0.1f;
+    // }
+    // ctx->set_topk(topk);
+    // ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    // auto &result2 = ctx->result();
+    // ASSERT_EQ(topk, result2.size());
+    // ASSERT_EQ(i, result2[0].key());
+    // ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    // ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+  cout << "Elapsed time: " << elapsed_time.micro_seconds() << " us" << endl;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    // auto &result1 = ctx->result();
+    // ASSERT_EQ(topk, result1.size());
+    // ASSERT_EQ(i, result1[0].key());
+
+    // for (size_t j = 0; j < dim; ++j) {
+    //   vec[j] = i + 0.1f;
+    // }
+    // ctx->set_topk(topk);
+    // ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    // auto &result2 = ctx->result();
+    // ASSERT_EQ(topk, result2.size());
+    // ASSERT_EQ(i, result2[0].key());
+    // ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    // ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+  cout << "Elapsed time: " << elapsed_time.micro_seconds() << " us" << endl;
+  read_streamer->close();
+  read_streamer.reset();
+}
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/algorithm/flat_sparse/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_flat_sparse
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
+    )
+endforeach()

tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc

@@ -0,0 +1,300 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "flat_sparse/flat_sparse_builder.h"
+#include <future>
+#include <iostream>
+#include <vector>
+#include <gtest/gtest.h>
+#include "tests/test_util.h"
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace std;
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+class FlatSparseBuilderTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+
+  static std::string _dir;
+  static shared_ptr<IndexMeta> _index_meta_ptr;
+};
+
+std::string FlatSparseBuilderTest::_dir("FlatSparseBuilderTest/");
+shared_ptr<IndexMeta> FlatSparseBuilderTest::_index_meta_ptr;
+
+void FlatSparseBuilderTest::SetUp(void) {
+  _index_meta_ptr.reset(new (nothrow) IndexMeta(IndexMeta::MetaType::MT_SPARSE,
+                                                IndexMeta::DataType::DT_FP32));
+  _index_meta_ptr->set_metric("InnerProductSparse", 0, Params());
+}
+
+void FlatSparseBuilderTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(_dir);
+}
+
+TEST_F(FlatSparseBuilderTest, TestGeneral) {
+  IndexBuilder::Pointer builder =
+      IndexFactory::CreateBuilder("FlatSparseBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder = make_shared<OnePassIndexSparseHolder<IndexMeta::DT_FP32>>();
+  uint32_t sparse_count = 4;
+  size_t doc_cnt = 1000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->train(holder));
+
+  ASSERT_EQ(0, builder->build(holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "TestGeneral";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_EQ(stats.built_costtime(), 0UL);
+  // ASSERT_GT(stats.dumped_costtime(), 0UL);
+
+  // cleanup and rebuild
+  ASSERT_EQ(0, builder->cleanup());
+
+  auto holder2 = make_shared<MultiPassIndexSparseHolder<IndexMeta::DT_FP32>>();
+  size_t doc_cnt2 = 2000UL;
+  for (size_t i = 0; i < doc_cnt2; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder2->emplace(i, vec));
+  }
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+  ASSERT_EQ(0, builder->train(holder2));
+  ASSERT_EQ(0, builder->build(holder2));
+  auto dumper2 = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper2, nullptr);
+  ASSERT_EQ(0, dumper2->create(path));
+  ASSERT_EQ(0, builder->dump(dumper2));
+  ASSERT_EQ(0, dumper2->close());
+
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt2, stats.built_count());
+  ASSERT_EQ(doc_cnt2, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_EQ(stats.built_costtime(), 0UL);
+}
+
+TEST_F(FlatSparseBuilderTest, TestIndexThreads) {
+  IndexBuilder::Pointer builder1 =
+      IndexFactory::CreateBuilder("FlatSparseBuilder");
+  ASSERT_NE(builder1, nullptr);
+  IndexBuilder::Pointer builder2 =
+      IndexFactory::CreateBuilder("FlatSparseBuilder");
+  ASSERT_NE(builder2, nullptr);
+
+  auto holder = make_shared<MultiPassIndexSparseHolder<IndexMeta::DT_FP32>>();
+
+  size_t doc_cnt = 1000UL;
+  uint32_t sparse_count = 32;
+
+  for (size_t i = 0; i < doc_cnt; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+  std::srand(Realtime::MilliSeconds());
+  auto threads =
+      std::make_shared<SingleQueueIndexThreads>(std::rand() % 4, false);
+  ASSERT_EQ(0, builder1->init(*_index_meta_ptr, params));
+  ASSERT_EQ(0, builder2->init(*_index_meta_ptr, params));
+
+  auto build_index1 = [&]() {
+    ASSERT_EQ(0, builder1->train(threads, holder));
+    ASSERT_EQ(0, builder1->build(threads, holder));
+  };
+  auto build_index2 = [&]() {
+    ASSERT_EQ(0, builder2->train(threads, holder));
+    ASSERT_EQ(0, builder2->build(threads, holder));
+  };
+
+  auto t1 = std::async(std::launch::async, build_index1);
+  auto t2 = std::async(std::launch::async, build_index2);
+  t1.wait();
+  t2.wait();
+
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "TestIndexThreads";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder1->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder2->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats1 = builder1->stats();
+  ASSERT_EQ(doc_cnt, stats1.built_count());
+  auto &stats2 = builder2->stats();
+  ASSERT_EQ(doc_cnt, stats2.built_count());
+}
+
+TEST_F(FlatSparseBuilderTest, TestHalfFloatConverter) {
+  IndexBuilder::Pointer builder =
+      IndexFactory::CreateBuilder("FlatSparseBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder = make_shared<OnePassIndexSparseHolder<IndexMeta::DT_FP32>>();
+  uint32_t sparse_count = 4;
+  size_t doc_cnt = 1000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params converter_params;
+  auto converter = IndexFactory::CreateConverter("HalfFloatSparseConverter");
+  converter->init(*_index_meta_ptr, converter_params);
+
+  IndexMeta index_meta = converter->meta();
+
+  converter->transform(holder);
+
+  auto converted_holder = converter->sparse_result();
+
+  Params params;
+  ASSERT_EQ(0, builder->init(index_meta, converter_params));
+
+  ASSERT_EQ(0, builder->train(converted_holder));
+
+  ASSERT_EQ(0, builder->build(converted_holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "TestHalFloatConverter";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_EQ(stats.built_costtime(), 0UL);
+  // ASSERT_GT(stats.dumped_costtime(), 0UL);
+
+  // cleanup and rebuild
+  ASSERT_EQ(0, builder->cleanup());
+
+  auto holder2 = make_shared<MultiPassIndexSparseHolder<IndexMeta::DT_FP32>>();
+  size_t doc_cnt2 = 2000UL;
+  for (size_t i = 0; i < doc_cnt2; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder2->emplace(i, vec));
+  }
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+  ASSERT_EQ(0, builder->train(holder2));
+  ASSERT_EQ(0, builder->build(holder2));
+  auto dumper2 = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper2, nullptr);
+  ASSERT_EQ(0, dumper2->create(path));
+  ASSERT_EQ(0, builder->dump(dumper2));
+  ASSERT_EQ(0, dumper2->close());
+
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt2, stats.built_count());
+  ASSERT_EQ(doc_cnt2, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_EQ(stats.built_costtime(), 0UL);
+}
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc

@@ -0,0 +1,819 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <future>
+#include <iostream>
+#include <random>
+#include <vector>
+#include <ailego/math/norm2_matrix.h>
+#include <ailego/utility/math_helper.h>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/vector.h>
+#include "tests/test_util.h"
+#include "zvec/core/framework/index_factory.h"
+#include "zvec/core/framework/index_meta.h"
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace std;
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+constexpr size_t static sparse_dim_count = 16;
+
+class FlatSparseSearcherTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+  void generate_sparse_data(
+      size_t cnt, uint32_t sparse_dim_count,
+      std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
+      std::vector<NumericalVector<float>> &sparse_vec_list, bool norm);
+
+  static std::string dir_;
+  static std::shared_ptr<IndexMeta> index_meta_ptr_;
+};
+
+std::string FlatSparseSearcherTest::dir_("searcher_test/");
+std::shared_ptr<IndexMeta> FlatSparseSearcherTest::index_meta_ptr_;
+
+void FlatSparseSearcherTest::generate_sparse_data(
+    size_t cnt, uint32_t sparse_dim_count,
+    std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
+    std::vector<NumericalVector<float>> &sparse_vec_list, bool norm) {
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(-1.0, 1.0);
+
+  for (size_t i = 0; i < cnt; ++i) {
+    // prepare sparse
+    NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
+    NumericalVector<float> sparse_vec(sparse_dim_count);
+
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_vec[j] = dist(gen);
+    }
+
+    float norm;
+    Norm2Matrix<float, 1>::Compute(sparse_vec.data(), sparse_dim_count, &norm);
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_vec[j] = sparse_vec[j] / norm;
+    }
+
+    sparse_indices_list.push_back(sparse_indices);
+    sparse_vec_list.push_back(sparse_vec);
+  }
+}
+
+
+void FlatSparseSearcherTest::SetUp(void) {
+  IndexLoggerBroker::SetLevel(2);
+
+  index_meta_ptr_.reset(new IndexMeta(IndexMeta::MetaType::MT_SPARSE,
+                                      IndexMeta::DataType::DT_FP32));
+  index_meta_ptr_->set_metric("InnerProductSparse", 0, Params());
+
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+void FlatSparseSearcherTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+TEST_F(FlatSparseSearcherTest, TestGeneral) {
+  // init storage
+  Params stg_params;
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_TRUE(storage != nullptr);
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true));
+
+
+  // init streamer
+  IndexStreamer::Pointer streamer =
+      IndexFactory::CreateStreamer("FlatSparseStreamer");
+  ASSERT_TRUE(streamer != nullptr);
+
+  IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
+                       IndexMeta::DataType::DT_FP32);
+  index_meta.set_metric("InnerProductSparse", 0, Params());
+
+  Params params;
+  ASSERT_EQ(0, streamer->init(index_meta, params));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  // generate sparse data
+  size_t sparse_dim_count = 32;
+  size_t cnt = 100U;
+  std::vector<NumericalVector<uint32_t>> sparse_indices_list;
+  std::vector<NumericalVector<float>> sparse_vec_list;
+
+  generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
+                       sparse_vec_list, true);
+
+  // test add data
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32);
+  for (size_t i = 0; i < cnt; i++) {
+    ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count,
+                                    sparse_indices_list[i].data(),
+                                    sparse_vec_list[i].data(), qmeta, ctx));
+  }
+
+  // test get data
+  uint32_t sparse_count;
+  std::string sparse_indices_buffer;
+  std::string sparse_values_buffer;
+  for (size_t i = 0; i < cnt; i++) {
+    ASSERT_EQ(
+        0, streamer->get_sparse_vector(i, &sparse_count, &sparse_indices_buffer,
+                                       &sparse_values_buffer));
+    ASSERT_EQ(sparse_dim_count, sparse_count);
+    const uint32_t *sparse_indices_ptr =
+        reinterpret_cast<const uint32_t *>(sparse_indices_buffer.data());
+    const float *sparse_values_ptr =
+        reinterpret_cast<const float *>(sparse_values_buffer.data());
+    for (size_t j = 0; j < sparse_count; ++j) {
+      ASSERT_EQ(sparse_indices_ptr[j], sparse_indices_list[i][j]);
+      ASSERT_FLOAT_EQ(sparse_values_ptr[j], sparse_vec_list[i][j]);
+      // std::cout << "1: " << sparse_values_ptr[j]
+      //           << " 2: " << sparse_vec_list[i][j] << std::endl;
+    }
+
+    // must clear ^_^
+    sparse_indices_buffer.clear();
+    sparse_values_buffer.clear();
+  }
+
+  // test dump
+  auto path = dir_ + "TestGeneral_dump";
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, streamer->dump(dumper));
+  ASSERT_EQ(0, streamer->close());
+  ASSERT_EQ(0, dumper->close());
+
+  // do searcher get vector
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("FlatSparseSearcher");
+  auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_TRUE(read_storage != nullptr);
+  ASSERT_TRUE(searcher != nullptr);
+  ASSERT_EQ(0, read_storage->open(path, false));
+  ASSERT_EQ(0, searcher->init(Params()));
+  ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
+
+  // test searcher get data
+  for (size_t i = 0; i < cnt; i++) {
+    ASSERT_EQ(
+        0, searcher->get_sparse_vector(i, &sparse_count, &sparse_indices_buffer,
+                                       &sparse_values_buffer));
+    ASSERT_EQ(sparse_dim_count, sparse_count);
+    const uint32_t *sparse_indices_ptr =
+        reinterpret_cast<const uint32_t *>(sparse_indices_buffer.data());
+    const float *sparse_values_ptr =
+        reinterpret_cast<const float *>(sparse_values_buffer.data());
+    for (size_t j = 0; j < sparse_count; ++j) {
+      ASSERT_EQ(sparse_indices_ptr[j], sparse_indices_list[i][j]);
+      ASSERT_FLOAT_EQ(sparse_values_ptr[j], sparse_vec_list[i][j]);
+      // std::cout << "1: " << sparse_values_ptr[j]
+      //           << " 2: " << sparse_vec_list[i][j] << std::endl;
+    }
+
+    // must clear ^_^
+    sparse_indices_buffer.clear();
+    sparse_values_buffer.clear();
+  }
+}
+
+TEST_F(FlatSparseSearcherTest, TestStreamerDump) {
+  IndexStreamer::Pointer streamer =
+      IndexFactory::CreateStreamer("FlatSparseStreamer");
+  ASSERT_NE(streamer, nullptr);
+
+  Params params;
+  Params stg_params;
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestStreamerDump.index", true));
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t cnt = 10000U;
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  std::vector<NumericalVector<uint32_t>> sparse_indices_list;
+  std::vector<NumericalVector<float>> sparse_vec_list;
+
+  generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
+                       sparse_vec_list, true);
+
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32);
+  for (size_t i = 0; i < cnt; i++) {
+    ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count,
+                                    sparse_indices_list[i].data(),
+                                    sparse_vec_list[i].data(), qmeta, ctx));
+  }
+
+  auto path = dir_ + "TestStreamerDump";
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, streamer->dump(dumper));
+  ASSERT_EQ(0, streamer->close());
+  ASSERT_EQ(0, dumper->close());
+
+  // do searcher knn
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("FlatSparseSearcher");
+  auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_EQ(0, read_storage->open(path, false));
+  ASSERT_TRUE(searcher != nullptr);
+  ASSERT_EQ(0, searcher->init(Params()));
+  ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
+  auto linearCtx = searcher->create_context();
+  auto knnCtx = searcher->create_context();
+  size_t topk = 200;
+  linearCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+  uint64_t knnTotalTime = 0;
+  uint64_t linearTotalTime = 0;
+
+  for (size_t i = 0; i < cnt; i += 50) {
+    const auto &sparse_indices = sparse_indices_list[i];
+    const auto &sparse_vec = sparse_vec_list[i];
+
+    auto t1 = Realtime::MicroSeconds();
+
+    ASSERT_EQ(0, searcher->search_impl(sparse_dim_count, sparse_indices.data(),
+                                       sparse_vec.data(), qmeta, knnCtx));
+
+    auto t2 = Realtime::MicroSeconds();
+
+    ASSERT_EQ(0,
+              searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
+                                       sparse_vec.data(), qmeta, linearCtx));
+
+    auto t3 = Realtime::MicroSeconds();
+
+    knnTotalTime += t2 - t1;
+    linearTotalTime += t3 - t2;
+
+    auto &knnResult = knnCtx->result();
+    auto &linearResult = linearCtx->result();
+
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(topk, knnResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      ASSERT_EQ(linearResult[k].key(), knnResult[k].key());
+    }
+  }
+
+  printf("linear: %zu, knn: %zu\n", (size_t)linearTotalTime,
+         (size_t)knnTotalTime);
+}
+
+TEST_F(FlatSparseSearcherTest, TestLoadClose) {
+  Params stg_params;
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_TRUE(storage != nullptr);
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true));
+
+
+  // init streamer
+  IndexStreamer::Pointer streamer =
+      IndexFactory::CreateStreamer("FlatSparseStreamer");
+  ASSERT_TRUE(streamer != nullptr);
+
+  IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
+                       IndexMeta::DataType::DT_FP32);
+  index_meta.set_metric("InnerProductSparse", 0, Params());
+
+  Params params;
+  ASSERT_EQ(0, streamer->init(index_meta, params));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  // generate sparse data
+  size_t sparse_dim_count = 32;
+  size_t cnt = 100U;
+  std::vector<NumericalVector<uint32_t>> sparse_indices_list;
+  std::vector<NumericalVector<float>> sparse_vec_list;
+
+  generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
+                       sparse_vec_list, true);
+
+  // test add data
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32);
+  for (size_t i = 0; i < cnt; i++) {
+    ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count,
+                                    sparse_indices_list[i].data(),
+                                    sparse_vec_list[i].data(), qmeta, ctx));
+  }
+
+  // test get data
+  uint32_t sparse_count;
+  std::string sparse_indices_buffer;
+  std::string sparse_values_buffer;
+  for (size_t i = 0; i < cnt; i++) {
+    ASSERT_EQ(
+        0, streamer->get_sparse_vector(i, &sparse_count, &sparse_indices_buffer,
+                                       &sparse_values_buffer));
+    ASSERT_EQ(sparse_dim_count, sparse_count);
+    const uint32_t *sparse_indices_ptr =
+        reinterpret_cast<const uint32_t *>(sparse_indices_buffer.data());
+    const float *sparse_values_ptr =
+        reinterpret_cast<const float *>(sparse_values_buffer.data());
+    for (size_t j = 0; j < sparse_count; ++j) {
+      ASSERT_EQ(sparse_indices_ptr[j], sparse_indices_list[i][j]);
+      ASSERT_FLOAT_EQ(sparse_values_ptr[j], sparse_vec_list[i][j]);
+      // std::cout << "1: " << sparse_values_ptr[j]
+      //           << " 2: " << sparse_vec_list[i][j] << std::endl;
+    }
+
+    // must clear ^_^
+    sparse_indices_buffer.clear();
+    sparse_values_buffer.clear();
+  }
+
+  // test dump
+  auto path = dir_ + "TestGeneral_dump";
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, streamer->dump(dumper));
+  ASSERT_EQ(0, streamer->close());
+  ASSERT_EQ(0, dumper->close());
+
+  // do searcher get vector
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("FlatSparseSearcher");
+  auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_TRUE(read_storage != nullptr);
+  ASSERT_TRUE(searcher != nullptr);
+  ASSERT_EQ(0, read_storage->open(path, false));
+  ASSERT_EQ(0, searcher->init(Params()));
+
+  uint32_t loop = 5;
+  while (loop--) {
+    std::cout << "loop: " << loop << std::endl;
+
+    ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
+
+    // test searcher get data
+    for (size_t i = 0; i < cnt; i++) {
+      ASSERT_EQ(0, searcher->get_sparse_vector(i, &sparse_count,
+                                               &sparse_indices_buffer,
+                                               &sparse_values_buffer));
+      ASSERT_EQ(sparse_dim_count, sparse_count);
+      const uint32_t *sparse_indices_ptr =
+          reinterpret_cast<const uint32_t *>(sparse_indices_buffer.data());
+      const float *sparse_values_ptr =
+          reinterpret_cast<const float *>(sparse_values_buffer.data());
+      for (size_t j = 0; j < sparse_count; ++j) {
+        ASSERT_EQ(sparse_indices_ptr[j], sparse_indices_list[i][j]);
+        ASSERT_FLOAT_EQ(sparse_values_ptr[j], sparse_vec_list[i][j]);
+        // std::cout << "1: " << sparse_values_ptr[j]
+        //           << " 2: " << sparse_vec_list[i][j] << std::endl;
+      }
+
+      // must clear ^_^
+      sparse_indices_buffer.clear();
+      sparse_values_buffer.clear();
+    }
+
+    ASSERT_EQ(searcher->unload(), 0);
+  }
+}
+
+TEST_F(FlatSparseSearcherTest, TestSearch) {
+  IndexStreamer::Pointer streamer =
+      IndexFactory::CreateStreamer("FlatSparseStreamer");
+  ASSERT_TRUE(streamer != nullptr);
+
+  Params params;
+  Params stg_params;
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true));
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t cnt = 5000UL;
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32);
+
+  NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
+  NumericalVector<float> sparse_velues(sparse_dim_count);
+  for (size_t i = 0; i < cnt; ++i) {
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_velues[j] = -1.0 * i - 1.0f;
+    }
+
+    ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count, sparse_indices.data(),
+                                    sparse_velues.data(), qmeta, ctx));
+  }
+
+  // test dump
+  auto path = dir_ + "TestGeneral_dump";
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, streamer->dump(dumper));
+  ASSERT_EQ(0, streamer->close());
+  ASSERT_EQ(0, dumper->close());
+
+  // do searcher get vector
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("FlatSparseSearcher");
+  auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_TRUE(read_storage != nullptr);
+  ASSERT_TRUE(searcher != nullptr);
+  ASSERT_EQ(0, read_storage->open(path, false));
+  ASSERT_EQ(0, searcher->init(Params()));
+  ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
+
+  size_t step = 50;
+  for (size_t i = 0; i < cnt; i += step) {
+    // std::cout << "search " << i << std::endl;
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_velues[j] = i + 1.0f;
+    }
+
+    ctx->set_topk(1U);
+    ASSERT_EQ(0,
+              searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
+                                       sparse_velues.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(1UL, result1.size());
+    ASSERT_EQ(0, result1[0].key());
+    // std::cout << result1[0].key() << " " << result1[0].score() << std::endl;
+
+    ctx->set_topk(3U);
+    ASSERT_EQ(0,
+              searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
+                                       sparse_velues.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(3UL, result2.size());
+    for (size_t i = 0; i < 3UL; ++i) {
+      // std::cout << result2[i].key() << " " << result2[i].score() <<
+      // std::endl;
+      ASSERT_EQ(i, result2[i].key());
+    }
+  }
+
+  ctx->set_topk(100U);
+  for (size_t j = 0; j < sparse_dim_count; ++j) {
+    sparse_indices[j] = j * 20;
+    sparse_velues[j] = 10.1f;
+  }
+
+  ASSERT_EQ(0, searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
+                                        sparse_velues.data(), qmeta, ctx));
+  auto &result = ctx->result();
+  ASSERT_EQ(100U, result.size());
+  for (size_t i = 0; i < 100; ++i) {
+    ASSERT_EQ(i, result[i].key());
+  }
+}
+
+TEST_F(FlatSparseSearcherTest, TestSearchPKeys) {
+  IndexStreamer::Pointer streamer =
+      IndexFactory::CreateStreamer("FlatSparseStreamer");
+  ASSERT_TRUE(streamer != nullptr);
+
+  Params params;
+  Params stg_params;
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchByKeys.index", true));
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t cnt = 5000UL;
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32);
+
+  std::vector<std::vector<uint64_t>> p_keys;
+  p_keys.resize(1);
+  p_keys[0].resize(cnt);
+
+  NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
+  NumericalVector<float> sparse_velues(sparse_dim_count);
+  for (size_t i = 0; i < cnt; ++i) {
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_velues[j] = -1.0 * i - 1.0f;
+    }
+
+    ASSERT_EQ(0, streamer->add_impl(i, sparse_dim_count, sparse_indices.data(),
+                                    sparse_velues.data(), qmeta, ctx));
+
+    p_keys[0][i] = i;
+  }
+
+  // test dump
+  auto path = dir_ + "TestGeneral_dump";
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, streamer->dump(dumper));
+  ASSERT_EQ(0, streamer->close());
+  ASSERT_EQ(0, dumper->close());
+
+  // do searcher get vector
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("FlatSparseSearcher");
+  auto read_storage = IndexFactory::CreateStorage("FileReadStorage");
+  ASSERT_TRUE(read_storage != nullptr);
+  ASSERT_TRUE(searcher != nullptr);
+  ASSERT_EQ(0, read_storage->open(path, false));
+  ASSERT_EQ(0, searcher->init(Params()));
+  ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
+
+  size_t topk = 3;
+  for (size_t i = 0; i < cnt; i += 50) {
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_velues[j] = i + 1.0f;
+    }
+    ctx->set_topk(1U);
+    ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
+                     sparse_dim_count, sparse_indices.data(),
+                     sparse_velues.data(), p_keys, qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(1UL, result1.size());
+    ASSERT_EQ(0, result1[0].key());
+
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
+                     sparse_dim_count, sparse_indices.data(),
+                     sparse_velues.data(), p_keys, qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(0, result2[0].key());
+    ASSERT_EQ(1, result2[1].key());
+    ASSERT_EQ(2, result2[2].key());
+  }
+
+  {
+    ctx->set_topk(100U);
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_velues[j] = 1.0f;
+    }
+    ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
+                     sparse_dim_count, sparse_indices.data(),
+                     sparse_velues.data(), p_keys, qmeta, ctx));
+    auto &result = ctx->result();
+    ASSERT_EQ(100U, result.size());
+    ASSERT_EQ(0, result[0].key());
+    ASSERT_EQ(1, result[1].key());
+    ASSERT_EQ(10, result[10].key());
+    ASSERT_EQ(20, result[20].key());
+    ASSERT_EQ(30, result[30].key());
+    ASSERT_EQ(35, result[35].key());
+    ASSERT_EQ(99, result[99].key());
+  }
+
+  {
+    ctx->set_topk(100U);
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_velues[j] = 10.0f;
+    }
+
+    p_keys[0] = {{cnt + 1, 10, 1, 15, cnt + 2}};
+    ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
+                     sparse_dim_count, sparse_indices.data(),
+                     sparse_velues.data(), p_keys, qmeta, ctx));
+    auto &result = ctx->result();
+    ASSERT_EQ(3U, result.size());
+    ASSERT_EQ(1, result[0].key());
+    ASSERT_EQ(10, result[1].key());
+    ASSERT_EQ(15, result[2].key());
+  }
+
+  {
+    ctx->set_topk(100U);
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_velues[j] = 9.0f;
+    }
+    p_keys[0].clear();
+    for (size_t j = 0; j < cnt; j += 10) {
+      p_keys[0].push_back((uint64_t)j);
+    }
+    ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
+                     sparse_dim_count, sparse_indices.data(),
+                     sparse_velues.data(), p_keys, qmeta, ctx));
+    auto &result = ctx->result();
+    ASSERT_EQ(100U, result.size());
+    ASSERT_EQ(0, result[0].key());
+    ASSERT_EQ(10, result[1].key());
+    ASSERT_EQ(100, result[10].key());
+    ASSERT_EQ(200, result[20].key());
+    ASSERT_EQ(300, result[30].key());
+    ASSERT_EQ(350, result[35].key());
+    ASSERT_EQ(990, result[99].key());
+  }
+}
+
+TEST_F(FlatSparseSearcherTest, TestMultiThread) {
+  IndexStreamer::Pointer streamer =
+      IndexFactory::CreateStreamer("FlatSparseStreamer");
+  ASSERT_TRUE(streamer != nullptr);
+
+  Params params;
+  constexpr size_t static sparse_dim_count = 32;
+  IndexMeta meta(IndexMeta::MetaType::MT_SPARSE, IndexMeta::DataType::DT_FP32);
+  meta.set_metric("InnerProductSparse", 0, Params());
+  ASSERT_EQ(0, streamer->init(meta, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto addVector = [&streamer](int baseKey, size_t addCnt) {
+    IndexQueryMeta qmeta(IndexMeta::DT_FP32);
+    size_t succAdd = 0;
+    auto ctx = streamer->create_context();
+    for (size_t i = 0; i < addCnt; i++) {
+      NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
+      NumericalVector<float> sparse_velues(sparse_dim_count);
+
+      for (size_t j = 0; j < sparse_dim_count; ++j) {
+        sparse_indices[j] = j * 20;
+        sparse_velues[j] = (float)i + baseKey;
+      }
+
+      succAdd += !streamer->add_impl(baseKey + i, sparse_dim_count,
+                                     sparse_indices.data(),
+                                     sparse_velues.data(), qmeta, ctx);
+    }
+    streamer->flush(0UL);
+    return succAdd;
+  };
+
+  auto t2 = std::async(std::launch::async, addVector, 1000, 1000);
+  auto t3 = std::async(std::launch::async, addVector, 2000, 1000);
+  auto t1 = std::async(std::launch::async, addVector, 0, 1000);
+  ASSERT_EQ(1000U, t1.get());
+  ASSERT_EQ(1000U, t2.get());
+  ASSERT_EQ(1000U, t3.get());
+  streamer->close();
+
+  // checking data
+  ASSERT_EQ(0, streamer->open(storage));
+  auto provider = streamer->create_sparse_provider();
+  auto iter = provider->create_iterator();
+  ASSERT_TRUE(!!iter);
+  size_t total = 0;
+  uint64_t min = 1000;
+  uint64_t max = 0;
+
+  std::set<uint64_t> keys;
+
+  while (iter->is_valid()) {
+    const uint32_t sparse_count = iter->sparse_count();
+    ASSERT_EQ(sparse_count, sparse_dim_count);
+
+    const float *data = reinterpret_cast<const float *>(iter->sparse_data());
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      ASSERT_EQ((float)iter->key(), data[j]);
+    }
+    total++;
+    min = std::min(min, iter->key());
+    max = std::max(max, iter->key());
+    keys.insert(iter->key());
+    iter->next();
+  }
+
+  ASSERT_EQ(3000, keys.size());
+
+  ASSERT_EQ(3000, total);
+  ASSERT_EQ(0, min);
+  ASSERT_EQ(2999, max);
+
+  // test dump
+  auto path = dir_ + "TestGeneral_dump";
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, streamer->dump(dumper));
+  ASSERT_EQ(0, streamer->close());
+  ASSERT_EQ(0, dumper->close());
+
+  // do searcher get vector
+  IndexSearcher::Pointer searcher =
+      IndexFactory::CreateSearcher("FlatSparseSearcher");
+  auto read_storage = IndexFactory::CreateStorage("MMapFileReadStorage");
+  ASSERT_TRUE(read_storage != nullptr);
+  ASSERT_TRUE(searcher != nullptr);
+  ASSERT_EQ(0, read_storage->open(path, false));
+  ASSERT_EQ(0, searcher->init(Params()));
+  ASSERT_EQ(0, searcher->load(read_storage, IndexMetric::Pointer()));
+
+  // ====== multi thread search
+  size_t topk = 10;
+  size_t cnt = 3000;
+  auto knnSearch = [&]() {
+    auto linearCtx = searcher->create_context();
+    auto linearByPkeysCtx = searcher->create_context();
+    auto ctx = searcher->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DT_FP32);
+    linearCtx->set_topk(topk);
+    linearByPkeysCtx->set_topk(topk);
+    ctx->set_topk(topk);
+    size_t totalCnts = 0;
+    size_t totalHits = 0;
+    for (size_t i = 0; i < cnt; i += 1) {
+      NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
+      NumericalVector<float> sparse_velues(sparse_dim_count);
+
+      for (size_t j = 0; j < sparse_dim_count; ++j) {
+        sparse_indices[j] = j * 20;
+        sparse_velues[j] = ((float)i + 1.1f);
+      }
+
+      ASSERT_EQ(0,
+                searcher->search_impl(sparse_dim_count, sparse_indices.data(),
+                                      sparse_velues.data(), qmeta, ctx));
+      ASSERT_EQ(
+          0, searcher->search_bf_impl(sparse_dim_count, sparse_indices.data(),
+                                      sparse_velues.data(), qmeta, linearCtx));
+      std::vector<std::vector<uint64_t>> p_keys = {{cnt - 1, cnt - 2, cnt - 3}};
+      ASSERT_EQ(0, searcher->search_bf_by_p_keys_impl(
+                       sparse_dim_count, sparse_indices.data(),
+                       sparse_velues.data(), p_keys, qmeta, linearByPkeysCtx));
+      auto &r1 = ctx->result();
+      ASSERT_EQ(topk, r1.size());
+      // std::cout << "r1 top1: " << r1[0].key() << ", score: " << r1[0].score()
+      //           << std::endl;
+      ASSERT_EQ(cnt - 1, r1[0].key());
+      auto &r2 = linearCtx->result();
+      ASSERT_EQ(topk, r2.size());
+      // std::cout << "r2 top1: " << r2[0].key() << ", score: " << r2[0].score()
+      //           << std::endl;
+      ASSERT_EQ(cnt - 1, r2[0].key());
+      auto &r3 = linearByPkeysCtx->result();
+      ASSERT_EQ(std::min(topk, p_keys[0].size()), r3.size());
+#if 0
+            printf("linear: %zd => %zd %zd %zd %zd %zd\n", i, r2[0].key,
+                   r2[1].key, r2[2].key, r2[3].key, r2[4].key);
+            printf("knn: %zd => %zd %zd %zd %zd %zd\n", i, r1[0].key, r1[1].key,
+                   r1[2].key, r1[3].key, r1[4].key);
+#endif
+      for (size_t k = 0; k < topk; ++k) {
+        totalCnts++;
+        for (size_t j = 0; j < topk; ++j) {
+          if (r2[j].key() == r1[k].key()) {
+            totalHits++;
+            break;
+          }
+        }
+      }
+    }
+    printf("%f\n", totalHits * 1.0f / totalCnts);
+    ASSERT_FLOAT_EQ(1.0f, totalHits * 1.0f / totalCnts);
+  };
+
+  auto s1 = std::async(std::launch::async, knnSearch);
+  auto s2 = std::async(std::launch::async, knnSearch);
+  auto s3 = std::async(std::launch::async, knnSearch);
+  s1.wait();
+  s2.wait();
+  s3.wait();
+}
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc

@@ -0,0 +1,230 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+#include <ailego/math/norm2_matrix.h>
+#include <ailego/utility/math_helper.h>
+#include <ailego/utility/memory_helper.h>
+#include <algorithm/flat_sparse/flat_sparse_utility.h>
+#include <gtest/gtest.h>
+#include <zvec/core/framework/index_framework.h>
+#include <zvec/core/framework/index_streamer.h>
+#include "tests/test_util.h"
+
+using namespace std;
+using namespace testing;
+using namespace zvec::ailego;
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+namespace zvec {
+namespace core {
+
+class FlatSparseStreamerTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+  void generate_sparse_data(
+      size_t cnt, uint32_t sparse_dim_count,
+      std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
+      std::vector<NumericalVector<float>> &sparse_vec_list, bool norm);
+
+  static std::string dir_;
+  static shared_ptr<IndexMeta> index_meta_ptr_;
+};
+
+std::string FlatSparseStreamerTest::dir_("FlatSparseStreamerTest/");
+shared_ptr<IndexMeta> FlatSparseStreamerTest::index_meta_ptr_;
+
+void FlatSparseStreamerTest::generate_sparse_data(
+    size_t cnt, uint32_t sparse_dim_count,
+    std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
+    std::vector<NumericalVector<float>> &sparse_vec_list, bool norm) {
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(-1.0, 1.0);
+
+  for (size_t i = 0; i < cnt; ++i) {
+    // prepare sparse
+    NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
+    NumericalVector<float> sparse_vec(sparse_dim_count);
+
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_vec[j] = dist(gen);
+    }
+
+    float norm;
+    ailego::Norm2Matrix<float, 1>::Compute(sparse_vec.data(), sparse_dim_count,
+                                           &norm);
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_vec[j] = sparse_vec[j] / norm;
+    }
+
+    sparse_indices_list.push_back(sparse_indices);
+    sparse_vec_list.push_back(sparse_vec);
+  }
+}
+
+void FlatSparseStreamerTest::SetUp(void) {
+  index_meta_ptr_.reset(new (nothrow) IndexMeta(IndexMeta::MetaType::MT_SPARSE,
+                                                IndexMeta::DataType::DT_FP32));
+  index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params());
+
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+void FlatSparseStreamerTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+TEST_F(FlatSparseStreamerTest, TestGeneral) {
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("FlatSparseStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  size_t sparse_dim_count = 32;
+
+  IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
+                       IndexMeta::DataType::DT_FP32);
+  index_meta.set_metric("InnerProductSparse", 0, ailego::Params());
+
+  ailego::Params params;
+
+  ailego::Params stg_params;
+  auto write_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_EQ(0, write_storage->init(stg_params));
+  ASSERT_EQ(0, write_storage->open(dir_ + "Test/FlatSparseSearch", true));
+  ASSERT_EQ(0, write_streamer->init(index_meta, params));
+  ASSERT_EQ(0, write_streamer->open(write_storage));
+
+  size_t cnt = 20000U;
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  std::vector<NumericalVector<uint32_t>> sparse_indices_list;
+  std::vector<NumericalVector<float>> sparse_vec_list;
+
+  generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
+                       sparse_vec_list, true);
+
+  IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE,
+                       IndexMeta::DataType::DT_FP32);
+  for (size_t i = 0; i < cnt; i++) {
+    ASSERT_EQ(0, write_streamer->add_impl(
+                     i, sparse_dim_count, sparse_indices_list[i].data(),
+                     sparse_vec_list[i].data(), qmeta, ctx));
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  write_storage->close();
+  write_storage.reset();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("FlatSparseStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/FlatSparseSearch", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+
+  auto linearCtx = read_streamer->create_context();
+  ASSERT_TRUE(!!linearCtx);
+
+  auto knnCtx = read_streamer->create_context();
+  ASSERT_TRUE(!!knnCtx);
+
+  // streamer->print_debug_info();
+  size_t topk = 200;
+  linearCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+
+  uint64_t knnTotalTime = 0;
+  uint64_t linearTotalTime = 0;
+
+  int totalHits = 0;
+  int totalCnts = 0;
+  int topk1Hits = 0;
+
+  for (size_t i = 0; i < cnt; i += 100) {
+    const auto &sparse_indices = sparse_indices_list[i];
+    const auto &sparse_vec = sparse_vec_list[i];
+
+    auto t1 = ailego::Realtime::MicroSeconds();
+
+    ASSERT_EQ(
+        0, read_streamer->search_impl(sparse_dim_count, sparse_indices.data(),
+                                      sparse_vec.data(), qmeta, knnCtx));
+
+    auto t2 = ailego::Realtime::MicroSeconds();
+
+    ASSERT_EQ(0, read_streamer->search_bf_impl(
+                     sparse_dim_count, sparse_indices.data(), sparse_vec.data(),
+                     qmeta, linearCtx));
+
+    auto t3 = ailego::Realtime::MicroSeconds();
+
+    knnTotalTime += t2 - t1;
+    linearTotalTime += t3 - t2;
+
+    // std::cout << "i: " << i << std::endl;
+
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+    topk1Hits += i == knnResult[0].key();
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      totalCnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linearResult[j].key() == knnResult[k].key()) {
+          totalHits++;
+          break;
+        }
+      }
+    }
+  }
+  float recall = totalHits * 1.0f / totalCnts;
+  float topk1Recall = topk1Hits * 100.0f / cnt;
+  // float cost = linearTotalTime * 1.0f / knnTotalTime;
+
+  std::cout << "knnTotalTime=" << knnTotalTime
+            << " linearTotalTime=" << linearTotalTime << std::endl;
+
+#if 0
+    printf("knnTotalTime=%zd linearTotalTime=%zd totalHits=%d totalCnts=%d "
+           "R@%zd=%f R@1=%f cost=%f\n",
+           knnTotalTime, linearTotalTime, totalHits, totalCnts, topk, recall,
+           topk1Recall, cost);
+#endif
+  EXPECT_GT(recall, 0.80f);
+  EXPECT_GT(topk1Recall, 0.80f);
+  // EXPECT_GT(cost, 2.0f);
+}
+
+}  // namespace core
+}  // namespace zvec
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/algorithm/hnsw/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_hnsw core_knn_flat
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm/hnsw
+    )
+endforeach()

tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cc

@@ -0,0 +1,517 @@
+#include <future>
+#include <string>
+#include <vector>
+#include <ailego/utility/math_helper.h>
+#include <ailego/utility/memory_helper.h>
+#include <algorithm/hnsw/hnsw_params.h>
+#include <gtest/gtest.h>
+#include <zvec/core/framework/index_framework.h>
+#include <zvec/core/framework/index_streamer.h>
+#include "tests/test_util.h"
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace std;
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+constexpr size_t static dim = 16;
+
+class HnswStreamerTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+  void hybrid_scale(std::vector<float> &dense_value,
+                    std::vector<float> &sparse_value, float alpha_scale);
+
+  static std::string dir_;
+  static std::shared_ptr<IndexMeta> index_meta_ptr_;
+};
+
+std::string HnswStreamerTest::dir_("hnsw_streamer_buffer_test_dir/");
+std::shared_ptr<IndexMeta> HnswStreamerTest::index_meta_ptr_;
+
+void HnswStreamerTest::SetUp(void) {
+  index_meta_ptr_.reset(new (std::nothrow)
+                            IndexMeta(IndexMeta::DataType::DT_FP32, dim));
+  index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params());
+
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+void HnswStreamerTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+TEST_F(HnswStreamerTest, TestHnswSearch) {
+  MemoryLimitPool::get_instance().init(2 * 1024UL * 1024UL * 1024UL);
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("HnswStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  params.set(PARAM_HNSW_STREAMER_GET_VECTOR_ENABLE, true);
+
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "Test/HnswSearch", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t cnt = 10000UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  storage->close();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("HnswStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSearch", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 3;
+  auto provider = read_streamer->create_provider();
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  ctx->set_topk(100U);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 10.1f;
+  }
+  ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
+  auto &result = ctx->result();
+  ASSERT_EQ(100U, result.size());
+  ASSERT_EQ(10, result[0].key());
+  ASSERT_EQ(11, result[1].key());
+  ASSERT_EQ(5, result[10].key());
+  ASSERT_EQ(0, result[20].key());
+  ASSERT_EQ(30, result[30].key());
+  ASSERT_EQ(35, result[35].key());
+  ASSERT_EQ(99, result[99].key());
+
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  read_streamer->close();
+  read_streamer.reset();
+  cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
+}
+
+TEST_F(HnswStreamerTest, TestHnswSearchBuffer) {
+  MemoryLimitPool::get_instance().init(2 * 1024UL * 1024UL * 1024UL);
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("HnswStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  params.set(PARAM_HNSW_STREAMER_GET_VECTOR_ENABLE, true);
+
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "Test/TestHnswSearchBuffer", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t cnt = 10000UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  storage->close();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("HnswStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/TestHnswSearchBuffer", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 3;
+  auto provider = read_streamer->create_provider();
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  ctx->set_topk(100U);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 10.1f;
+  }
+  ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
+  auto &result = ctx->result();
+  ASSERT_EQ(100U, result.size());
+  ASSERT_EQ(10, result[0].key());
+  ASSERT_EQ(11, result[1].key());
+  ASSERT_EQ(5, result[10].key());
+  ASSERT_EQ(0, result[20].key());
+  ASSERT_EQ(30, result[30].key());
+  ASSERT_EQ(35, result[35].key());
+  ASSERT_EQ(99, result[99].key());
+
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  read_streamer->close();
+  read_streamer.reset();
+  cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
+}
+
+TEST_F(HnswStreamerTest, TestHnswSearchBufferMMap) {
+  MemoryLimitPool::get_instance().init(2 * 1024UL * 1024UL * 1024UL);
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("HnswStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  params.set(PARAM_HNSW_STREAMER_GET_VECTOR_ENABLE, true);
+
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "Test/TestHnswSearchBufferMMap", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t cnt = 10000UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  storage->close();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("HnswStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/TestHnswSearchBufferMMap", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 3;
+  auto provider = read_streamer->create_provider();
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  ctx->set_topk(100U);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 10.1f;
+  }
+  ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
+  auto &result = ctx->result();
+  ASSERT_EQ(100U, result.size());
+  ASSERT_EQ(10, result[0].key());
+  ASSERT_EQ(11, result[1].key());
+  ASSERT_EQ(5, result[10].key());
+  ASSERT_EQ(0, result[20].key());
+  ASSERT_EQ(30, result[30].key());
+  ASSERT_EQ(35, result[35].key());
+  ASSERT_EQ(99, result[99].key());
+
+  ElapsedTime elapsed_time;
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  read_streamer->close();
+  read_streamer.reset();
+  cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
+}
+
+TEST_F(HnswStreamerTest, TestHnswSearchMMap) {
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("HnswStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  Params params;
+  params.set(PARAM_HNSW_STREAMER_GET_VECTOR_ENABLE, true);
+
+  ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "Test/HnswSearchMMap", true));
+  ASSERT_EQ(0, write_streamer->open(storage));
+
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  size_t cnt = 10000UL;
+  IndexQueryMeta qmeta(IndexMeta::DT_FP32, dim);
+  for (size_t i = 0; i < cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    write_streamer->add_impl(i, vec.data(), qmeta, ctx);
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  storage->close();
+
+  ElapsedTime elapsed_time;
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("HnswStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSearchMMap", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+  size_t topk = 3;
+  auto provider = read_streamer->create_provider();
+  for (size_t i = 0; i < cnt; i += 1) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(topk, result1.size());
+    IndexStorage::MemoryBlock block;
+    ASSERT_EQ(0, provider->get_vector(result1[0].key(), block));
+    const float *data = (float *)block.data();
+    for (size_t j = 0; j < dim; ++j) {
+      ASSERT_FLOAT_EQ(data[j], i);
+    }
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, read_streamer->search_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+
+  ctx->set_topk(100U);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 10.1f;
+  }
+  ASSERT_EQ(0, read_streamer->search_bf_impl(vec.data(), qmeta, ctx));
+  auto &result = ctx->result();
+  ASSERT_EQ(100U, result.size());
+  ASSERT_EQ(10, result[0].key());
+  ASSERT_EQ(11, result[1].key());
+  ASSERT_EQ(5, result[10].key());
+  ASSERT_EQ(0, result[20].key());
+  ASSERT_EQ(30, result[30].key());
+  ASSERT_EQ(35, result[35].key());
+  ASSERT_EQ(99, result[99].key());
+
+  read_streamer->close();
+  read_streamer.reset();
+  cout << "Elapsed time: " << elapsed_time.milli_seconds() << " ms" << endl;
+}
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/algorithm/hnsw_rabitq/CMakeLists.txt

@@ -0,0 +1,34 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
+
+if(APPLE)
+  set(APPLE_FRAMEWORK_LIBS
+    -framework CoreFoundation
+    -framework CoreGraphics
+    -framework CoreData
+    -framework CoreText
+    -framework Security
+    -framework Foundation
+    -Wl,-U,_MallocExtension_ReleaseFreeMemory
+    -Wl,-U,_ProfilerStart
+    -Wl,-U,_ProfilerStop
+    -Wl,-U,_RegisterThriftProtocol
+  )
+endif()
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_hnsw_rabitq core_knn_flat core_knn_cluster
+      ${CMAKE_THREAD_LIBS_INIT}
+      ${CMAKE_DL_LIBS}
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm/hnsw_rabitq
+      LDFLAGS ${APPLE_FRAMEWORK_LIBS}
+    )
+  cc_test_suite(hnsw_rabitq ${CC_TARGET})
+endforeach()

tests/core/algorithm/hnsw_rabitq/hnsw_rabitq_streamer_test.cc

@@ -0,0 +1,660 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "hnsw_rabitq_streamer.h"
+#include <memory>
+#include <string>
+#include <gtest/gtest.h>
+#include "zvec/ailego/container/params.h"
+#include "zvec/ailego/utility/file_helper.h"
+#include "zvec/core/framework/index_holder.h"
+#include "zvec/core/framework/index_streamer.h"
+#include "hnsw_rabitq_streamer.h"
+#include "rabitq_converter.h"
+#include "rabitq_reformer.h"
+
+using namespace std;
+using namespace zvec::ailego;
+
+namespace zvec {
+namespace core {
+
+constexpr size_t static dim = 128;
+
+class HnswRabitqStreamerTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+
+  static std::string dir_;
+  static shared_ptr<IndexMeta> index_meta_ptr_;
+};
+
+std::string HnswRabitqStreamerTest::dir_("hnswRabitqStreamerTest");
+shared_ptr<IndexMeta> HnswRabitqStreamerTest::index_meta_ptr_;
+
+void HnswRabitqStreamerTest::SetUp(void) {
+  index_meta_ptr_.reset(new (nothrow)
+                            IndexMeta(IndexMeta::DataType::DT_FP32, dim));
+  index_meta_ptr_->set_metric("SquaredEuclidean", 0, ailego::Params());
+}
+
+void HnswRabitqStreamerTest::TearDown(void) {
+  ailego::FileHelper::RemovePath(dir_.c_str());
+}
+
+TEST_F(HnswRabitqStreamerTest, TestBuildAndSearch) {
+  auto holder =
+      make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 1000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  RabitqConverter converter;
+  converter.init(*index_meta_ptr_, ailego::Params());
+  ASSERT_EQ(converter.train(holder), 0);
+  std::shared_ptr<IndexReformer> index_reformer;
+  ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
+  auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
+  IndexStreamer::Pointer streamer =
+      std::make_shared<HnswRabitqStreamer>(holder, reformer);
+
+  ailego::Params params;
+  params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
+  params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
+  params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
+  params.set("proxima.hnsw_rabitq.general.dimension", dim);
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/Test/AddVector", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto context = streamer->create_context();
+  for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
+    IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+    ASSERT_EQ(0,
+              streamer->add_impl(it->key(), it->data(), query_meta, context));
+  }
+  streamer->flush(0UL);
+
+  // Perform search verification
+  NumericalVector<float> query_vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    query_vec[j] = static_cast<float>(j) / 1000.0f;
+  }
+
+  IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+
+  context->set_topk(10);
+  ASSERT_EQ(0, streamer->search_impl(query_vec.data(), query_meta, 1, context));
+
+  const auto &result = context->result(0);
+  ASSERT_GT(result.size(), 0UL);
+  ASSERT_LE(result.size(), 10UL);
+
+  // reopen and load reformer from storage
+  ASSERT_EQ(0, streamer->close());
+  IndexStreamer::Pointer new_streamer =
+      std::make_shared<HnswRabitqStreamer>(holder);
+  ASSERT_EQ(0, new_streamer->init(*index_meta_ptr_, params));
+  ASSERT_EQ(0, new_streamer->open(storage));
+}
+
+TEST_F(HnswRabitqStreamerTest, TestLinearSearch) {
+  auto holder =
+      make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 1000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  RabitqConverter converter;
+  converter.init(*index_meta_ptr_, ailego::Params());
+  ASSERT_EQ(converter.train(holder), 0);
+  std::shared_ptr<IndexReformer> index_reformer;
+  ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
+  auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
+  IndexStreamer::Pointer streamer =
+      std::make_shared<HnswRabitqStreamer>(holder, reformer);
+
+  ailego::Params params;
+  params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
+  params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
+  params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
+  params.set("proxima.hnsw_rabitq.general.dimension", dim);
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto context = streamer->create_context();
+  for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
+    IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+    ASSERT_EQ(0,
+              streamer->add_impl(it->key(), it->data(), query_meta, context));
+  }
+
+  // Test linear search with exact match
+  size_t topk = 3;
+  IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+  NumericalVector<float> query_vec(dim);
+
+  for (size_t i = 0; i < doc_cnt; i += 100) {
+    for (size_t j = 0; j < dim; ++j) {
+      query_vec[j] = static_cast<float>(i);
+    }
+    context->set_topk(1U);
+    ASSERT_EQ(0,
+              streamer->search_bf_impl(query_vec.data(), query_meta, context));
+    auto &result1 = context->result();
+    ASSERT_EQ(1UL, result1.size());
+    ASSERT_EQ(i, result1[0].key());
+
+    // Test with slight offset
+    for (size_t j = 0; j < dim; ++j) {
+      query_vec[j] = static_cast<float>(i) + 0.1f;
+    }
+    context->set_topk(topk);
+    ASSERT_EQ(0,
+              streamer->search_bf_impl(query_vec.data(), query_meta, context));
+    auto &result2 = context->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+  }
+}
+
+TEST_F(HnswRabitqStreamerTest, TestKnnSearch) {
+  auto holder =
+      make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 2000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  RabitqConverter converter;
+  converter.init(*index_meta_ptr_, ailego::Params());
+  ASSERT_EQ(converter.train(holder), 0);
+  std::shared_ptr<IndexReformer> index_reformer;
+  ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
+  auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
+  IndexStreamer::Pointer streamer =
+      std::make_shared<HnswRabitqStreamer>(holder, reformer);
+
+  ailego::Params params;
+  params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
+  params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
+  params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 10U);
+  params.set("proxima.hnsw_rabitq.streamer.efconstruction", 100U);
+  params.set("proxima.hnsw_rabitq.streamer.ef", 50U);
+  params.set("proxima.hnsw_rabitq.general.dimension", dim);
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearch", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto context = streamer->create_context();
+  for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
+    IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+    ASSERT_EQ(0,
+              streamer->add_impl(it->key(), it->data(), query_meta, context));
+  }
+
+  // Compare KNN search with brute force search
+  auto linear_ctx = streamer->create_context();
+  auto knn_ctx = streamer->create_context();
+  size_t topk = 50;
+  linear_ctx->set_topk(topk);
+  knn_ctx->set_topk(topk);
+
+  int total_hits = 0;
+  int total_cnts = 0;
+  int topk1_hits = 0;
+  IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+  NumericalVector<float> query_vec(dim);
+
+  for (size_t i = 0; i < doc_cnt; i += 100) {
+    for (size_t j = 0; j < dim; ++j) {
+      query_vec[j] = static_cast<float>(i) + 0.1f;
+    }
+
+    ASSERT_EQ(0,
+              streamer->search_impl(query_vec.data(), query_meta, 1, knn_ctx));
+    ASSERT_EQ(
+        0, streamer->search_bf_impl(query_vec.data(), query_meta, linear_ctx));
+
+    auto &knn_result = knn_ctx->result(0);
+    ASSERT_EQ(topk, knn_result.size());
+    topk1_hits += (i == knn_result[0].key());
+
+    auto &linear_result = linear_ctx->result();
+    ASSERT_EQ(topk, linear_result.size());
+    ASSERT_EQ(i, linear_result[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      total_cnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linear_result[j].key() == knn_result[k].key()) {
+          total_hits++;
+          break;
+        }
+      }
+    }
+  }
+
+  float recall = total_hits * 1.0f / total_cnts;
+  float topk1_recall = topk1_hits * 100.0f / static_cast<float>(doc_cnt);
+  EXPECT_GT(recall, 0.60f);
+  // actual: no guarantee
+  // TODO(jiliang.ljl): check if ok?
+  EXPECT_GT(topk1_recall, 0.00f);
+}
+
+TEST_F(HnswRabitqStreamerTest, TestRandomData) {
+  auto holder =
+      make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 1500UL;
+
+  // Add random vectors
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(rand()) / static_cast<float>(RAND_MAX);
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  RabitqConverter converter;
+  converter.init(*index_meta_ptr_, ailego::Params());
+  ASSERT_EQ(converter.train(holder), 0);
+  std::shared_ptr<IndexReformer> index_reformer;
+  ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
+  auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
+  IndexStreamer::Pointer streamer =
+      std::make_shared<HnswRabitqStreamer>(holder, reformer);
+
+  ailego::Params params;
+  params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 32U);
+  params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 16U);
+  params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 20U);
+  params.set("proxima.hnsw_rabitq.streamer.efconstruction", 200U);
+  params.set("proxima.hnsw_rabitq.streamer.ef", 100U);
+  params.set("proxima.hnsw_rabitq.general.dimension", dim);
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/TestRandomData", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto context = streamer->create_context();
+  for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
+    IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+    ASSERT_EQ(0,
+              streamer->add_impl(it->key(), it->data(), query_meta, context));
+  }
+
+  // Test with random queries
+  auto linear_ctx = streamer->create_context();
+  auto knn_ctx = streamer->create_context();
+  size_t topk = 50;
+  linear_ctx->set_topk(topk);
+  knn_ctx->set_topk(topk);
+
+  int total_hits = 0;
+  int total_cnts = 0;
+  int topk1_hits = 0;
+  IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+  NumericalVector<float> query_vec(dim);
+
+  size_t query_cnt = 200;
+  for (size_t i = 0; i < query_cnt; i++) {
+    for (size_t j = 0; j < dim; ++j) {
+      query_vec[j] = static_cast<float>(rand()) / static_cast<float>(RAND_MAX);
+    }
+
+    ASSERT_EQ(
+        0, streamer->search_bf_impl(query_vec.data(), query_meta, linear_ctx));
+    ASSERT_EQ(0,
+              streamer->search_impl(query_vec.data(), query_meta, 1, knn_ctx));
+
+    auto &knn_result = knn_ctx->result(0);
+    ASSERT_EQ(topk, knn_result.size());
+
+    auto &linear_result = linear_ctx->result();
+    ASSERT_EQ(topk, linear_result.size());
+
+    topk1_hits += (linear_result[0].key() == knn_result[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      total_cnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linear_result[j].key() == knn_result[k].key()) {
+          total_hits++;
+          break;
+        }
+      }
+    }
+  }
+
+  float recall = total_hits * 1.0f / total_cnts;
+  float topk1_recall = topk1_hits * 1.0f / query_cnt;
+  EXPECT_GT(recall, 0.50f);
+  EXPECT_GT(topk1_recall, 0.70f);
+}
+
+TEST_F(HnswRabitqStreamerTest, TestOpenClose) {
+  auto holder =
+      make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 500UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  RabitqConverter converter;
+  converter.init(*index_meta_ptr_, ailego::Params());
+  ASSERT_EQ(converter.train(holder), 0);
+  std::shared_ptr<IndexReformer> index_reformer;
+  ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
+  auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
+
+  ailego::Params params;
+  params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
+  params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
+  params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
+  params.set("proxima.hnsw_rabitq.general.dimension", dim);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/TestOpenClose", true));
+
+  IndexStreamer::Pointer streamer =
+      std::make_shared<HnswRabitqStreamer>(holder, reformer);
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto context = streamer->create_context();
+  IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+
+  // Add first half of vectors
+  for (size_t i = 0; i < doc_cnt / 2; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), query_meta, context));
+  }
+
+  ASSERT_EQ(0, streamer->flush(0UL));
+  ASSERT_EQ(0, streamer->close());
+
+  // Reopen and add second half
+  IndexStreamer::Pointer streamer2 =
+      std::make_shared<HnswRabitqStreamer>(holder);
+  ASSERT_EQ(0, streamer2->init(*index_meta_ptr_, params));
+  ASSERT_EQ(0, streamer2->open(storage));
+
+  auto context2 = streamer2->create_context();
+  for (size_t i = doc_cnt / 2; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer2->add_impl(i, vec.data(), query_meta, context2));
+  }
+
+  ASSERT_EQ(0, streamer2->flush(0UL));
+
+  // Verify search works after reopen
+  NumericalVector<float> query_vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    query_vec[j] = 10.0f;
+  }
+
+  context2->set_topk(5);
+  ASSERT_EQ(0,
+            streamer2->search_impl(query_vec.data(), query_meta, 1, context2));
+  const auto &result = context2->result(0);
+  ASSERT_EQ(5UL, result.size());
+  ASSERT_EQ(10UL, result[0].key());
+}
+
+TEST_F(HnswRabitqStreamerTest, TestCreateIterator) {
+  auto holder =
+      make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 300UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  RabitqConverter converter;
+  converter.init(*index_meta_ptr_, ailego::Params());
+  ASSERT_EQ(converter.train(holder), 0);
+  std::shared_ptr<IndexReformer> index_reformer;
+  ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
+  auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
+  IndexStreamer::Pointer streamer =
+      std::make_shared<HnswRabitqStreamer>(holder, reformer);
+
+  ailego::Params params;
+  params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
+  params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
+  params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
+  params.set("proxima.hnsw_rabitq.general.dimension", dim);
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/TestCreateIterator", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto context = streamer->create_context();
+  IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, dim);
+
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), query_meta, context));
+  }
+
+  streamer->flush(0UL);
+
+  // Test iterator
+  auto provider = streamer->create_provider();
+  auto iter = provider->create_iterator();
+  ASSERT_TRUE(!!iter);
+
+  size_t count = 0;
+  while (iter->is_valid()) {
+    ASSERT_EQ(count, iter->key());
+    // const float *data = (const float *)iter->data();
+    // for (size_t j = 0; j < dim; ++j) {
+    //   ASSERT_EQ(static_cast<float>(count), data[j]);
+    // }
+    iter->next();
+    count++;
+  }
+  ASSERT_EQ(doc_cnt, count);
+
+  // Test get_vector
+  // for (size_t i = 0; i < doc_cnt; i++) {
+  //   const float *data = (const float *)provider->get_vector(i);
+  //   ASSERT_NE(data, nullptr);
+  //   for (size_t j = 0; j < dim; ++j) {
+  //     ASSERT_EQ(static_cast<float>(i), data[j]);
+  //   }
+  // }
+}
+
+TEST_F(HnswRabitqStreamerTest, TestDimensions) {
+  std::vector<size_t> dimensions = {1,    2,    4,    8,    16,   32,   33,
+                                    63,   64,   128,  256,  512,  1024, 2047,
+                                    2048, 2049, 4095, 4096, 4097, 8192, 16384};
+  size_t doc_cnt = 100;
+
+  for (size_t test_dim : dimensions) {
+    std::cout << "Testing dimension: " << test_dim << std::endl;
+
+    IndexMeta index_meta(IndexMeta::DataType::DT_FP32, test_dim);
+    index_meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+    auto holder =
+        make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(
+            test_dim);
+    IndexStreamer::Pointer streamer =
+        std::make_shared<HnswRabitqStreamer>(holder);
+
+    ailego::Params params;
+    params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
+    params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
+    params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
+    params.set("proxima.hnsw_rabitq.general.dimension", test_dim);
+
+    int ret = streamer->init(index_meta, params);
+
+    // dimension <= 63 or >= 4096: init() should return -31
+    if (test_dim <= 63 || test_dim >= 4096) {
+      ASSERT_EQ(-31, ret) << "expected init to fail with -31, dim=" << test_dim;
+      std::cout << "Dimension " << test_dim
+                << " correctly rejected with ret=" << ret << std::endl;
+      continue;
+    }
+
+    // Valid dimensions: verify full streaming build succeeds
+    ASSERT_EQ(0, ret) << "init failed, dim=" << test_dim;
+
+    for (size_t i = 0; i < doc_cnt; i++) {
+      NumericalVector<float> vec(test_dim);
+      for (size_t j = 0; j < test_dim; ++j) {
+        vec[j] = static_cast<float>(i * test_dim + j) / 1000.0f;
+      }
+      ASSERT_TRUE(holder->emplace(i, std::move(vec))) << "dim=" << test_dim;
+    }
+
+    RabitqConverter converter;
+    converter.init(index_meta, ailego::Params());
+    ASSERT_EQ(0, converter.train(holder))
+        << "converter train failed, dim=" << test_dim;
+    std::shared_ptr<IndexReformer> index_reformer;
+    ASSERT_EQ(0, converter.to_reformer(&index_reformer)) << "dim=" << test_dim;
+    auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
+
+    // Recreate streamer with reformer
+    streamer = std::make_shared<HnswRabitqStreamer>(holder, reformer);
+    ASSERT_EQ(0, streamer->init(index_meta, params))
+        << "init with reformer failed, dim=" << test_dim;
+
+    auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+    ASSERT_NE(nullptr, storage);
+    ailego::Params stg_params;
+    ASSERT_EQ(0, storage->init(stg_params));
+    std::string storage_path =
+        dir_ + "/TestDimensions_" + std::to_string(test_dim);
+    ASSERT_EQ(0, storage->open(storage_path, true))
+        << "storage open failed, dim=" << test_dim;
+    ASSERT_EQ(0, streamer->open(storage))
+        << "streamer open failed, dim=" << test_dim;
+
+    auto context = streamer->create_context();
+    IndexQueryMeta query_meta(IndexMeta::DataType::DT_FP32, test_dim);
+    for (auto it = holder->create_iterator(); it->is_valid(); it->next()) {
+      ASSERT_EQ(0,
+                streamer->add_impl(it->key(), it->data(), query_meta, context))
+          << "add failed, dim=" << test_dim << ", key=" << it->key();
+    }
+    ASSERT_EQ(0, streamer->flush(0UL)) << "flush failed, dim=" << test_dim;
+
+    std::cout << "Dimension " << test_dim << " passed" << std::endl;
+  }
+}
+
+TEST_F(HnswRabitqStreamerTest, TestExBitsMismatch) {
+  auto holder =
+      make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 500UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = static_cast<float>(i * dim + j) / 1000.0f;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  // Train converter with default total_bits (7, ex_bits=6)
+  RabitqConverter converter;
+  converter.init(*index_meta_ptr_, ailego::Params());
+  ASSERT_EQ(converter.train(holder), 0);
+  std::shared_ptr<IndexReformer> index_reformer;
+  ASSERT_EQ(converter.to_reformer(&index_reformer), 0);
+  auto reformer = std::dynamic_pointer_cast<RabitqReformer>(index_reformer);
+
+  // Create streamer with total_bits=2 (ex_bits=1), mismatching the reformer
+  IndexStreamer::Pointer streamer =
+      std::make_shared<HnswRabitqStreamer>(holder, reformer);
+
+  ailego::Params params;
+  params.set("proxima.hnsw_rabitq.streamer.max_neighbor_count", 16U);
+  params.set("proxima.hnsw_rabitq.streamer.upper_neighbor_count", 8U);
+  params.set("proxima.hnsw_rabitq.streamer.scaling_factor", 5U);
+  params.set("proxima.hnsw_rabitq.general.dimension", dim);
+  params.set(PARAM_RABITQ_TOTAL_BITS, 2U);
+  ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params));
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "/TestExBitsMismatch", true));
+
+  // open() should detect ex_bits mismatch and return IndexError_Mismatch
+  ASSERT_EQ(IndexError_Mismatch, streamer->open(storage));
+}
+
+}  // namespace core
+}  // namespace zvec

tests/core/algorithm/hnsw_sparse/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_hnsw_sparse
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm/hnsw_sparse
+    )
+endforeach()

tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc

@@ -0,0 +1,469 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "hnsw_sparse_builder.h"
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <future>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/vector.h>
+#include "tests/test_util.h"
+#include "zvec/core/framework/index_framework.h"
+#include "hnsw_sparse_params.h"
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+using namespace std;
+using namespace testing;
+using namespace zvec::ailego;
+
+namespace zvec {
+namespace core {
+
+class HnswSparseBuilderTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+
+  static std::string _dir;
+  static shared_ptr<IndexMeta> _index_meta_ptr;
+};
+
+std::string HnswSparseBuilderTest::_dir("HnswSparseBuilderTest/");
+shared_ptr<IndexMeta> HnswSparseBuilderTest::_index_meta_ptr;
+
+void HnswSparseBuilderTest::SetUp(void) {
+  _index_meta_ptr.reset(new (nothrow) IndexMeta(IndexMeta::MetaType::MT_SPARSE,
+                                                IndexMeta::DataType::DT_FP32));
+  _index_meta_ptr->set_metric("InnerProductSparse", 0, ailego::Params());
+}
+
+void HnswSparseBuilderTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(_dir);
+}
+
+TEST_F(HnswSparseBuilderTest, TestGeneral) {
+  IndexBuilder::Pointer builder =
+      IndexFactory::CreateBuilder("HnswSparseBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<OnePassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
+  uint32_t sparse_count = 4;
+  size_t doc_cnt = 1000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  ailego::Params params;
+  params.set(PARAM_HNSW_SPARSE_BUILDER_THREAD_COUNT, 1);
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->train(holder));
+
+  ASSERT_EQ(0, builder->build(holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "TestGeneral";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_GT(stats.built_costtime(), 0UL);
+  // ASSERT_GT(stats.dumped_costtime(), 0UL);
+
+  // cleanup and rebuild
+  ASSERT_EQ(0, builder->cleanup());
+
+  auto holder2 =
+      make_shared<MultiPassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
+  size_t doc_cnt2 = 2000UL;
+  for (size_t i = 0; i < doc_cnt2; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder2->emplace(i, vec));
+  }
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+  ASSERT_EQ(0, builder->train(holder2));
+  ASSERT_EQ(0, builder->build(holder2));
+  auto dumper2 = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper2, nullptr);
+  ASSERT_EQ(0, dumper2->create(path));
+  ASSERT_EQ(0, builder->dump(dumper2));
+  ASSERT_EQ(0, dumper2->close());
+
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt2, stats.built_count());
+  ASSERT_EQ(doc_cnt2, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_GT(stats.built_costtime(), 0UL);
+}
+
+TEST_F(HnswSparseBuilderTest, TestMemquota) {
+  IndexBuilder::Pointer builder =
+      IndexFactory::CreateBuilder("HnswSparseBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<OnePassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
+  size_t doc_cnt = 1000UL;
+  uint32_t sparse_count = 32;
+
+  for (size_t i = 0; i < doc_cnt; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  ailego::Params params;
+  params.set("proxima.hnsw.sparse_builder.memory_quota", 100000UL);
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+  ASSERT_EQ(0, builder->train(holder));
+  ASSERT_EQ(IndexError_NoMemory, builder->build(holder));
+}
+
+TEST_F(HnswSparseBuilderTest, TestIndexThreads) {
+  IndexBuilder::Pointer builder1 =
+      IndexFactory::CreateBuilder("HnswSparseBuilder");
+  ASSERT_NE(builder1, nullptr);
+  IndexBuilder::Pointer builder2 =
+      IndexFactory::CreateBuilder("HnswSparseBuilder");
+  ASSERT_NE(builder2, nullptr);
+
+  auto holder =
+      make_shared<MultiPassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
+
+  size_t doc_cnt = 1000UL;
+  uint32_t sparse_count = 32;
+
+  for (size_t i = 0; i < doc_cnt; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  ailego::Params params;
+  std::srand(ailego::Realtime::MilliSeconds());
+  auto threads =
+      std::make_shared<SingleQueueIndexThreads>(std::rand() % 4, false);
+  ASSERT_EQ(0, builder1->init(*_index_meta_ptr, params));
+  ASSERT_EQ(0, builder2->init(*_index_meta_ptr, params));
+
+  auto build_index1 = [&]() {
+    ASSERT_EQ(0, builder1->train(threads, holder));
+    ASSERT_EQ(0, builder1->build(threads, holder));
+  };
+  auto build_index2 = [&]() {
+    ASSERT_EQ(0, builder2->train(threads, holder));
+    ASSERT_EQ(0, builder2->build(threads, holder));
+  };
+
+  auto t1 = std::async(std::launch::async, build_index1);
+  auto t2 = std::async(std::launch::async, build_index2);
+  t1.wait();
+  t2.wait();
+
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "TestIndexThreads";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder1->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder2->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats1 = builder1->stats();
+  ASSERT_EQ(doc_cnt, stats1.built_count());
+  auto &stats2 = builder2->stats();
+  ASSERT_EQ(doc_cnt, stats2.built_count());
+}
+
+TEST_F(HnswSparseBuilderTest, TestHalfFloatConverter) {
+  IndexBuilder::Pointer builder =
+      IndexFactory::CreateBuilder("HnswSparseBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  auto holder =
+      make_shared<OnePassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
+  uint32_t sparse_count = 4;
+  size_t doc_cnt = 1000UL;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  ailego::Params converter_params;
+  auto converter = IndexFactory::CreateConverter("HalfFloatSparseConverter");
+  converter->init(*_index_meta_ptr, converter_params);
+
+  IndexMeta index_meta = converter->meta();
+
+  converter->transform(holder);
+
+  auto converted_holder = converter->sparse_result();
+
+  ailego::Params params;
+  params.set(PARAM_HNSW_SPARSE_BUILDER_THREAD_COUNT, 1);
+  ASSERT_EQ(0, builder->init(index_meta, converter_params));
+
+  ASSERT_EQ(0, builder->train(converted_holder));
+
+  ASSERT_EQ(0, builder->build(converted_holder));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "TestHalFloatConverter";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_GT(stats.built_costtime(), 0UL);
+  // ASSERT_GT(stats.dumped_costtime(), 0UL);
+
+  // cleanup and rebuild
+  ASSERT_EQ(0, builder->cleanup());
+
+  auto holder2 =
+      make_shared<MultiPassIndexSparseHolder<IndexMeta::DataType::DT_FP32>>();
+  size_t doc_cnt2 = 2000UL;
+  for (size_t i = 0; i < doc_cnt2; i++) {
+    SparseVector<float> vec;
+
+    NumericalVector<uint32_t> sparse_indices(sparse_count);
+    NumericalVector<float> sparse_values(sparse_count);
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices[j] = 20 * j;
+      sparse_values[j] = i;
+    }
+
+    vec.add_sparses(sparse_indices, sparse_values);
+
+    ASSERT_TRUE(holder2->emplace(i, vec));
+  }
+
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+  ASSERT_EQ(0, builder->train(holder2));
+  ASSERT_EQ(0, builder->build(holder2));
+  auto dumper2 = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper2, nullptr);
+  ASSERT_EQ(0, dumper2->create(path));
+  ASSERT_EQ(0, builder->dump(dumper2));
+  ASSERT_EQ(0, dumper2->close());
+
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt2, stats.built_count());
+  ASSERT_EQ(doc_cnt2, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_GT(stats.built_costtime(), 0UL);
+}
+
+TEST_F(HnswSparseBuilderTest, TestIndptr) {
+  IndexBuilder::Pointer builder =
+      IndexFactory::CreateBuilder("HnswSparseBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  uint32_t sparse_count = 4;
+  size_t doc_cnt = 1000UL;
+
+  std::vector<uint64_t> keys;
+  keys.reserve(doc_cnt);
+
+  std::vector<uint64_t> sparse_indptr;
+  sparse_indptr.reserve(doc_cnt + 1);
+
+  std::vector<uint32_t> sparse_indices;
+  sparse_indices.reserve(doc_cnt * sparse_count);
+
+  std::vector<float> sparse_values;
+  sparse_values.reserve(doc_cnt * sparse_count);
+
+  size_t sparse_count_total = 0;
+  sparse_indptr.push_back(0);
+  for (size_t i = 0; i < doc_cnt; i++) {
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices.push_back(20 * j);
+      sparse_values.push_back(i);
+    }
+
+    keys.push_back(i);
+
+    sparse_count_total += sparse_count;
+    sparse_indptr.push_back(sparse_count_total);
+  }
+
+  ailego::Params params;
+  params.set(PARAM_HNSW_SPARSE_BUILDER_THREAD_COUNT, 1);
+  ASSERT_EQ(0, builder->init(*_index_meta_ptr, params));
+
+  ASSERT_EQ(0, builder->build(doc_cnt, keys.data(), sparse_indptr.data(),
+                              sparse_indices.data(), sparse_values.data()));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "TestIndptr";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_GT(stats.built_costtime(), 0UL);
+  // ASSERT_GT(stats.dumped_costtime(), 0UL);
+}
+
+TEST_F(HnswSparseBuilderTest, TestIndptrFp16) {
+  IndexBuilder::Pointer builder =
+      IndexFactory::CreateBuilder("HnswSparseBuilder");
+  ASSERT_NE(builder, nullptr);
+
+  uint32_t sparse_count = 4;
+  size_t doc_cnt = 1000UL;
+
+  std::vector<uint64_t> keys;
+  keys.reserve(doc_cnt);
+
+  std::vector<uint64_t> sparse_indptr;
+  sparse_indptr.reserve(doc_cnt + 1);
+
+  std::vector<uint32_t> sparse_indices;
+  sparse_indices.reserve(doc_cnt * sparse_count);
+
+  std::vector<float> sparse_values;
+  sparse_values.reserve(doc_cnt * sparse_count);
+
+  size_t sparse_count_total = 0;
+  sparse_indptr.push_back(0);
+  for (size_t i = 0; i < doc_cnt; i++) {
+    for (size_t j = 0; j < sparse_count; ++j) {
+      sparse_indices.push_back(20 * j);
+      sparse_values.push_back(i);
+    }
+
+    keys.push_back(i);
+
+    sparse_count_total += sparse_count;
+    sparse_indptr.push_back(sparse_count_total);
+  }
+
+  IndexMeta meta(IndexMeta::MetaType::MT_SPARSE, IndexMeta::DataType::DT_FP16);
+  ailego::Params params;
+  params.set(PARAM_HNSW_SPARSE_BUILDER_THREAD_COUNT, 1);
+  ASSERT_EQ(0, builder->init(meta, params));
+
+  IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE,
+                       IndexMeta::DataType::DT_FP32);
+  ASSERT_EQ(0, builder->build(qmeta, doc_cnt, keys.data(), sparse_indptr.data(),
+                              sparse_indices.data(), sparse_values.data()));
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  string path = _dir + "TestIndptrFp16";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats = builder->stats();
+  ASSERT_EQ(0UL, stats.trained_count());
+  ASSERT_EQ(doc_cnt, stats.built_count());
+  ASSERT_EQ(doc_cnt, stats.dumped_count());
+  ASSERT_EQ(0UL, stats.discarded_count());
+  ASSERT_EQ(0UL, stats.trained_costtime());
+  ASSERT_GT(stats.built_costtime(), 0UL);
+  // ASSERT_GT(stats.dumped_costtime(), 0UL);
+}
+
+}  // namespace core
+}  // namespace zvec
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp

@@ -0,0 +1,360 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <future>
+#include <iostream>
+#include <memory>
+#include <ailego/math/norm_matrix.h>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/vector.h>
+#include "tests/test_util.h"
+#include "hnsw_sparse_streamer.h"
+
+using namespace std;
+using namespace testing;
+using namespace zvec::ailego;
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+namespace zvec {
+namespace core {
+
+class HnswSparseStreamerTest : public testing::Test {
+ protected:
+  void SetUp(void);
+  void TearDown(void);
+  void generate_sparse_data(
+      size_t cnt, uint32_t sparse_dim_count,
+      std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
+      std::vector<NumericalVector<float>> &sparse_vec_list, bool norm);
+
+  static std::string dir_;
+  static shared_ptr<IndexMeta> index_meta_ptr_;
+};
+
+std::string HnswSparseStreamerTest::dir_(
+    "hnsw_sparse_streamer_buffer_test_dir/");
+shared_ptr<IndexMeta> HnswSparseStreamerTest::index_meta_ptr_;
+
+void HnswSparseStreamerTest::generate_sparse_data(
+    size_t cnt, uint32_t sparse_dim_count,
+    std::vector<NumericalVector<uint32_t>> &sparse_indices_list,
+    std::vector<NumericalVector<float>> &sparse_vec_list, bool norm) {
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(-1.0, 1.0);
+
+  for (size_t i = 0; i < cnt; ++i) {
+    // prepare sparse
+    NumericalVector<uint32_t> sparse_indices(sparse_dim_count);
+    NumericalVector<float> sparse_vec(sparse_dim_count);
+
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_indices[j] = j * 20;
+      sparse_vec[j] = dist(gen);
+    }
+
+    float norm;
+    ailego::Norm2Matrix<float, 1>::Compute(sparse_vec.data(), sparse_dim_count,
+                                           &norm);
+    for (size_t j = 0; j < sparse_dim_count; ++j) {
+      sparse_vec[j] = sparse_vec[j] / norm;
+    }
+
+    sparse_indices_list.push_back(sparse_indices);
+    sparse_vec_list.push_back(sparse_vec);
+  }
+}
+
+void HnswSparseStreamerTest::SetUp(void) {
+  index_meta_ptr_.reset(new (nothrow) IndexMeta(IndexMeta::MetaType::MT_SPARSE,
+                                                IndexMeta::DataType::DT_FP32));
+  index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params());
+
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+void HnswSparseStreamerTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+TEST_F(HnswSparseStreamerTest, TestGeneral) {
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("HnswSparseStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  size_t sparse_dim_count = 32;
+
+  IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
+                       IndexMeta::DataType::DT_FP32);
+  index_meta.set_metric("InnerProductSparse", 0, ailego::Params());
+
+  ailego::Params params;
+  params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 20);
+  params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 16);
+  params.set(PARAM_HNSW_SPARSE_STREAMER_EFCONSTRUCTION, 10);
+  params.set(PARAM_HNSW_SPARSE_STREAMER_EF, 5);
+  params.set(PARAM_HNSW_SPARSE_STREAMER_BRUTE_FORCE_THRESHOLD, 1000U);
+
+  ailego::Params stg_params;
+  auto write_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_EQ(0, write_storage->init(stg_params));
+  ASSERT_EQ(0, write_storage->open(dir_ + "Test/HnswSparseSearch", true));
+  ASSERT_EQ(0, write_streamer->init(index_meta, params));
+  ASSERT_EQ(0, write_streamer->open(write_storage));
+
+  size_t cnt = 20000U;
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  std::vector<NumericalVector<uint32_t>> sparse_indices_list;
+  std::vector<NumericalVector<float>> sparse_vec_list;
+
+  generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
+                       sparse_vec_list, true);
+
+  IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE,
+                       IndexMeta::DataType::DT_FP32);
+  for (size_t i = 0; i < cnt; i++) {
+    ASSERT_EQ(0, write_streamer->add_impl(
+                     i, sparse_dim_count, sparse_indices_list[i].data(),
+                     sparse_vec_list[i].data(), qmeta, ctx));
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  write_storage->close();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("HnswSparseStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("BufferStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSparseSearch", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+
+  auto linearCtx = read_streamer->create_context();
+  ASSERT_TRUE(!!linearCtx);
+
+  auto knnCtx = read_streamer->create_context();
+  ASSERT_TRUE(!!knnCtx);
+
+  // streamer->print_debug_info();
+  size_t topk = 200;
+  linearCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+
+  uint64_t knnTotalTime = 0;
+  uint64_t linearTotalTime = 0;
+
+  int totalHits = 0;
+  int totalCnts = 0;
+  int topk1Hits = 0;
+
+  for (size_t i = 0; i < cnt; i += 100) {
+    const auto &sparse_indices = sparse_indices_list[i];
+    const auto &sparse_vec = sparse_vec_list[i];
+
+    auto t1 = ailego::Realtime::MicroSeconds();
+
+    ASSERT_EQ(
+        0, read_streamer->search_impl(sparse_dim_count, sparse_indices.data(),
+                                      sparse_vec.data(), qmeta, knnCtx));
+
+    auto t2 = ailego::Realtime::MicroSeconds();
+
+    ASSERT_EQ(0, read_streamer->search_bf_impl(
+                     sparse_dim_count, sparse_indices.data(), sparse_vec.data(),
+                     qmeta, linearCtx));
+
+    auto t3 = ailego::Realtime::MicroSeconds();
+
+    knnTotalTime += t2 - t1;
+    linearTotalTime += t3 - t2;
+
+    // std::cout << "i: " << i << std::endl;
+
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+    topk1Hits += i == knnResult[0].key();
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      totalCnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linearResult[j].key() == knnResult[k].key()) {
+          totalHits++;
+          break;
+        }
+      }
+    }
+  }
+  float recall = totalHits * 1.0f / totalCnts;
+  float topk1Recall = topk1Hits * 100.0f / cnt;
+  float cost = linearTotalTime * 1.0f / knnTotalTime;
+#if 0
+    printf("knnTotalTime=%zd linearTotalTime=%zd totalHits=%d totalCnts=%d "
+           "R@%zd=%f R@1=%f cost=%f\n",
+           knnTotalTime, linearTotalTime, totalHits, totalCnts, topk, recall,
+           topk1Recall, cost);
+#endif
+  EXPECT_GT(recall, 0.80f);
+  EXPECT_GT(topk1Recall, 0.80f);
+  // EXPECT_GT(cost, 2.0f);
+}
+
+TEST_F(HnswSparseStreamerTest, TestHnswSearchMMap) {
+  IndexStreamer::Pointer write_streamer =
+      IndexFactory::CreateStreamer("HnswSparseStreamer");
+  ASSERT_TRUE(write_streamer != nullptr);
+
+  size_t sparse_dim_count = 32;
+
+  IndexMeta index_meta(IndexMeta::MetaType::MT_SPARSE,
+                       IndexMeta::DataType::DT_FP32);
+  index_meta.set_metric("InnerProductSparse", 0, ailego::Params());
+
+  ailego::Params params;
+  params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 20);
+  params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 16);
+  params.set(PARAM_HNSW_SPARSE_STREAMER_EFCONSTRUCTION, 10);
+  params.set(PARAM_HNSW_SPARSE_STREAMER_EF, 5);
+  params.set(PARAM_HNSW_SPARSE_STREAMER_BRUTE_FORCE_THRESHOLD, 1000U);
+
+  ailego::Params stg_params;
+  auto write_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_EQ(0, write_storage->init(stg_params));
+  ASSERT_EQ(0, write_storage->open(dir_ + "Test/HnswSparseSearch", true));
+  ASSERT_EQ(0, write_streamer->init(index_meta, params));
+  ASSERT_EQ(0, write_streamer->open(write_storage));
+
+  size_t cnt = 20000U;
+  auto ctx = write_streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  std::vector<NumericalVector<uint32_t>> sparse_indices_list;
+  std::vector<NumericalVector<float>> sparse_vec_list;
+
+  generate_sparse_data(cnt, sparse_dim_count, sparse_indices_list,
+                       sparse_vec_list, true);
+
+  IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE,
+                       IndexMeta::DataType::DT_FP32);
+  for (size_t i = 0; i < cnt; i++) {
+    ASSERT_EQ(0, write_streamer->add_impl(
+                     i, sparse_dim_count, sparse_indices_list[i].data(),
+                     sparse_vec_list[i].data(), qmeta, ctx));
+  }
+  write_streamer->flush(0UL);
+  write_streamer->close();
+  write_streamer.reset();
+  write_storage->close();
+
+  IndexStreamer::Pointer read_streamer =
+      IndexFactory::CreateStreamer("HnswSparseStreamer");
+  ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, params));
+  auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, read_storage);
+  ASSERT_EQ(0, read_storage->init(stg_params));
+  ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSparseSearch", false));
+  ASSERT_EQ(0, read_streamer->open(read_storage));
+
+  auto linearCtx = read_streamer->create_context();
+  ASSERT_TRUE(!!linearCtx);
+
+  auto knnCtx = read_streamer->create_context();
+  ASSERT_TRUE(!!knnCtx);
+
+  // streamer->print_debug_info();
+  size_t topk = 200;
+  linearCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+
+  uint64_t knnTotalTime = 0;
+  uint64_t linearTotalTime = 0;
+
+  int totalHits = 0;
+  int totalCnts = 0;
+  int topk1Hits = 0;
+
+  for (size_t i = 0; i < cnt; i += 100) {
+    const auto &sparse_indices = sparse_indices_list[i];
+    const auto &sparse_vec = sparse_vec_list[i];
+
+    auto t1 = ailego::Realtime::MicroSeconds();
+
+    ASSERT_EQ(
+        0, read_streamer->search_impl(sparse_dim_count, sparse_indices.data(),
+                                      sparse_vec.data(), qmeta, knnCtx));
+
+    auto t2 = ailego::Realtime::MicroSeconds();
+
+    ASSERT_EQ(0, read_streamer->search_bf_impl(
+                     sparse_dim_count, sparse_indices.data(), sparse_vec.data(),
+                     qmeta, linearCtx));
+
+    auto t3 = ailego::Realtime::MicroSeconds();
+
+    knnTotalTime += t2 - t1;
+    linearTotalTime += t3 - t2;
+
+    // std::cout << "i: " << i << std::endl;
+
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+    topk1Hits += i == knnResult[0].key();
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      totalCnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linearResult[j].key() == knnResult[k].key()) {
+          totalHits++;
+          break;
+        }
+      }
+    }
+  }
+  float recall = totalHits * 1.0f / totalCnts;
+  float topk1Recall = topk1Hits * 100.0f / cnt;
+  float cost = linearTotalTime * 1.0f / knnTotalTime;
+#if 0
+    printf("knnTotalTime=%zd linearTotalTime=%zd totalHits=%d totalCnts=%d "
+           "R@%zd=%f R@1=%f cost=%f\n",
+           knnTotalTime, linearTotalTime, totalHits, totalCnts, topk, recall,
+           topk1Recall, cost);
+#endif
+  EXPECT_GT(recall, 0.80f);
+  EXPECT_GT(topk1Recall, 0.80f);
+  // EXPECT_GT(cost, 2.0f);
+}
+
+}  // namespace core
+}  // namespace zvec
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/algorithm/ivf/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_cluster core_knn_flat core_knn_ivf
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm/ivf
+    )
+endforeach()

tests/core/algorithm/ivf/ivf_builder_test.cc

@@ -0,0 +1,528 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "ivf_builder.h"
+#include <future>
+#include <iostream>
+#include <vector>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/vector.h>
+
+using namespace zvec::core;
+using namespace zvec::ailego;
+using namespace std;
+
+class IVFBuilderTest : public testing::Test {
+ protected:
+  void SetUp() override;
+  void TearDown() override;
+
+  void prepare_index_holder(uint32_t base_key, uint32_t num);
+
+  IndexMeta index_meta_;
+  Params params_;
+  uint32_t dimension_;
+  IndexHolder::Pointer holder_;
+  IndexThreads::Pointer threads_{};
+};
+
+void IVFBuilderTest::SetUp() {
+  dimension_ = 8U;
+
+  index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
+  index_meta_.set_metric("SquaredEuclidean", 0, Params());
+
+  params_.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "8");
+  params_.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster");
+  std::mt19937 gen((std::random_device())());
+  bool v = std::uniform_int_distribution<size_t>(0, 1)(gen);
+  if (v) {
+    threads_ = std::make_shared<SingleQueueIndexThreads>();
+  }
+}
+
+void IVFBuilderTest::TearDown() {}
+
+void IVFBuilderTest::prepare_index_holder(uint32_t base_key, uint32_t num) {
+  MultiPassIndexHolder<IndexMeta::DataType::DT_FP32> *holder =
+      new MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>(dimension_);
+  uint32_t key = base_key;
+  for (size_t i = 0; i < num; ++i) {
+    NumericalVector<float> vec(dimension_);
+    for (size_t j = 0; j < dimension_; ++j) {
+      vec[j] = 1.0f * i;
+    }
+    holder->emplace(key + i, vec);
+  }
+
+  holder_.reset(holder);
+}
+
+TEST_F(IVFBuilderTest, TestInitSuccess) {
+  IVFBuilder builder;
+  int ret = builder.init(index_meta_, params_);
+  EXPECT_EQ(0, ret);
+}
+
+TEST_F(IVFBuilderTest, TestInitFailedWithInvalidMetric) {
+  IVFBuilder builder;
+  index_meta_.set_metric("invalid", 0, Params());
+  int ret = builder.init(index_meta_, params_);
+  EXPECT_EQ(IndexError_NoExist, ret);
+}
+
+TEST_F(IVFBuilderTest, TestInitFailedWithInvalidCentroidsNum) {
+  IVFBuilder builder;
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "2");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
+
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(IndexError_InvalidArgument, ret);
+}
+
+TEST_F(IVFBuilderTest, TestTrainWithHolder1Level) {
+  IVFBuilder builder;
+  int ret = builder.init(index_meta_, params_);
+  EXPECT_EQ(0, ret);
+
+  prepare_index_holder(0, 1000);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  auto centroid_index = builder.centroid_index();
+  EXPECT_GT(centroid_index->centroids_count(), 0u);
+}
+
+TEST_F(IVFBuilderTest, TestTrainWithHolder2Level) {
+  IVFBuilder builder;
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+
+  prepare_index_holder(0, 1000);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  auto centroid_index = builder.centroid_index();
+  EXPECT_EQ(centroid_index->centroids_count(), 8);
+}
+
+TEST_F(IVFBuilderTest, TestTrainWithTrainer2Level) {
+  IndexTrainer::Pointer trainer =
+      IndexFactory::CreateTrainer("StratifiedClusterTrainer");
+  ASSERT_TRUE(!!trainer);
+
+  prepare_index_holder(0, 1000);
+
+  Params params;
+  params.set("zvec.stratified.trainer.cluster_count", "4*2");
+  ASSERT_EQ(0, trainer->init(index_meta_, params));
+  ASSERT_EQ(0, trainer->train(threads_, holder_));
+
+  IVFBuilder builder;
+  int ret = builder.init(index_meta_, params_);
+  EXPECT_EQ(0, ret);
+
+
+  ret = builder.train(trainer);
+  EXPECT_EQ(0, ret);
+
+  auto centroid_index = builder.centroid_index();
+  EXPECT_EQ(centroid_index->centroids_count(), 8);
+}
+
+TEST_F(IVFBuilderTest, TestTrainWithTrainer1Level) {
+  IVFBuilder builder;
+
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster");
+
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+
+  IndexTrainer::Pointer trainer =
+      IndexFactory::CreateTrainer("StratifiedClusterTrainer");
+  ASSERT_TRUE(!!trainer);
+
+  prepare_index_holder(0, 1000);
+
+  Params params1;
+  params1.set("zvec.stratified.trainer.cluster_count", "4");
+  ASSERT_EQ(0, trainer->init(index_meta_, params1));
+  ASSERT_EQ(0, trainer->train(threads_, holder_));
+
+  ret = builder.train(trainer);
+  EXPECT_EQ(0, ret);
+
+  auto centroid_index = builder.centroid_index();
+  EXPECT_EQ(centroid_index->centroids_count(), 4);
+}
+
+TEST_F(IVFBuilderTest, TestBuildWith2Level) {
+  IVFBuilder builder;
+
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+
+  prepare_index_holder(0, 1000);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  ret = builder.build(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  EXPECT_EQ((size_t)1000, builder.stats().built_count());
+}
+
+TEST_F(IVFBuilderTest, TestBuildWith1Level) {
+  IVFBuilder builder;
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster");
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+
+  prepare_index_holder(0, 1000);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  ret = builder.build(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  EXPECT_EQ((size_t)1000, builder.stats().built_count());
+}
+
+TEST_F(IVFBuilderTest, TestDump) {
+  IVFBuilder builder;
+  int ret = builder.init(index_meta_, params_);
+  EXPECT_EQ(0, ret);
+
+  prepare_index_holder(0, 1000);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  ret = builder.build(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
+  ret = dumper->create("path");
+  EXPECT_EQ(0, ret);
+
+  ret = builder.dump(dumper);
+  EXPECT_EQ((size_t)1000, builder.stats().built_count());
+  EXPECT_EQ((size_t)1000, builder.stats().dumped_count());
+  EXPECT_EQ((size_t)0, builder.stats().discarded_count());
+}
+
+#if 0
+TEST_F(IVFBuilderTest, TestBuildWithNoEnoughMemory)
+{
+    IVFBuilder builder;
+    Params params;
+    params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
+    params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
+
+    dimension_ = 256;
+    index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
+
+    int ret = builder.init(index_meta_, params);
+    EXPECT_EQ(0, ret);
+
+    prepare_index_holder(0, 1000);
+
+    ret = builder.train(threads_, holder_);
+    EXPECT_EQ(0, ret);
+
+    ret = builder.build(threads_, holder_);
+    EXPECT_EQ(IndexError_IndexFull, ret);
+}
+#endif
+
+TEST_F(IVFBuilderTest, TestBuildWithEnoughMemory) {
+  IVFBuilder builder;
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
+
+  dimension_ = 256;
+  index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
+
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+
+  prepare_index_holder(0, 1000);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  ret = builder.build(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
+  ret = dumper->create("path");
+  EXPECT_EQ(0, ret);
+
+  ret = builder.dump(dumper);
+  EXPECT_EQ((size_t)1000, builder.stats().built_count());
+  EXPECT_EQ((size_t)1000, builder.stats().dumped_count());
+  EXPECT_EQ((size_t)0, builder.stats().discarded_count());
+}
+
+#if 0
+TEST_F(IVFBuilderTest, TestBuildWithRowMajorAndNoEnoughMemory)
+{
+    IVFBuilder builder;
+    Params params;
+    params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
+    params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
+
+    dimension_ = 256;
+    index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
+    index_meta_.set_major_order(IndexMeta::MajorOrder::MO_ROW);
+
+    int ret = builder.init(index_meta_, params);
+    EXPECT_EQ(0, ret);
+
+    prepare_index_holder(0, 1000);
+
+    ret = builder.train(threads_, holder_);
+    EXPECT_EQ(0, ret);
+
+    ret = builder.build(threads_, holder_);
+    EXPECT_EQ(IndexError_IndexFull, ret);
+}
+#endif
+
+TEST_F(IVFBuilderTest, TestBuildWithRowMajorAndMemory) {
+  IVFBuilder builder;
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
+
+  dimension_ = 256;
+  index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
+  index_meta_.set_major_order(IndexMeta::MajorOrder::MO_ROW);
+
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+
+  prepare_index_holder(0, 1000);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  ret = builder.build(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
+  ret = dumper->create("path");
+  EXPECT_EQ(0, ret);
+
+  ret = builder.dump(dumper);
+  EXPECT_EQ((size_t)1000, builder.stats().built_count());
+  EXPECT_EQ((size_t)1000, builder.stats().dumped_count());
+  EXPECT_EQ((size_t)0, builder.stats().discarded_count());
+}
+
+TEST_F(IVFBuilderTest, TestBuildWithEmptyCentroid) {
+  IVFBuilder builder;
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "2*2");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
+
+  dimension_ = 256;
+  index_meta_.set_meta(IndexMeta::DataType::DT_FP32, dimension_);
+  index_meta_.set_major_order(IndexMeta::MajorOrder::MO_ROW);
+
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+  size_t doc_cnt = 10;
+
+  MultiPassIndexHolder<IndexMeta::DataType::DT_FP32> *holder =
+      new MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>(dimension_);
+  for (size_t i = 0; i < doc_cnt; ++i) {
+    NumericalVector<float> vec(dimension_);
+    for (size_t j = 0; j < dimension_; ++j) {
+      vec[j] = 1.0f;
+    }
+    holder->emplace(i, vec);
+  }
+  holder_.reset(holder);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  ret = builder.build(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
+  ret = dumper->create("path");
+  EXPECT_EQ(0, ret);
+
+  ret = builder.dump(dumper);
+  EXPECT_EQ((size_t)10, builder.stats().built_count());
+  EXPECT_EQ((size_t)10, builder.stats().dumped_count());
+  EXPECT_EQ((size_t)0, builder.stats().discarded_count());
+}
+
+TEST_F(IVFBuilderTest, TestTrainClusterParams) {
+  IVFBuilder builder;
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "2*2");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster");
+  prepare_index_holder(0, 1000);
+  EXPECT_EQ(0, builder.init(index_meta_, params));
+  EXPECT_EQ(0, builder.train(threads_, holder_));
+  EXPECT_EQ(0, builder.build(threads_, holder_));
+
+  IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
+  EXPECT_EQ(0, dumper->create("test.index"));
+  EXPECT_EQ(0, builder.dump(dumper));
+}
+
+TEST_F(IVFBuilderTest, TestBuildWithConverterClass) {
+  IVFBuilder builder;
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster");
+  params.set(PARAM_IVF_BUILDER_CONVERTER_CLASS, "HalfFloatConverter");
+
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+
+  prepare_index_holder(0, 1000);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  ret = builder.build(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  auto centroid_index = builder.centroid_index();
+  EXPECT_GT(centroid_index->centroids_count(), 0u);
+
+  IndexDumper::Pointer dumper = IndexFactory::CreateDumper("MemoryDumper");
+  ret = dumper->create("path");
+  EXPECT_EQ(0, ret);
+
+  ret = builder.dump(dumper);
+  EXPECT_EQ((size_t)1000, builder.stats().built_count());
+  EXPECT_EQ((size_t)1000, builder.stats().dumped_count());
+  EXPECT_EQ((size_t)0, builder.stats().discarded_count());
+}
+
+TEST_F(IVFBuilderTest, TestBuildWithConverterClassMultiLevel) {
+  IVFBuilder builder;
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "4*2");
+  params.set(PARAM_IVF_BUILDER_CLUSTER_CLASS, "KmeansCluster*KmeansCluster");
+  params.set(PARAM_IVF_BUILDER_CONVERTER_CLASS, "HalfFloatConverter");
+
+  int ret = builder.init(index_meta_, params);
+  EXPECT_EQ(0, ret);
+
+  prepare_index_holder(0, 1000);
+
+  ret = builder.train(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  ret = builder.build(threads_, holder_);
+  EXPECT_EQ(0, ret);
+
+  auto centroid_index = builder.centroid_index();
+  EXPECT_EQ(centroid_index->centroids_count(), 8);
+
+  IndexDumper::Pointer dumper = IndexFactory::CreateDumper("FileDumper");
+  ret = dumper->create("./ivf_converter_test.index");
+  EXPECT_EQ(0, ret);
+
+  ret = builder.dump(dumper);
+  EXPECT_EQ((size_t)1000, builder.stats().built_count());
+  EXPECT_EQ((size_t)1000, builder.stats().dumped_count());
+  EXPECT_EQ((size_t)0, builder.stats().discarded_count());
+  EXPECT_EQ(0, dumper->close());
+  File::RemovePath("./ivf_converter_test.index");
+}
+
+TEST_F(IVFBuilderTest, TestIndexThreads) {
+  IndexBuilder::Pointer builder1 = IndexFactory::CreateBuilder("IVFBuilder");
+  ASSERT_NE(builder1, nullptr);
+  IndexBuilder::Pointer builder2 = IndexFactory::CreateBuilder("IVFBuilder");
+  ASSERT_NE(builder2, nullptr);
+
+  size_t dim = 128UL;
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
+  std::srand(Realtime::MilliSeconds());
+  auto holder =
+      std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(dim);
+  size_t doc_cnt = 1000;
+  for (size_t i = 0; i < doc_cnt; i++) {
+    NumericalVector<float> vec(dim);
+    for (size_t j = 0; j < dim; ++j) {
+      vec[j] = i;
+    }
+    ASSERT_TRUE(holder->emplace(i, vec));
+  }
+
+  Params params;
+  params.set(PARAM_IVF_BUILDER_CENTROID_COUNT, "2*2");
+  ASSERT_EQ(0, builder1->init(meta, params));
+  ASSERT_EQ(0, builder2->init(meta, params));
+
+  auto threads =
+      std::make_shared<SingleQueueIndexThreads>(std::rand() % 4, false);
+  auto build_index1 = [&]() {
+    ASSERT_EQ(0, builder1->train(threads, holder));
+    ASSERT_EQ(0, builder1->build(threads, holder));
+  };
+  auto build_index2 = [&]() {
+    ASSERT_EQ(0, builder2->train(threads, holder));
+    ASSERT_EQ(0, builder2->build(threads, holder));
+  };
+
+  auto t1 = std::async(std::launch::async, build_index1);
+  auto t2 = std::async(std::launch::async, build_index2);
+  t1.wait();
+  t2.wait();
+
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_NE(dumper, nullptr);
+
+  std::string path = "./hc_index";
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder1->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+  ASSERT_EQ(0, dumper->create(path));
+  ASSERT_EQ(0, builder2->dump(dumper));
+  ASSERT_EQ(0, dumper->close());
+
+  auto &stats1 = builder1->stats();
+  ASSERT_EQ(doc_cnt, stats1.built_count());
+  auto &stats2 = builder2->stats();
+  ASSERT_EQ(doc_cnt, stats2.built_count());
+}

tests/core/algorithm/vamana/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_metric core_quantizer core_knn_vamana core_knn_hnsw core_knn_flat
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm/vamana
+    )
+endforeach()

tests/core/algorithm/vamana/vamana_streamer_test.cc

@@ -0,0 +1,892 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "vamana_streamer.h"
+#include <sys/stat.h>
+#include <sys/types.h>
+#ifndef _MSC_VER
+#include <fcntl.h>
+#include <unistd.h>
+#endif
+#include <future>
+#include <iostream>
+#include <memory>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/vector.h>
+#include "tests/test_util.h"
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+#endif
+
+using namespace std;
+using namespace testing;
+using namespace zvec::ailego;
+
+namespace zvec {
+namespace core {
+
+constexpr size_t kDim = 16;
+
+class VamanaStreamerTest : public testing::Test {
+ protected:
+  void SetUp(void) override;
+  void TearDown(void) override;
+
+  IndexStreamer::Pointer CreateVamanaStreamer(
+      const ailego::Params &extra_params = ailego::Params());
+
+  static std::string dir_;
+  static shared_ptr<IndexMeta> index_meta_ptr_;
+};
+
+std::string VamanaStreamerTest::dir_("vamana_streamer_test_dir/");
+shared_ptr<IndexMeta> VamanaStreamerTest::index_meta_ptr_;
+
+void VamanaStreamerTest::SetUp(void) {
+  index_meta_ptr_.reset(new (nothrow)
+                            IndexMeta(IndexMeta::DataType::DT_FP32, kDim));
+  index_meta_ptr_->set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+void VamanaStreamerTest::TearDown(void) {
+  zvec::test_util::RemoveTestPath(dir_);
+}
+
+IndexStreamer::Pointer VamanaStreamerTest::CreateVamanaStreamer(
+    const ailego::Params &extra_params) {
+  auto streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  if (!streamer) return nullptr;
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 32U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 100U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 500U);
+  params.merge(extra_params);
+
+  if (streamer->init(*index_meta_ptr_, params) != 0) {
+    return nullptr;
+  }
+  return streamer;
+}
+
+TEST_F(VamanaStreamerTest, TestAddVector) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestAddVector", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  for (size_t i = 0; i < 1000UL; i++) {
+    NumericalVector<float> vec(kDim);
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+
+  streamer->flush(0UL);
+  streamer.reset();
+}
+
+TEST_F(VamanaStreamerTest, TestLinearSearch) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t cnt = 5000UL;
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  NumericalVector<float> vec(kDim);
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+
+  size_t topk = 3;
+  for (size_t i = 0; i < cnt; i += 1) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ctx->set_topk(1U);
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, ctx));
+    auto &result1 = ctx->result();
+    ASSERT_EQ(1UL, result1.size());
+    ASSERT_EQ(i, result1[0].key());
+
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i) + 0.1f;
+    }
+    ctx->set_topk(topk);
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, ctx));
+    auto &result2 = ctx->result();
+    ASSERT_EQ(topk, result2.size());
+    ASSERT_EQ(i, result2[0].key());
+    ASSERT_EQ(i == cnt - 1 ? i - 1 : i + 1, result2[1].key());
+    ASSERT_EQ(i == 0 ? 2 : (i == cnt - 1 ? i - 2 : i - 1), result2[2].key());
+  }
+}
+
+TEST_F(VamanaStreamerTest, TestKnnSearch) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  ailego::Params stg_params;
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearch.index", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  NumericalVector<float> vec(kDim);
+  size_t cnt = 5000U;
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+
+  auto linearCtx = streamer->create_context();
+  auto knnCtx = streamer->create_context();
+  size_t topk = 100;
+  linearCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+  int totalHits = 0;
+  int totalCnts = 0;
+  int topk1Hits = 0;
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i) + 0.1f;
+    }
+    ASSERT_EQ(0, streamer->search_impl(vec.data(), qmeta, knnCtx));
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, linearCtx));
+
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+    topk1Hits += i == knnResult[0].key();
+
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+
+    for (size_t k = 0; k < topk; ++k) {
+      totalCnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linearResult[j].key() == knnResult[k].key()) {
+          totalHits++;
+          break;
+        }
+      }
+    }
+  }
+  float recall = totalHits * 1.0f / totalCnts;
+  float topk1Recall = topk1Hits * 1.0f / cnt;
+  EXPECT_GT(recall, 0.90f);
+  EXPECT_GT(topk1Recall, 0.95f);
+}
+
+TEST_F(VamanaStreamerTest, TestOpenClose) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  constexpr size_t dim_large = 128;
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim_large);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 32U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 100U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+
+  streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+  ASSERT_EQ(0, streamer->init(meta, params));
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestOpenClose.index", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t testCnt = 200;
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim_large);
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  for (size_t i = 0; i < testCnt; i++) {
+    std::vector<float> vec(dim_large);
+    for (size_t d = 0; d < dim_large; ++d) {
+      vec[d] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+
+  ASSERT_EQ(0, streamer->flush(0UL));
+  ASSERT_EQ(0, streamer->close());
+
+  // Re-open and verify data
+  ASSERT_EQ(0, streamer->open(storage));
+  auto provider = streamer->create_provider();
+  auto iter = provider->create_iterator();
+  ASSERT_TRUE(!!iter);
+  size_t total = 0;
+  while (iter->is_valid()) {
+    float *data = (float *)iter->data();
+    for (size_t d = 0; d < dim_large; ++d) {
+      ASSERT_FLOAT_EQ(static_cast<float>(iter->key()), data[d]);
+    }
+    total++;
+    iter->next();
+  }
+  ASSERT_EQ(testCnt, total);
+}
+
+TEST_F(VamanaStreamerTest, TestKnnMultiThread) {
+  constexpr size_t dim = 32;
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 64U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 500U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 200U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 1000U);
+  params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 30U * 1024U * 1024U);
+  params.set(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, true);
+
+  auto streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+  ASSERT_EQ(0, streamer->init(meta, params));
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestKnnMultiThread", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  auto addVector = [&streamer, dim](int baseKey, size_t addCnt) {
+    NumericalVector<float> vec(dim);
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    size_t succAdd = 0;
+    auto ctx = streamer->create_context();
+    for (size_t i = 0; i < addCnt; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i + baseKey);
+      }
+      succAdd += !streamer->add_impl(baseKey + i, vec.data(), qmeta, ctx);
+    }
+    streamer->flush(0UL);
+    return succAdd;
+  };
+  auto t1 = std::async(std::launch::async, addVector, 0, 1000);
+  auto t2 = std::async(std::launch::async, addVector, 1000, 1000);
+  auto t3 = std::async(std::launch::async, addVector, 2000, 1000);
+  ASSERT_EQ(1000U, t1.get());
+  ASSERT_EQ(1000U, t2.get());
+  ASSERT_EQ(1000U, t3.get());
+  streamer->close();
+
+  // Verify data
+  ASSERT_EQ(0, streamer->open(storage));
+  auto provider = streamer->create_provider();
+  auto iter = provider->create_iterator();
+  ASSERT_TRUE(!!iter);
+  size_t total = 0;
+  uint64_t minKey = 10000;
+  uint64_t maxKey = 0;
+  while (iter->is_valid()) {
+    float *data = (float *)iter->data();
+    for (size_t d = 0; d < dim; ++d) {
+      ASSERT_FLOAT_EQ(static_cast<float>(iter->key()), data[d]);
+    }
+    total++;
+    minKey = std::min(minKey, iter->key());
+    maxKey = std::max(maxKey, iter->key());
+    iter->next();
+  }
+  ASSERT_EQ(3000, total);
+  ASSERT_EQ(0, minKey);
+  ASSERT_EQ(2999, maxKey);
+
+  // Multi-thread search
+  size_t topk = 100;
+  size_t cnt = 3000;
+  auto knnSearch = [&]() {
+    NumericalVector<float> vec(dim);
+    auto linearCtx = streamer->create_context();
+    auto knnCtx = streamer->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    linearCtx->set_topk(topk);
+    knnCtx->set_topk(topk);
+    size_t totalCnts = 0;
+    size_t totalHits = 0;
+    for (size_t i = 0; i < cnt; i += 1) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i) + 0.1f;
+      }
+      ASSERT_EQ(0, streamer->search_impl(vec.data(), qmeta, knnCtx));
+      ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, linearCtx));
+      auto &knnResult = knnCtx->result();
+      ASSERT_EQ(topk, knnResult.size());
+      auto &linearResult = linearCtx->result();
+      ASSERT_EQ(topk, linearResult.size());
+      ASSERT_EQ(i, linearResult[0].key());
+      for (size_t k = 0; k < topk; ++k) {
+        totalCnts++;
+        for (size_t j = 0; j < topk; ++j) {
+          if (linearResult[j].key() == knnResult[k].key()) {
+            totalHits++;
+            break;
+          }
+        }
+      }
+    }
+    ASSERT_TRUE((totalHits * 1.0f / totalCnts) > 0.80f);
+  };
+  auto s1 = std::async(std::launch::async, knnSearch);
+  auto s2 = std::async(std::launch::async, knnSearch);
+  auto s3 = std::async(std::launch::async, knnSearch);
+  s1.wait();
+  s2.wait();
+  s3.wait();
+}
+
+TEST_F(VamanaStreamerTest, TestContiguousMemory) {
+  ailego::Params extra;
+  extra.set(PARAM_VAMANA_STREAMER_USE_CONTIGUOUS_MEMORY, true);
+  extra.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 2000U);
+  auto streamer = CreateVamanaStreamer(extra);
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestContiguous.index", true));
+
+  // First build with default mmap mode
+  {
+    auto builder_streamer = CreateVamanaStreamer();
+    ASSERT_NE(nullptr, builder_streamer);
+    ASSERT_EQ(0, builder_streamer->open(storage));
+    auto ctx = builder_streamer->create_context();
+    ASSERT_TRUE(!!ctx);
+
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+    NumericalVector<float> vec(kDim);
+    size_t cnt = 3000UL;
+    for (size_t i = 0; i < cnt; i++) {
+      for (size_t j = 0; j < kDim; ++j) {
+        vec[j] = static_cast<float>(i);
+      }
+      ASSERT_EQ(0, builder_streamer->add_impl(i, vec.data(), qmeta, ctx));
+    }
+    ASSERT_EQ(0, builder_streamer->flush(0UL));
+    ASSERT_EQ(0, builder_streamer->close());
+  }
+
+  // Re-open with contiguous memory mode for search
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t cnt = 3000UL;
+  size_t topk = 50;
+  NumericalVector<float> vec(kDim);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  auto linearCtx = streamer->create_context();
+  auto knnCtx = streamer->create_context();
+  linearCtx->set_topk(topk);
+  knnCtx->set_topk(topk);
+  int totalHits = 0;
+  int totalCnts = 0;
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i) + 0.1f;
+    }
+    ASSERT_EQ(0, streamer->search_impl(vec.data(), qmeta, knnCtx));
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, linearCtx));
+    auto &knnResult = knnCtx->result();
+    ASSERT_EQ(topk, knnResult.size());
+    auto &linearResult = linearCtx->result();
+    ASSERT_EQ(topk, linearResult.size());
+    ASSERT_EQ(i, linearResult[0].key());
+    for (size_t k = 0; k < topk; ++k) {
+      totalCnts++;
+      for (size_t j = 0; j < topk; ++j) {
+        if (linearResult[j].key() == knnResult[k].key()) {
+          totalHits++;
+          break;
+        }
+      }
+    }
+  }
+  float recall = totalHits * 1.0f / totalCnts;
+  EXPECT_GT(recall, 0.90f);
+}
+
+TEST_F(VamanaStreamerTest, TestContiguousMultiThreadSearch) {
+  constexpr size_t dim = 32;
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  // Build with mmap mode
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestContiguousMT", true));
+
+  {
+    ailego::Params build_params;
+    build_params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 64U);
+    build_params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 128U);
+    build_params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+    build_params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+    build_params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 30U * 1024U * 1024U);
+    build_params.set(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, true);
+
+    auto builder = IndexFactory::CreateStreamer("VamanaStreamer");
+    ASSERT_NE(nullptr, builder);
+    ASSERT_EQ(0, builder->init(meta, build_params));
+    ASSERT_EQ(0, builder->open(storage));
+
+    auto ctx = builder->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    NumericalVector<float> vec(dim);
+    for (size_t i = 0; i < 3000; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i);
+      }
+      ASSERT_EQ(0, builder->add_impl(i, vec.data(), qmeta, ctx));
+    }
+    ASSERT_EQ(0, builder->flush(0UL));
+    ASSERT_EQ(0, builder->close());
+  }
+
+  // Re-open with contiguous memory
+  ailego::Params search_params;
+  search_params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 64U);
+  search_params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 128U);
+  search_params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  search_params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  search_params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 30U * 1024U * 1024U);
+  search_params.set(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, true);
+  search_params.set(PARAM_VAMANA_STREAMER_USE_CONTIGUOUS_MEMORY, true);
+
+  auto searcher = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, searcher);
+  ASSERT_EQ(0, searcher->init(meta, search_params));
+  ASSERT_EQ(0, searcher->open(storage));
+
+  size_t topk = 50;
+  size_t cnt = 3000;
+  auto knnSearch = [&]() {
+    NumericalVector<float> vec(dim);
+    auto linearCtx = searcher->create_context();
+    auto knnCtx = searcher->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    linearCtx->set_topk(topk);
+    knnCtx->set_topk(topk);
+    size_t totalCnts = 0;
+    size_t totalHits = 0;
+    for (size_t i = 0; i < cnt; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i) + 0.1f;
+      }
+      ASSERT_EQ(0, searcher->search_impl(vec.data(), qmeta, knnCtx));
+      ASSERT_EQ(0, searcher->search_bf_impl(vec.data(), qmeta, linearCtx));
+      auto &knnResult = knnCtx->result();
+      ASSERT_EQ(topk, knnResult.size());
+      auto &linearResult = linearCtx->result();
+      ASSERT_EQ(topk, linearResult.size());
+      ASSERT_EQ(i, linearResult[0].key());
+      for (size_t k = 0; k < topk; ++k) {
+        totalCnts++;
+        for (size_t j = 0; j < topk; ++j) {
+          if (linearResult[j].key() == knnResult[k].key()) {
+            totalHits++;
+            break;
+          }
+        }
+      }
+    }
+    ASSERT_TRUE((totalHits * 1.0f / totalCnts) > 0.80f);
+  };
+  auto s1 = std::async(std::launch::async, knnSearch);
+  auto s2 = std::async(std::launch::async, knnSearch);
+  auto s3 = std::async(std::launch::async, knnSearch);
+  s1.wait();
+  s2.wait();
+  s3.wait();
+}
+
+TEST_F(VamanaStreamerTest, TestProvider) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestProvider", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  size_t cnt = 500;
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  NumericalVector<float> vec(kDim);
+  for (size_t i = 0; i < cnt; i++) {
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(i);
+    }
+    ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+  }
+  ASSERT_EQ(0, streamer->flush(0UL));
+
+  auto provider = streamer->create_provider();
+  ASSERT_NE(nullptr, provider);
+  auto iter = provider->create_iterator();
+  ASSERT_TRUE(!!iter);
+  size_t total = 0;
+  while (iter->is_valid()) {
+    ASSERT_NE(nullptr, iter->data());
+    float *data = (float *)iter->data();
+    for (size_t d = 0; d < kDim; ++d) {
+      ASSERT_FLOAT_EQ(static_cast<float>(iter->key()), data[d]);
+    }
+    total++;
+    iter->next();
+  }
+  ASSERT_EQ(cnt, total);
+}
+
+TEST_F(VamanaStreamerTest, TestAddAndSearch) {
+  auto streamer = CreateVamanaStreamer();
+  ASSERT_NE(nullptr, streamer);
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearch.index", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  NumericalVector<float> vec(kDim);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kDim);
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+
+  // Add and search interleaved
+  for (size_t batch = 0; batch < 5; batch++) {
+    size_t base = batch * 200;
+    for (size_t i = 0; i < 200; i++) {
+      for (size_t j = 0; j < kDim; ++j) {
+        vec[j] = static_cast<float>(base + i);
+      }
+      ASSERT_EQ(0, streamer->add_impl(base + i, vec.data(), qmeta, ctx));
+    }
+
+    // Search for recently added vectors
+    size_t current_cnt = (batch + 1) * 200;
+    size_t topk = std::min(current_cnt, (size_t)10);
+    auto searchCtx = streamer->create_context();
+    searchCtx->set_topk(topk);
+    for (size_t j = 0; j < kDim; ++j) {
+      vec[j] = static_cast<float>(base);
+    }
+    ASSERT_EQ(0, streamer->search_bf_impl(vec.data(), qmeta, searchCtx));
+    auto &result = searchCtx->result();
+    ASSERT_EQ(topk, result.size());
+    ASSERT_EQ(base, result[0].key());
+  }
+}
+
+TEST_F(VamanaStreamerTest, TestKnnConcurrentAddAndSearch) {
+  constexpr size_t dim = 32;
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 64U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 128U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 500U);
+  params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 30U * 1024U * 1024U);
+  params.set(PARAM_VAMANA_STREAMER_GET_VECTOR_ENABLE, true);
+
+  auto streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+  ASSERT_EQ(0, streamer->init(meta, params));
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestConcurrentAddSearch", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  // First add some base data
+  {
+    auto ctx = streamer->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    NumericalVector<float> vec(dim);
+    for (size_t i = 0; i < 2000; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i);
+      }
+      ASSERT_EQ(0, streamer->add_impl(i, vec.data(), qmeta, ctx));
+    }
+  }
+
+  std::atomic<bool> stop_search{false};
+
+  // Concurrent add
+  auto addFuture = std::async(std::launch::async, [&]() {
+    auto ctx = streamer->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    NumericalVector<float> vec(dim);
+    for (size_t i = 2000; i < 3000; i++) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = static_cast<float>(i);
+      }
+      streamer->add_impl(i, vec.data(), qmeta, ctx);
+    }
+    stop_search.store(true);
+  });
+
+  // Concurrent search
+  auto searchFuture = std::async(std::launch::async, [&]() {
+    auto ctx = streamer->create_context();
+    IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+    NumericalVector<float> vec(dim);
+    ctx->set_topk(10);
+    while (!stop_search.load()) {
+      for (size_t j = 0; j < dim; ++j) {
+        vec[j] = 100.1f;
+      }
+      int ret = streamer->search_impl(vec.data(), qmeta, ctx);
+      ASSERT_EQ(0, ret);
+      auto &result = ctx->result();
+      ASSERT_GT(result.size(), 0UL);
+    }
+  });
+
+  addFuture.wait();
+  searchFuture.wait();
+}
+
+// Test concurrent build (parallel add_impl) which was crashing due to
+// unprotected node_chunks_ / node_chunk_bases_ access during chunk allocation.
+TEST_F(VamanaStreamerTest, TestConcurrentBuild) {
+  constexpr size_t dim = kDim;
+  constexpr size_t total_vectors = 5000;
+  constexpr size_t thread_count = 4;
+
+  ailego::Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 32U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 100U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 500U);
+  params.set(PARAM_VAMANA_STREAMER_MAX_INDEX_SIZE, 50U * 1024U * 1024U);
+
+  IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
+  meta.set_metric("SquaredEuclidean", 0, ailego::Params());
+
+  auto streamer = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+  ASSERT_EQ(0, streamer->init(meta, params));
+
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ailego::Params stg_params;
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestConcurrentBuild", true));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  // Parallel insertion from multiple threads (mimics local_builder behavior)
+  std::atomic<int> error_count{0};
+  std::vector<std::future<void>> futures;
+
+  for (size_t t = 0; t < thread_count; ++t) {
+    futures.push_back(std::async(std::launch::async, [&, t]() {
+      auto ctx = streamer->create_context();
+      ASSERT_TRUE(!!ctx);
+      IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+      NumericalVector<float> vec(dim);
+
+      for (size_t i = t; i < total_vectors; i += thread_count) {
+        for (size_t j = 0; j < dim; ++j) {
+          vec[j] = static_cast<float>(i) + static_cast<float>(j) * 0.01f;
+        }
+        int ret = streamer->add_impl(i, vec.data(), qmeta, ctx);
+        if (ret != 0) {
+          error_count.fetch_add(1);
+          return;
+        }
+      }
+    }));
+  }
+
+  for (auto &f : futures) {
+    f.wait();
+  }
+  ASSERT_EQ(0, error_count.load());
+
+  // Verify search still works correctly after concurrent build
+  auto search_ctx = streamer->create_context();
+  ASSERT_TRUE(!!search_ctx);
+  search_ctx->set_topk(1);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim);
+  NumericalVector<float> vec(dim);
+  for (size_t j = 0; j < dim; ++j) {
+    vec[j] = 0.0f;
+  }
+  ASSERT_EQ(0, streamer->search_impl(vec.data(), qmeta, search_ctx));
+  auto &result = search_ctx->result();
+  ASSERT_GT(result.size(), 0UL);
+}
+
+// Test Vamana + INT8 quantization + rotation end-to-end
+TEST_F(VamanaStreamerTest, TestInt8WithRotate) {
+  constexpr size_t kTestDim = 128;
+  constexpr size_t kCnt = 2000U;
+  constexpr size_t kTopk = 10;
+
+  IndexStreamer::Pointer streamer =
+      IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer);
+
+  Params params;
+  params.set(PARAM_VAMANA_STREAMER_MAX_DEGREE, 32U);
+  params.set(PARAM_VAMANA_STREAMER_SEARCH_LIST_SIZE, 100U);
+  params.set(PARAM_VAMANA_STREAMER_ALPHA, 1.2f);
+  params.set(PARAM_VAMANA_STREAMER_EF, 64U);
+  params.set(PARAM_VAMANA_STREAMER_BRUTE_FORCE_THRESHOLD, 500U);
+
+  IndexMeta index_meta_raw(IndexMeta::DataType::DT_FP32, kTestDim);
+  index_meta_raw.set_metric("SquaredEuclidean", 0, Params());
+
+  // Create INT8 converter with rotation enabled
+  Params converter_params;
+  converter_params.set("integer_streaming.converter.enable_rotate", true);
+  auto converter = IndexFactory::CreateConverter("Int8StreamingConverter");
+  ASSERT_NE(nullptr, converter);
+  ASSERT_EQ(0, converter->init(index_meta_raw, converter_params));
+
+  IndexMeta index_meta = converter->meta();
+
+  auto reformer = IndexFactory::CreateReformer(index_meta.reformer_name());
+  ASSERT_NE(nullptr, reformer);
+  ASSERT_EQ(0, reformer->init(index_meta.reformer_params()));
+
+  Params stg_params;
+  auto storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage);
+  ASSERT_EQ(0, storage->init(stg_params));
+  ASSERT_EQ(0, storage->open(dir_ + "TestInt8WithRotate.index", true));
+  ASSERT_EQ(0, streamer->init(index_meta, params));
+  ASSERT_EQ(0, streamer->open(storage));
+
+  // Add 2000 vectors
+  auto ctx = streamer->create_context();
+  ASSERT_TRUE(!!ctx);
+  IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, kTestDim);
+
+  std::mt19937 gen(42);
+  std::uniform_real_distribution<float> dist(-1.0f, 1.0f);
+  for (size_t i = 0; i < kCnt; i++) {
+    NumericalVector<float> vec(kTestDim);
+    for (size_t j = 0; j < kTestDim; ++j) vec[j] = dist(gen);
+
+    std::string new_vec;
+    IndexQueryMeta new_meta;
+    ASSERT_EQ(0, reformer->convert(vec.data(), qmeta, &new_vec, &new_meta));
+    ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx));
+  }
+
+  streamer->flush(0UL);
+  streamer.reset();
+  storage.reset();
+
+  // Reopen: reformer should auto-detect rotator from storage
+  auto storage2 = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_NE(nullptr, storage2);
+  ASSERT_EQ(0, storage2->init(stg_params));
+  ASSERT_EQ(0, storage2->open(dir_ + "TestInt8WithRotate.index", false));
+
+  auto streamer2 = IndexFactory::CreateStreamer("VamanaStreamer");
+  ASSERT_NE(nullptr, streamer2);
+  ASSERT_EQ(0, streamer2->init(index_meta, params));
+  ASSERT_EQ(0, streamer2->open(storage2));
+
+  auto reformer2 = IndexFactory::CreateReformer(index_meta.reformer_name());
+  ASSERT_NE(nullptr, reformer2);
+  ASSERT_EQ(0, reformer2->init(index_meta.reformer_params()));
+  ASSERT_EQ(0, reformer2->load(storage2));
+
+  // Search: verify knn results are non-empty
+  auto knnCtx = streamer2->create_context();
+  knnCtx->set_topk(kTopk);
+  auto linearCtx = streamer2->create_context();
+  linearCtx->set_topk(kTopk);
+
+  NumericalVector<float> query(kTestDim);
+  for (size_t j = 0; j < kTestDim; ++j) query[j] = dist(gen);
+
+  std::string new_query;
+  IndexQueryMeta new_qmeta;
+  ASSERT_EQ(0,
+            reformer2->transform(query.data(), qmeta, &new_query, &new_qmeta));
+  ASSERT_EQ(0, streamer2->search_impl(new_query.data(), new_qmeta, knnCtx));
+  ASSERT_EQ(0,
+            streamer2->search_bf_impl(new_query.data(), new_qmeta, linearCtx));
+
+  EXPECT_EQ(kTopk, knnCtx->result().size());
+  EXPECT_EQ(kTopk, linearCtx->result().size());
+}
+
+}  // namespace core
+}  // namespace zvec
+
+#if defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic pop
+#endif

tests/core/framework/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework
+      SRCS ${CC_SRCS}
+      INCS ../../src
+    )
+endforeach()

tests/core/interface/CMakeLists.txt

@@ -0,0 +1,15 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_metric core_interface core_knn_flat core_utility core_quantizer sparsehash core_knn_hnsw core_mix_reducer
+          core_knn_flat_sparse core_knn_hnsw_sparse core_knn_ivf core_knn_hnsw_rabitq core_knn_vamana
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/algorithm
+    )
+endforeach()

tests/core/interface/index_group_by_test.cc

@@ -0,0 +1,539 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <numeric>
+#include <set>
+#include <string>
+#include <vector>
+#include <gtest/gtest.h>
+#include "tests/test_util.h"
+#if RABITQ_SUPPORTED
+#include "core/algorithm/hnsw_rabitq/rabitq_converter.h"
+#include "zvec/core/framework/index_provider.h"
+#endif
+#include "zvec/core/interface/index.h"
+#include "zvec/core/interface/index_factory.h"
+#include "zvec/core/interface/index_param.h"
+#include "zvec/core/interface/index_param_builders.h"
+
+using namespace zvec::core_interface;
+
+namespace {
+
+constexpr uint32_t kDimension = 4;
+constexpr uint32_t kNumDocs = 12;
+constexpr uint32_t kNumGroups = 3;
+constexpr uint32_t kGroupTopk = 2;
+constexpr uint32_t kSearchTopk = 100;
+
+struct GroupByCase {
+  std::string name;
+  BaseIndexParam::Pointer index_param;
+  BaseIndexQueryParam::Pointer query_param;
+  bool is_sparse = false;
+  uint32_t dimension = kDimension;
+  bool with_refiner = false;
+};
+
+std::shared_ptr<std::vector<uint64_t>> AllPks() {
+  auto pks = std::make_shared<std::vector<uint64_t>>();
+  pks->reserve(kNumDocs);
+  for (uint32_t i = 0; i < kNumDocs; ++i) {
+    pks->push_back(i);
+  }
+  return pks;
+}
+
+void AttachGroupBy(const BaseIndexQueryParam::Pointer &query_param) {
+  query_param->group_by_param = std::make_shared<GroupByParam>();
+  query_param->group_by_param->group_count = kNumGroups;
+  query_param->group_by_param->group_topk = kGroupTopk;
+  query_param->group_by_param->group_by = [](uint64_t key) {
+    return std::to_string(key % kNumGroups);
+  };
+}
+
+BaseIndexParam::Pointer DenseFlatParam(uint32_t dimension = kDimension) {
+  return FlatIndexParamBuilder()
+      .WithMetricType(MetricType::kInnerProduct)
+      .WithDataType(DataType::DT_FP32)
+      .WithDimension(dimension)
+      .WithIsSparse(false)
+      .Build();
+}
+
+BaseIndexParam::Pointer SparseFlatParam() {
+  return FlatIndexParamBuilder()
+      .WithMetricType(MetricType::kInnerProduct)
+      .WithDataType(DataType::DT_FP32)
+      .WithIsSparse(true)
+      .Build();
+}
+
+BaseIndexParam::Pointer DenseHnswParam(uint32_t dimension = kDimension) {
+  return HNSWIndexParamBuilder()
+      .WithMetricType(MetricType::kInnerProduct)
+      .WithDataType(DataType::DT_FP32)
+      .WithDimension(dimension)
+      .WithIsSparse(false)
+      .WithEFConstruction(100)
+      .Build();
+}
+
+BaseIndexParam::Pointer SparseHnswParam() {
+  return HNSWIndexParamBuilder()
+      .WithMetricType(MetricType::kInnerProduct)
+      .WithDataType(DataType::DT_FP32)
+      .WithIsSparse(true)
+      .WithEFConstruction(100)
+      .Build();
+}
+
+BaseIndexQueryParam::Pointer FlatQuery(bool fetch_vector = false) {
+  return FlatQueryParamBuilder()
+      .with_topk(kSearchTopk)
+      .with_fetch_vector(fetch_vector)
+      .build();
+}
+
+BaseIndexQueryParam::Pointer FlatQuery(bool fetch_vector, bool is_linear,
+                                       bool with_bf_pks) {
+  auto builder = FlatQueryParamBuilder()
+                     .with_topk(kSearchTopk)
+                     .with_fetch_vector(fetch_vector)
+                     .with_is_linear(is_linear);
+  if (with_bf_pks) {
+    builder.with_bf_pks(AllPks());
+  }
+  return builder.build();
+}
+
+BaseIndexQueryParam::Pointer HnswQuery(bool fetch_vector = false,
+                                       bool is_linear = false,
+                                       bool with_bf_pks = false) {
+  auto builder = HNSWQueryParamBuilder()
+                     .with_topk(kSearchTopk)
+                     .with_ef_search(kSearchTopk)
+                     .with_fetch_vector(fetch_vector)
+                     .with_is_linear(is_linear);
+  if (with_bf_pks) {
+    builder.with_bf_pks(AllPks());
+  }
+  return builder.build();
+}
+
+#if RABITQ_SUPPORTED
+BaseIndexParam::Pointer DenseHnswRabitqParam(uint32_t dimension) {
+  using namespace zvec::ailego;
+  using namespace zvec::core;
+
+  constexpr size_t kTrainCount = 500;
+  auto holder =
+      std::make_shared<MultiPassIndexProvider<IndexMeta::DataType::DT_FP32>>(
+          dimension);
+  for (size_t i = 0; i < kTrainCount; ++i) {
+    NumericalVector<float> vec(dimension, static_cast<float>(i));
+    EXPECT_TRUE(holder->emplace(i, vec));
+  }
+
+  auto index_meta =
+      std::make_shared<IndexMeta>(IndexMeta::DataType::DT_FP32, dimension);
+  index_meta->set_metric("InnerProduct", 0, Params());
+  RabitqConverter converter;
+  EXPECT_EQ(0, converter.init(*index_meta, Params()));
+  EXPECT_EQ(0, converter.train(holder));
+
+  std::shared_ptr<IndexReformer> reformer;
+  EXPECT_EQ(0, converter.to_reformer(&reformer));
+
+  return HNSWRabitqIndexParamBuilder()
+      .WithMetricType(MetricType::kInnerProduct)
+      .WithDataType(DataType::DT_FP32)
+      .WithDimension(dimension)
+      .WithIsSparse(false)
+      .WithEFConstruction(100)
+      .WithProvider(holder)
+      .WithReformer(reformer)
+      .Build();
+}
+
+BaseIndexQueryParam::Pointer HnswRabitqQuery(bool fetch_vector = false,
+                                             bool is_linear = false,
+                                             bool with_bf_pks = false) {
+  auto builder = HNSWRabitqQueryParamBuilder()
+                     .with_topk(kSearchTopk)
+                     .with_ef_search(kSearchTopk)
+                     .with_fetch_vector(fetch_vector)
+                     .with_is_linear(is_linear);
+  if (with_bf_pks) {
+    builder.with_bf_pks(AllPks());
+  }
+  return builder.build();
+}
+#endif
+
+#if DISKANN_SUPPORTED
+BaseIndexParam::Pointer DenseDiskAnnParam(uint32_t dimension = kDimension) {
+  return DiskAnnIndexParamBuilder()
+      .WithMetricType(MetricType::kInnerProduct)
+      .WithDataType(DataType::DT_FP32)
+      .WithDimension(dimension)
+      .WithIsSparse(false)
+      .WithMaxDegree(32)
+      .WithListSize(kSearchTopk)
+      .WithPqChunkNum(0)
+      .Build();
+}
+
+BaseIndexQueryParam::Pointer DiskAnnQuery(bool fetch_vector = false,
+                                          bool is_linear = false,
+                                          bool with_bf_pks = false) {
+  auto query = std::make_shared<DiskAnnQueryParam>();
+  query->topk = kSearchTopk;
+  query->list_size = kSearchTopk;
+  query->fetch_vector = fetch_vector;
+  query->is_linear = is_linear;
+  if (with_bf_pks) {
+    query->bf_pks = AllPks();
+  }
+  return query;
+}
+#endif
+
+class GroupByInterfaceTest : public ::testing::Test {
+ protected:
+  void RunOk(const GroupByCase &test_case) {
+    Run(test_case, /*expect_error=*/false);
+  }
+
+  void RunRejected(const GroupByCase &test_case) {
+    Run(test_case, /*expect_error=*/true);
+  }
+
+ private:
+  struct QueryHolder {
+    std::vector<float> values;
+    std::vector<uint32_t> indices;
+    VectorData data;
+  };
+
+  void Run(const GroupByCase &test_case, bool expect_error) {
+    const std::string index_name = "test_groupby_" + test_case.name;
+    const std::string source_index_name = index_name + "_source";
+    zvec::test_util::RemoveTestFiles(index_name + "*");
+    zvec::test_util::RemoveTestFiles(source_index_name + "*");
+
+    auto source = IndexFactory::CreateAndInitIndex(*FlatSourceParam(test_case));
+    ASSERT_NE(nullptr, source) << test_case.name;
+    ASSERT_EQ(0, source->Open(source_index_name,
+                              {StorageOptions::StorageType::kMMAP, true}))
+        << test_case.name;
+
+    for (uint32_t i = 0; i < kNumDocs; ++i) {
+      AddDoc(source, i, test_case);
+    }
+    ASSERT_EQ(0, source->Train()) << test_case.name;
+
+    auto index = IndexFactory::CreateAndInitIndex(*test_case.index_param);
+    ASSERT_NE(nullptr, index) << test_case.name;
+    ASSERT_EQ(
+        0, index->Open(index_name, {StorageOptions::StorageType::kMMAP, true}))
+        << test_case.name;
+    ASSERT_EQ(0, index->Merge({source}, IndexFilter())) << test_case.name;
+
+    auto query_param = test_case.query_param->Clone();
+    AttachGroupBy(query_param);
+    if (test_case.with_refiner) {
+      query_param->refiner_param = std::make_shared<RefinerParam>();
+      query_param->refiner_param->scale_factor_ = 1.0f;
+      query_param->refiner_param->reference_index = source;
+    }
+    auto query = MakeQuery(test_case);
+
+    SearchResult result;
+    const int ret = index->Search(query.data, query_param, &result);
+    if (expect_error) {
+      ASSERT_NE(0, ret) << test_case.name;
+    } else {
+      ASSERT_EQ(0, ret) << test_case.name;
+      AssertGroupedResult(result, query_param, test_case);
+    }
+
+    ASSERT_EQ(0, index->Close()) << test_case.name;
+    ASSERT_EQ(0, source->Close()) << test_case.name;
+    zvec::test_util::RemoveTestFiles(index_name + "*");
+    zvec::test_util::RemoveTestFiles(source_index_name + "*");
+  }
+
+  BaseIndexParam::Pointer FlatSourceParam(const GroupByCase &test_case) {
+    if (test_case.is_sparse) {
+      return SparseFlatParam();
+    }
+    return DenseFlatParam(test_case.dimension);
+  }
+
+  void AddDoc(const Index::Pointer &index, uint32_t key,
+              const GroupByCase &test_case) {
+    std::vector<float> values(test_case.dimension, static_cast<float>(key));
+    if (test_case.is_sparse) {
+      std::vector<uint32_t> indices(test_case.dimension);
+      std::iota(indices.begin(), indices.end(), 0u);
+      VectorData data{
+          SparseVector{test_case.dimension, indices.data(), values.data()}};
+      ASSERT_EQ(0, index->Add(data, key)) << key;
+      return;
+    }
+    VectorData data{DenseVector{values.data()}};
+    ASSERT_EQ(0, index->Add(data, key)) << key;
+  }
+
+  QueryHolder MakeQuery(const GroupByCase &test_case) {
+    QueryHolder holder;
+    holder.values.assign(test_case.dimension, 1.0f);
+    if (test_case.is_sparse) {
+      holder.indices.resize(test_case.dimension);
+      std::iota(holder.indices.begin(), holder.indices.end(), 0u);
+      holder.data = VectorData{SparseVector{
+          test_case.dimension, holder.indices.data(), holder.values.data()}};
+    } else {
+      holder.data = VectorData{DenseVector{holder.values.data()}};
+    }
+    return holder;
+  }
+
+  void AssertGroupedResult(const SearchResult &result,
+                           const BaseIndexQueryParam::Pointer &query_param,
+                           const GroupByCase &test_case) {
+    ASSERT_TRUE(result.doc_list_.empty());
+    ASSERT_EQ(kNumGroups, result.group_doc_list_.size());
+
+    std::set<std::string> group_ids;
+    for (const auto &group : result.group_doc_list_) {
+      group_ids.insert(group.group_id());
+      ASSERT_LE(group.docs().size(), kGroupTopk);
+      ASSERT_GE(group.docs().size(), 1u);
+
+      const uint32_t expected_mod = std::stoul(group.group_id());
+      for (const auto &doc : group.docs()) {
+        ASSERT_EQ(expected_mod, doc.key() % kNumGroups);
+      }
+      for (size_t i = 1; i < group.docs().size(); ++i) {
+        ASSERT_GE(group.docs()[i - 1].score(), group.docs()[i].score());
+      }
+    }
+    for (uint32_t group = 0; group < kNumGroups; ++group) {
+      ASSERT_TRUE(group_ids.count(std::to_string(group)) > 0);
+    }
+
+    if (!query_param->fetch_vector) {
+      return;
+    }
+    if (test_case.is_sparse) {
+      AssertSparseVectorsFetched(result, test_case.dimension);
+    } else {
+      AssertDenseVectorsFetched(result, test_case.dimension, test_case.name);
+    }
+  }
+
+  void AssertDenseVectorsFetched(const SearchResult &result, uint32_t dimension,
+                                 const std::string &case_name = "") {
+    const bool has_reverted = !result.group_reverted_vector_list_.empty();
+    if (has_reverted) {
+      ASSERT_EQ(result.group_doc_list_.size(),
+                result.group_reverted_vector_list_.size());
+    }
+    for (size_t group_idx = 0; group_idx < result.group_doc_list_.size();
+         ++group_idx) {
+      const auto &group = result.group_doc_list_[group_idx];
+      const std::vector<std::string> *group_vectors = nullptr;
+      if (has_reverted) {
+        group_vectors = &result.group_reverted_vector_list_[group_idx];
+        ASSERT_EQ(group.docs().size(), group_vectors->size());
+      }
+      for (size_t doc_idx = 0; doc_idx < group.docs().size(); ++doc_idx) {
+        const auto &doc = group.docs()[doc_idx];
+        const float expected = static_cast<float>(doc.key());
+        const float *vector = nullptr;
+        if (has_reverted) {
+          vector =
+              reinterpret_cast<const float *>((*group_vectors)[doc_idx].data());
+        } else if (doc.vector() != nullptr) {
+          vector = reinterpret_cast<const float *>(doc.vector());
+        } else {
+          // DiskAnn stores fetched vectors in vector_string_ rather than
+          // the raw pointer field.
+          ASSERT_FALSE(doc.vector_string().empty())
+              << case_name << " key=" << doc.key();
+          vector = reinterpret_cast<const float *>(doc.vector_string().data());
+        }
+        for (uint32_t i = 0; i < dimension; ++i) {
+          ASSERT_FLOAT_EQ(expected, vector[i])
+              << case_name << " key=" << doc.key() << " i=" << i;
+        }
+      }
+    }
+  }
+
+  void AssertSparseVectorsFetched(const SearchResult &result,
+                                  uint32_t dimension) {
+    const bool has_reverted =
+        !result.group_reverted_sparse_values_list_.empty();
+    if (has_reverted) {
+      ASSERT_EQ(result.group_doc_list_.size(),
+                result.group_reverted_sparse_values_list_.size());
+    }
+    for (size_t group_idx = 0; group_idx < result.group_doc_list_.size();
+         ++group_idx) {
+      const auto &group = result.group_doc_list_[group_idx];
+      const std::vector<std::string> *group_sparse_values = nullptr;
+      if (has_reverted) {
+        group_sparse_values =
+            &result.group_reverted_sparse_values_list_[group_idx];
+        ASSERT_EQ(group.docs().size(), group_sparse_values->size());
+      }
+      for (size_t doc_idx = 0; doc_idx < group.docs().size(); ++doc_idx) {
+        const auto &doc = group.docs()[doc_idx];
+        const auto &sparse = doc.sparse_doc();
+        ASSERT_EQ(dimension, sparse.sparse_count());
+        const auto *indices =
+            reinterpret_cast<const uint32_t *>(sparse.sparse_indices().data());
+        const float *values = nullptr;
+        if (has_reverted) {
+          values = reinterpret_cast<const float *>(
+              (*group_sparse_values)[doc_idx].data());
+        } else {
+          values =
+              reinterpret_cast<const float *>(sparse.sparse_values().data());
+        }
+        const float expected = static_cast<float>(doc.key());
+        for (uint32_t i = 0; i < dimension; ++i) {
+          ASSERT_EQ(i, indices[i]);
+          ASSERT_FLOAT_EQ(expected, values[i]);
+        }
+      }
+    }
+  }
+};
+
+}  // namespace
+
+TEST_F(GroupByInterfaceTest, Dense) {
+  std::vector<GroupByCase> cases{
+      {"dense_flat_graph", DenseFlatParam(), FlatQuery()},
+      {"dense_flat_linear", DenseFlatParam(),
+       FlatQuery(/*fetch_vector=*/false, /*is_linear=*/true,
+                 /*with_bf_pks=*/false)},
+      {"dense_flat_bf_pks", DenseFlatParam(),
+       FlatQuery(/*fetch_vector=*/false, /*is_linear=*/false,
+                 /*with_bf_pks=*/true)},
+      {"dense_flat_fetch_vector", DenseFlatParam(),
+       FlatQuery(/*fetch_vector=*/true, /*is_linear=*/false,
+                 /*with_bf_pks=*/false)},
+      {"dense_hnsw_graph", DenseHnswParam(), HnswQuery()},
+      {"dense_hnsw_linear", DenseHnswParam(),
+       HnswQuery(/*fetch_vector=*/false, /*is_linear=*/true)},
+      {"dense_hnsw_bf_pks", DenseHnswParam(),
+       HnswQuery(/*fetch_vector=*/false, /*is_linear=*/false,
+                 /*with_bf_pks=*/true)},
+      {"dense_hnsw_fetch_vector", DenseHnswParam(),
+       HnswQuery(/*fetch_vector=*/true)},
+#if RABITQ_SUPPORTED
+      {"dense_hnsw_rabitq_graph", DenseHnswRabitqParam(64), HnswRabitqQuery(),
+       /*is_sparse=*/false, /*dimension=*/64},
+      {"dense_hnsw_rabitq_linear", DenseHnswRabitqParam(64),
+       HnswRabitqQuery(/*fetch_vector=*/false, /*is_linear=*/true),
+       /*is_sparse=*/false, /*dimension=*/64},
+      {"dense_hnsw_rabitq_bf_pks", DenseHnswRabitqParam(64),
+       HnswRabitqQuery(/*fetch_vector=*/false, /*is_linear=*/false,
+                       /*with_bf_pks=*/true),
+       /*is_sparse=*/false, /*dimension=*/64},
+  // Note: fetch_vector is not supported for RabitQ because the entity
+  // stores quantized binary data (not original float vectors), and
+  // RabitqReformer does not implement revert().
+
+#endif
+  };
+
+  for (const auto &test_case : cases) {
+    RunOk(test_case);
+  }
+}
+
+TEST_F(GroupByInterfaceTest, Sparse) {
+  std::vector<GroupByCase> cases{
+      {"sparse_flat_graph", SparseFlatParam(), FlatQuery(),
+       /*is_sparse=*/true},
+      {"sparse_hnsw_graph", SparseHnswParam(), HnswQuery(),
+       /*is_sparse=*/true},
+      {"sparse_hnsw_linear", SparseHnswParam(),
+       HnswQuery(/*fetch_vector=*/false, /*is_linear=*/true),
+       /*is_sparse=*/true},
+      {"sparse_hnsw_bf_pks", SparseHnswParam(),
+       HnswQuery(/*fetch_vector=*/false, /*is_linear=*/false,
+                 /*with_bf_pks=*/true),
+       /*is_sparse=*/true},
+      {"sparse_hnsw_fetch_vector", SparseHnswParam(),
+       HnswQuery(/*fetch_vector=*/true), /*is_sparse=*/true},
+  };
+
+  for (const auto &test_case : cases) {
+    RunOk(test_case);
+  }
+}
+
+TEST_F(GroupByInterfaceTest, UnsupportedIndexTypes) {
+  std::vector<GroupByCase> cases{
+      {"unsupported_vamana",
+       VamanaIndexParamBuilder()
+           .WithMetricType(MetricType::kInnerProduct)
+           .WithDataType(DataType::DT_FP32)
+           .WithDimension(kDimension)
+           .WithIsSparse(false)
+           .WithMaxDegree(32)
+           .WithSearchListSize(100)
+           .WithAlpha(1.2f)
+           .Build(),
+       VamanaQueryParamBuilder()
+           .with_topk(kSearchTopk)
+           .with_ef_search(kSearchTopk)
+           .build()},
+      {"unsupported_ivf",
+       IVFIndexParamBuilder()
+           .WithMetricType(MetricType::kInnerProduct)
+           .WithDataType(DataType::DT_FP32)
+           .WithDimension(kDimension)
+           .WithIsSparse(false)
+           .WithNList(4)
+           .Build(),
+       IVFQueryParamBuilder().with_topk(kSearchTopk).build()},
+      {"unsupported_refiner", DenseHnswParam(), HnswQuery(),
+       /*is_sparse=*/false,
+       /*dimension=*/kDimension,
+       /*with_refiner=*/true},
+#if DISKANN_SUPPORTED
+      {"unsupported_diskann_graph", DenseDiskAnnParam(), DiskAnnQuery()},
+      {"unsupported_diskann_linear", DenseDiskAnnParam(),
+       DiskAnnQuery(/*fetch_vector=*/false, /*is_linear=*/true)},
+      {"unsupported_diskann_bf_pks", DenseDiskAnnParam(),
+       DiskAnnQuery(/*fetch_vector=*/false, /*is_linear=*/false,
+                    /*with_bf_pks=*/true)},
+      {"unsupported_diskann_fetch_vector", DenseDiskAnnParam(),
+       DiskAnnQuery(/*fetch_vector=*/true)},
+#endif
+  };
+
+  for (const auto &test_case : cases) {
+    RunRejected(test_case);
+  }
+}

tests/core/metric/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_metric core_quantizer
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core/
+    )
+endforeach()

tests/core/metric/cosine_metric_test.cc

@@ -0,0 +1,606 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <iostream>
+#include <ailego/math/norm_matrix.h>
+#include <gtest/gtest.h>
+#include <zvec/ailego/utility/float_helper.h>
+#include "zvec/core/framework/index_factory.h"
+
+
+using namespace zvec;
+using namespace zvec::core;
+using namespace zvec::ailego;
+
+static void Norm2(std::vector<Float16> &vec, std::string *out) {
+  float norm = 0.0f;
+
+  out->resize(vec.size() * sizeof(Float16) + sizeof(float));
+
+  Norm2Matrix<Float16, 1>::Compute(vec.data(), vec.size(), &norm);
+
+  Float16 *buf = reinterpret_cast<Float16 *>(&(*out)[0]);
+
+  for (uint32_t i = 0; i < vec.size(); ++i) {
+    buf[i] = vec[i] / norm;
+  }
+
+  float *norm_buf =
+      reinterpret_cast<float *>(&(*out)[vec.size() * sizeof(Float16)]);
+
+  memcpy(norm_buf, &norm, sizeof(float));
+}
+
+static void Norm2(std::vector<float> &vec, std::string *out) {
+  float norm = 0.0f;
+
+  out->resize((vec.size() + 1) * sizeof(float));
+
+  Norm2Matrix<float, 1>::Compute(vec.data(), vec.size(), &norm);
+
+  float *buf = reinterpret_cast<float *>(&(*out)[0]);
+  for (uint32_t i = 0; i < vec.size(); ++i) {
+    buf[i] = vec[i] / norm;
+  }
+
+  buf[vec.size()] = norm;
+}
+
+static size_t ExtraDimension(IndexMeta::DataType type) {
+  // The extra quantized params storage size to save for each vector
+  if (type == IndexMeta::DT_FP32) return 1;
+  if (type == IndexMeta::DT_FP16) return 2;
+
+  return 0;
+}
+
+TEST(CosineMeasure_General_Test, General) {
+  auto measure = IndexFactory::CreateMetric("Cosine");
+  EXPECT_TRUE(measure);
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DT_INT16, 64);
+  ASSERT_NE(0, measure->init(meta, Params()));
+  meta.set_meta(IndexMeta::DT_FP16, 64);
+  ASSERT_EQ(0, measure->init(meta, Params()));
+  meta.set_meta(IndexMeta::DT_FP32, 64);
+  ASSERT_EQ(0, measure->init(meta, Params()));
+  meta.set_meta(IndexMeta::DT_INT8, 64);
+  ASSERT_NE(0, measure->init(meta, Params()));
+
+  meta.set_meta(IndexMeta::DT_BINARY32, 64);
+  ASSERT_NE(0, measure->init(meta, Params()));
+  meta.set_meta(IndexMeta::DT_BINARY64, 64);
+  ASSERT_NE(0, measure->init(meta, Params()));
+  meta.set_meta(IndexMeta::DT_INT4, 64);
+  ASSERT_NE(0, measure->init(meta, Params()));
+
+  IndexMeta meta2;
+  meta2.set_meta(IndexMeta::DT_BINARY32, 64);
+  EXPECT_FALSE(measure->is_matched(meta2));
+  EXPECT_TRUE(
+      measure->is_matched(meta, IndexQueryMeta(IndexMeta::DT_FP32, 64)));
+  EXPECT_FALSE(
+      measure->is_matched(meta, IndexQueryMeta(IndexMeta::DT_FP32, 63)));
+
+  EXPECT_FALSE(measure->distance_matrix(0, 0));
+  EXPECT_FALSE(measure->distance_matrix(3, 5));
+  EXPECT_FALSE(measure->distance_matrix(31, 65));
+  EXPECT_TRUE(measure->distance_matrix(1, 1));
+  EXPECT_FALSE(measure->distance_matrix(2, 1));
+  EXPECT_FALSE(measure->distance_matrix(2, 2));
+  EXPECT_FALSE(measure->distance_matrix(4, 1));
+  EXPECT_FALSE(measure->distance_matrix(4, 2));
+  EXPECT_FALSE(measure->distance_matrix(4, 4));
+  EXPECT_FALSE(measure->distance_matrix(8, 1));
+  EXPECT_FALSE(measure->distance_matrix(8, 2));
+  EXPECT_FALSE(measure->distance_matrix(8, 4));
+  EXPECT_FALSE(measure->distance_matrix(8, 8));
+  EXPECT_FALSE(measure->distance_matrix(16, 1));
+  EXPECT_FALSE(measure->distance_matrix(16, 2));
+  EXPECT_FALSE(measure->distance_matrix(16, 4));
+  EXPECT_FALSE(measure->distance_matrix(16, 8));
+  EXPECT_FALSE(measure->distance_matrix(16, 16));
+  EXPECT_FALSE(measure->distance_matrix(32, 1));
+  EXPECT_FALSE(measure->distance_matrix(32, 2));
+  EXPECT_FALSE(measure->distance_matrix(32, 4));
+  EXPECT_FALSE(measure->distance_matrix(32, 8));
+  EXPECT_FALSE(measure->distance_matrix(32, 16));
+  EXPECT_FALSE(measure->distance_matrix(32, 32));
+
+  EXPECT_FALSE(measure->support_normalize());
+  float result = 1.0f;
+  measure->normalize(&result);
+  EXPECT_FLOAT_EQ(1.0f, result);
+}
+
+TEST(CosineMeasure_General_Test, TestDistanceFp32) {
+  {
+    constexpr uint32_t dimension = 2;
+    IndexMeta meta;
+    meta.set_meta(IndexMeta::DT_FP32, dimension);
+
+    auto measure = IndexFactory::CreateMetric("Cosine");
+    ASSERT_TRUE(measure);
+    Params params;
+    ASSERT_EQ(0, measure->init(meta, params));
+    ASSERT_EQ(false, measure->support_train());
+
+    auto distance = measure->distance();
+    ASSERT_NE(distance, nullptr);
+    auto dist_matrix = measure->distance_matrix(1, 1);
+    ASSERT_NE(dist_matrix, nullptr);
+
+    std::vector<float> a = {0.2f, 0.9f};
+    std::vector<float> b = {0.3f, 0.5f};
+
+    std::string a_out;
+    std::string b_out;
+
+    Norm2(a, &a_out);
+    Norm2(b, &b_out);
+
+    float result = 0.0f;
+    distance(a_out.data(), b_out.data(),
+             dimension + ExtraDimension(IndexMeta::DT_FP32), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.00001f, std::abs(result - 0.05131668f));
+
+    dist_matrix(a_out.data(), b_out.data(),
+                dimension + ExtraDimension(IndexMeta::DT_FP32), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.00001f, std::abs(result - 0.05131668f));
+  }
+
+  {
+    constexpr uint32_t dimension = 3;
+    IndexMeta meta;
+    meta.set_meta(IndexMeta::DT_FP32, dimension);
+
+    auto measure = IndexFactory::CreateMetric("Cosine");
+    ASSERT_TRUE(measure);
+    Params params;
+    ASSERT_EQ(0, measure->init(meta, params));
+    ASSERT_EQ(false, measure->support_train());
+
+    auto distance = measure->distance();
+    ASSERT_NE(distance, nullptr);
+    auto dist_matrix = measure->distance_matrix(1, 1);
+    ASSERT_NE(dist_matrix, nullptr);
+
+    std::vector<float> a = {0.2f, 0.9f, 0.6f};
+    std::vector<float> b = {0.3f, 0.5f, 0.7f};
+
+    std::string a_out;
+    std::string b_out;
+
+    Norm2(a, &a_out);
+    Norm2(b, &b_out);
+
+    float result = 0.0f;
+    distance(a_out.data(), b_out.data(),
+             dimension + ExtraDimension(IndexMeta::DT_FP32), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.00001f, std::abs(result - 0.07199293f));
+
+    dist_matrix(a_out.data(), b_out.data(),
+                dimension + ExtraDimension(IndexMeta::DT_FP32), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.00001f, std::abs(result - 0.07199293f));
+  }
+
+  {
+    constexpr uint32_t dimension = 11;
+    IndexMeta meta;
+    meta.set_meta(IndexMeta::DT_FP32, dimension);
+
+    auto measure = IndexFactory::CreateMetric("Cosine");
+    ASSERT_TRUE(measure);
+    Params params;
+    ASSERT_EQ(0, measure->init(meta, params));
+    ASSERT_EQ(false, measure->support_train());
+
+    auto distance = measure->distance();
+    ASSERT_NE(distance, nullptr);
+    auto dist_matrix = measure->distance_matrix(1, 1);
+    ASSERT_NE(dist_matrix, nullptr);
+
+    std::vector<float> a = {1.0f, 2.0f, 3.0f, 0.2f, 0.3f, 0.1f,
+                            5.2f, 2.1f, 7.1f, 6.8f, 1.2f};
+    std::vector<float> b = {2.0f, 4.0f, 6.0f, 0.6f, 0.7f, 0.9f,
+                            1.0f, 2.3f, 3.4f, 4.5f, 6.4f};
+
+
+    std::string a_out;
+    std::string b_out;
+
+    Norm2(a, &a_out);
+    Norm2(b, &b_out);
+
+    float result = 0.0f;
+    distance(a_out.data(), b_out.data(),
+             dimension + ExtraDimension(IndexMeta::DT_FP32), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.00001f, std::abs(result - 0.2803060f));
+
+    dist_matrix(a_out.data(), b_out.data(),
+                dimension + ExtraDimension(IndexMeta::DT_FP32), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.00001f, std::abs(result - 0.2803060f));
+  }
+}
+
+TEST(CosineMeasure_General_Test, TestDistanceFp16) {
+  {
+    constexpr uint32_t dimension = 2;
+    IndexMeta meta;
+    meta.set_meta(IndexMeta::DT_FP16, dimension);
+
+    auto measure = IndexFactory::CreateMetric("Cosine");
+    ASSERT_TRUE(measure);
+    Params params;
+    ASSERT_EQ(0, measure->init(meta, params));
+    ASSERT_EQ(false, measure->support_train());
+
+    auto distance = measure->distance();
+    ASSERT_NE(distance, nullptr);
+    auto dist_matrix = measure->distance_matrix(1, 1);
+    ASSERT_NE(dist_matrix, nullptr);
+
+    std::vector<Float16> a = {0.2f, 0.9f};
+    std::vector<Float16> b = {0.3f, 0.5f};
+
+    std::string a_out;
+    std::string b_out;
+
+    Norm2(a, &a_out);
+    Norm2(b, &b_out);
+
+    float result = 0.0f;
+    distance(a_out.data(), b_out.data(),
+             dimension + ExtraDimension(IndexMeta::DT_FP16), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.001f, std::abs(result - 0.05131668f));
+
+    dist_matrix(a_out.data(), b_out.data(),
+                dimension + ExtraDimension(IndexMeta::DT_FP16), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.001f, std::abs(result - 0.05131668f));
+  }
+
+  {
+    constexpr uint32_t dimension = 3;
+    IndexMeta meta;
+    meta.set_meta(IndexMeta::DT_FP16, dimension);
+
+    auto measure = IndexFactory::CreateMetric("Cosine");
+    ASSERT_TRUE(measure);
+    Params params;
+    ASSERT_EQ(0, measure->init(meta, params));
+    ASSERT_EQ(false, measure->support_train());
+
+    auto distance = measure->distance();
+    ASSERT_NE(distance, nullptr);
+    auto dist_matrix = measure->distance_matrix(1, 1);
+    ASSERT_NE(dist_matrix, nullptr);
+
+    std::vector<Float16> a = {0.2f, 0.9f, 0.6f};
+    std::vector<Float16> b = {0.3f, 0.5f, 0.7f};
+
+    std::string a_out;
+    std::string b_out;
+
+    Norm2(a, &a_out);
+    Norm2(b, &b_out);
+
+    float result = 0.0f;
+    distance(a_out.data(), b_out.data(),
+             dimension + ExtraDimension(IndexMeta::DT_FP16), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.001f, std::abs(result - 0.07199293f));
+
+    dist_matrix(a_out.data(), b_out.data(),
+                dimension + ExtraDimension(IndexMeta::DT_FP16), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.001f, std::abs(result - 0.07199293f));
+  }
+
+  {
+    constexpr uint32_t dimension = 11;
+    IndexMeta meta;
+    meta.set_meta(IndexMeta::DT_FP16, dimension);
+
+    auto measure = IndexFactory::CreateMetric("Cosine");
+    ASSERT_TRUE(measure);
+    Params params;
+    ASSERT_EQ(0, measure->init(meta, params));
+    ASSERT_EQ(false, measure->support_train());
+
+    auto distance = measure->distance();
+    ASSERT_NE(distance, nullptr);
+    auto dist_matrix = measure->distance_matrix(1, 1);
+    ASSERT_NE(dist_matrix, nullptr);
+
+    std::vector<Float16> a = {1.0f, 2.0f, 3.0f, 0.2f, 0.3f, 0.1f,
+                              5.2f, 2.1f, 7.1f, 6.8f, 1.2f};
+    std::vector<Float16> b = {2.0f, 4.0f, 6.0f, 0.6f, 0.7f, 0.9f,
+                              1.0f, 2.3f, 3.4f, 4.5f, 6.4f};
+
+    std::string a_out;
+    std::string b_out;
+
+    Norm2(a, &a_out);
+    Norm2(b, &b_out);
+
+    float result = 0.0f;
+    dist_matrix(a_out.data(), b_out.data(),
+                dimension + ExtraDimension(IndexMeta::DT_FP16), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.001f, std::abs(result - 0.2803060f));
+
+    dist_matrix(a_out.data(), b_out.data(),
+                dimension + ExtraDimension(IndexMeta::DT_FP16), &result);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&result);
+    }
+
+    EXPECT_GE(0.001f, std::abs(result - 0.2803060f));
+  }
+}
+
+TEST(CosineMeasure_General_Test, TestDistanceBatchFp16Simple) {
+  {
+    constexpr uint32_t dimension = 2;
+    IndexMeta meta;
+    meta.set_meta(IndexMeta::DT_FP16, dimension);
+
+    auto measure = IndexFactory::CreateMetric("Cosine");
+    ASSERT_TRUE(measure);
+    Params params;
+    ASSERT_EQ(0, measure->init(meta, params));
+    ASSERT_EQ(false, measure->support_train());
+
+    auto dist_batch = measure->batch_distance();
+    ASSERT_NE(dist_batch, nullptr);
+
+    std::vector<Float16> a = {0.2f, 0.9f};
+    std::vector<Float16> b = {0.3f, 0.5f};
+
+    std::string a_out;
+    std::string b_out;
+
+
+    Norm2(a, &a_out);
+    Norm2(b, &b_out);
+
+    float results[2] = {0.0f, 0.0f};
+
+    const void *vecs[2];
+    vecs[0] = a_out.data();
+    vecs[1] = b_out.data();
+    dist_batch(vecs, b_out.data(), 2,
+               dimension + ExtraDimension(IndexMeta::DT_FP16), results);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&results[0]);
+      measure->normalize(&results[1]);
+    }
+
+    EXPECT_GE(0.001f, std::abs(results[0] - 0.05131668f));
+    EXPECT_GE(0.001f, std::abs(results[1] - 0.0f));
+  }
+}
+
+TEST(CosineMeasure_General_Test, TestDistanceBatchFp32Simple) {
+  {
+    constexpr uint32_t dimension = 2;
+    IndexMeta meta;
+    meta.set_meta(IndexMeta::DT_FP32, dimension);
+
+    auto measure = IndexFactory::CreateMetric("Cosine");
+    ASSERT_TRUE(measure);
+    Params params;
+    ASSERT_EQ(0, measure->init(meta, params));
+    ASSERT_EQ(false, measure->support_train());
+
+    auto dist_batch = measure->batch_distance();
+    ASSERT_NE(dist_batch, nullptr);
+
+    std::vector<float> a = {0.2f, 0.9f};
+    std::vector<float> b = {0.3f, 0.5f};
+
+    std::string a_out;
+    std::string b_out;
+
+    Norm2(a, &a_out);
+    Norm2(b, &b_out);
+
+    float results[2] = {0.0f, 0.0f};
+
+    const void *vecs[2];
+    vecs[0] = a_out.data();
+    vecs[1] = b_out.data();
+    dist_batch(vecs, b_out.data(), 2,
+               dimension + ExtraDimension(IndexMeta::DT_FP32), results);
+
+    if (measure->support_normalize()) {
+      measure->normalize(&results[0]);
+      measure->normalize(&results[1]);
+    }
+
+    EXPECT_GE(0.00001f, std::abs(results[0] - 0.05131668f));
+    EXPECT_GE(0.00001f, std::abs(results[1] - 0.0f));
+  }
+}
+
+template <typename T>
+void calculate_distance(std::vector<T> &a, std::vector<T> &b, size_t dimension,
+                        IndexMeta::DataType data_type, size_t batch_size,
+                        float expected_distance, float epsilon = 0.00001f) {
+  IndexMeta meta;
+  meta.set_meta(data_type, dimension);
+
+  auto measure = IndexFactory::CreateMetric("Cosine");
+  ASSERT_TRUE(measure);
+  Params params;
+  ASSERT_EQ(0, measure->init(meta, params));
+  ASSERT_EQ(false, measure->support_train());
+
+  auto dist_batch = measure->batch_distance();
+  ASSERT_NE(dist_batch, nullptr);
+
+  std::string a_out;
+  std::string b_out;
+
+  Norm2(a, &a_out);
+  Norm2(b, &b_out);
+
+  float results[2] = {0.0f, 0.0f};
+
+  const void *vecs[2];
+  vecs[0] = a_out.data();
+  vecs[1] = b_out.data();
+  dist_batch(vecs, b_out.data(), batch_size,
+             dimension + ExtraDimension(data_type), results);
+
+  if (measure->support_normalize()) {
+    measure->normalize(&results[0]);
+    measure->normalize(&results[1]);
+  }
+
+  EXPECT_GE(epsilon, std::abs(results[0] - expected_distance));
+  EXPECT_GE(epsilon, std::abs(results[1] - 0.0f));
+}
+
+
+TEST(CosineMeasure_General_Test, TestDistanceBatch) {
+  {
+    constexpr uint32_t dimension = 2;
+
+    {
+      std::vector<float> a = {0.2f, 0.9f};
+      std::vector<float> b = {0.3f, 0.5f};
+
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP32, 1, 0.05131668f,
+                         0.00001f);
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP32, 2, 0.05131668f,
+                         0.00001f);
+    }
+    {
+      std::vector<Float16> a = {0.2f, 0.9f};
+      std::vector<Float16> b = {0.3f, 0.5f};
+
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP16, 1, 0.05131668f,
+                         0.001f);
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP16, 2, 0.05131668f,
+                         0.001f);
+    }
+  }
+
+  {
+    constexpr uint32_t dimension = 3;
+
+
+    {
+      std::vector<float> a = {0.2f, 0.9f, 0.6f};
+      std::vector<float> b = {0.3f, 0.5f, 0.7f};
+
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP32, 1, 0.07199293f,
+                         0.00001f);
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP32, 2, 0.07199293f,
+                         0.00001f);
+    }
+    {
+      std::vector<Float16> a = {0.2f, 0.9f, 0.6f};
+      std::vector<Float16> b = {0.3f, 0.5f, 0.7f};
+
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP16, 1, 0.07199293f,
+                         0.001f);
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP16, 2, 0.07199293f,
+                         0.001f);
+    }
+  }
+
+  {
+    constexpr uint32_t dimension = 11;
+
+    {
+      std::vector<float> a = {1.0f, 2.0f, 3.0f, 0.2f, 0.3f, 0.1f,
+                              5.2f, 2.1f, 7.1f, 6.8f, 1.2f};
+      std::vector<float> b = {2.0f, 4.0f, 6.0f, 0.6f, 0.7f, 0.9f,
+                              1.0f, 2.3f, 3.4f, 4.5f, 6.4f};
+
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP32, 1, 0.2803060f,
+                         0.00001f);
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP32, 2, 0.2803060f,
+                         0.00001f);
+    }
+
+    {
+      std::vector<Float16> a = {1.0f, 2.0f, 3.0f, 0.2f, 0.3f, 0.1f,
+                                5.2f, 2.1f, 7.1f, 6.8f, 1.2f};
+      std::vector<Float16> b = {2.0f, 4.0f, 6.0f, 0.6f, 0.7f, 0.9f,
+                                1.0f, 2.3f, 3.4f, 4.5f, 6.4f};
+
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP16, 1, 0.2803060f,
+                         0.001f);
+      calculate_distance(a, b, dimension, IndexMeta::DT_FP16, 2, 0.2803060f,
+                         0.001f);
+    }
+  }
+}

tests/core/metric/euclidean_metric_test.cc

@@ -0,0 +1,134 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <iostream>
+#include <gtest/gtest.h>
+#include "zvec/core/framework/index_factory.h"
+
+using namespace zvec;
+using namespace zvec::core;
+
+TEST(SquaredEuclideanMetric, General) {
+  auto metric = IndexFactory::CreateMetric("SquaredEuclidean");
+  EXPECT_TRUE(metric);
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_INT16, 64);
+  ASSERT_NE(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_FP16, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_FP32, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_INT4, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_INT8, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+
+  IndexMeta meta2;
+  meta2.set_meta(IndexMeta::DataType::DT_BINARY32, 64);
+  EXPECT_TRUE(metric->is_matched(meta));
+  EXPECT_FALSE(metric->is_matched(meta2));
+  EXPECT_TRUE(metric->is_matched(
+      meta, IndexQueryMeta(IndexMeta::DataType::DT_INT8, 64)));
+  EXPECT_FALSE(metric->is_matched(
+      meta, IndexQueryMeta(IndexMeta::DataType::DT_INT8, 63)));
+
+  EXPECT_FALSE(metric->distance_matrix(0, 0));
+  EXPECT_FALSE(metric->distance_matrix(3, 5));
+  EXPECT_FALSE(metric->distance_matrix(31, 65));
+  EXPECT_TRUE(metric->distance_matrix(1, 1));
+  EXPECT_TRUE(metric->distance_matrix(2, 1));
+  EXPECT_TRUE(metric->distance_matrix(2, 2));
+  EXPECT_TRUE(metric->distance_matrix(4, 1));
+  EXPECT_TRUE(metric->distance_matrix(4, 2));
+  EXPECT_TRUE(metric->distance_matrix(4, 4));
+  EXPECT_TRUE(metric->distance_matrix(8, 1));
+  EXPECT_TRUE(metric->distance_matrix(8, 2));
+  EXPECT_TRUE(metric->distance_matrix(8, 4));
+  EXPECT_TRUE(metric->distance_matrix(8, 8));
+  EXPECT_FALSE(metric->distance_matrix(8, 32));
+  EXPECT_FALSE(metric->distance_matrix(8, 9));
+  EXPECT_TRUE(metric->distance_matrix(16, 1));
+  EXPECT_TRUE(metric->distance_matrix(16, 2));
+  EXPECT_TRUE(metric->distance_matrix(16, 4));
+  EXPECT_TRUE(metric->distance_matrix(16, 8));
+  EXPECT_TRUE(metric->distance_matrix(16, 16));
+  EXPECT_FALSE(metric->distance_matrix(16, 17));
+  EXPECT_TRUE(metric->distance_matrix(32, 1));
+  EXPECT_TRUE(metric->distance_matrix(32, 2));
+  EXPECT_TRUE(metric->distance_matrix(32, 4));
+  EXPECT_TRUE(metric->distance_matrix(32, 8));
+  EXPECT_TRUE(metric->distance_matrix(32, 16));
+  EXPECT_TRUE(metric->distance_matrix(32, 32));
+
+  EXPECT_FALSE(metric->support_normalize());
+  float result = 1.0f;
+  metric->normalize(&result);
+  EXPECT_FLOAT_EQ(1.0f, result);
+}
+
+TEST(EuclideanMetric, General) {
+  auto metric = IndexFactory::CreateMetric("Euclidean");
+  EXPECT_TRUE(metric);
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_INT16, 64);
+  ASSERT_NE(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_FP16, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_FP32, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_INT4, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_INT8, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+
+  IndexMeta meta2;
+  meta2.set_meta(IndexMeta::DataType::DT_BINARY32, 64);
+  EXPECT_TRUE(metric->is_matched(meta));
+  EXPECT_FALSE(metric->is_matched(meta2));
+  EXPECT_TRUE(metric->is_matched(
+      meta, IndexQueryMeta(IndexMeta::DataType::DT_INT8, 64)));
+  EXPECT_FALSE(metric->is_matched(
+      meta, IndexQueryMeta(IndexMeta::DataType::DT_INT8, 63)));
+
+  EXPECT_FALSE(metric->distance_matrix(0, 0));
+  EXPECT_FALSE(metric->distance_matrix(3, 5));
+  EXPECT_FALSE(metric->distance_matrix(31, 65));
+  EXPECT_TRUE(metric->distance_matrix(1, 1));
+  EXPECT_TRUE(metric->distance_matrix(2, 1));
+  EXPECT_TRUE(metric->distance_matrix(2, 2));
+  EXPECT_TRUE(metric->distance_matrix(4, 1));
+  EXPECT_TRUE(metric->distance_matrix(4, 2));
+  EXPECT_TRUE(metric->distance_matrix(4, 4));
+  EXPECT_TRUE(metric->distance_matrix(8, 1));
+  EXPECT_TRUE(metric->distance_matrix(8, 2));
+  EXPECT_TRUE(metric->distance_matrix(8, 4));
+  EXPECT_TRUE(metric->distance_matrix(8, 8));
+  EXPECT_TRUE(metric->distance_matrix(16, 1));
+  EXPECT_TRUE(metric->distance_matrix(16, 2));
+  EXPECT_TRUE(metric->distance_matrix(16, 4));
+  EXPECT_TRUE(metric->distance_matrix(16, 8));
+  EXPECT_TRUE(metric->distance_matrix(16, 16));
+  EXPECT_TRUE(metric->distance_matrix(32, 1));
+  EXPECT_TRUE(metric->distance_matrix(32, 2));
+  EXPECT_TRUE(metric->distance_matrix(32, 4));
+  EXPECT_TRUE(metric->distance_matrix(32, 8));
+  EXPECT_TRUE(metric->distance_matrix(32, 16));
+  EXPECT_TRUE(metric->distance_matrix(32, 32));
+
+  EXPECT_FALSE(metric->support_normalize());
+  float result = 1.0f;
+  metric->normalize(&result);
+  EXPECT_FLOAT_EQ(1.0f, result);
+}

tests/core/metric/inner_product_metric_test.cc

@@ -0,0 +1,77 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <iostream>
+#include <gtest/gtest.h>
+#include "zvec/core/framework/index_factory.h"
+
+using namespace zvec;
+using namespace zvec::core;
+
+TEST(InnerProductMetric, General) {
+  auto metric = IndexFactory::CreateMetric("InnerProduct");
+  ASSERT_TRUE(metric);
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_BINARY32, 64);
+  ASSERT_NE(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_BINARY64, 64);
+  ASSERT_NE(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_FP16, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_FP32, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_INT4, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+  meta.set_meta(IndexMeta::DataType::DT_INT8, 64);
+  ASSERT_EQ(0, metric->init(meta, ailego::Params()));
+
+  IndexMeta meta2;
+  meta2.set_meta(IndexMeta::DataType::DT_BINARY32, 64);
+  EXPECT_TRUE(metric->is_matched(meta));
+  EXPECT_FALSE(metric->is_matched(meta2));
+  EXPECT_TRUE(metric->is_matched(
+      meta, IndexQueryMeta(IndexMeta::DataType::DT_INT8, 64)));
+  EXPECT_FALSE(metric->is_matched(
+      meta, IndexQueryMeta(IndexMeta::DataType::DT_INT8, 63)));
+
+  EXPECT_FALSE(metric->distance_matrix(0, 0));
+  EXPECT_FALSE(metric->distance_matrix(3, 5));
+  EXPECT_FALSE(metric->distance_matrix(31, 65));
+  EXPECT_TRUE(metric->distance_matrix(1, 1));
+  EXPECT_TRUE(metric->distance_matrix(2, 1));
+  EXPECT_TRUE(metric->distance_matrix(2, 2));
+  EXPECT_TRUE(metric->distance_matrix(4, 1));
+  EXPECT_TRUE(metric->distance_matrix(4, 2));
+  EXPECT_TRUE(metric->distance_matrix(4, 4));
+  EXPECT_TRUE(metric->distance_matrix(8, 1));
+  EXPECT_TRUE(metric->distance_matrix(8, 2));
+  EXPECT_TRUE(metric->distance_matrix(8, 4));
+  EXPECT_TRUE(metric->distance_matrix(8, 8));
+  EXPECT_TRUE(metric->distance_matrix(16, 1));
+  EXPECT_TRUE(metric->distance_matrix(16, 2));
+  EXPECT_TRUE(metric->distance_matrix(16, 4));
+  EXPECT_TRUE(metric->distance_matrix(16, 8));
+  EXPECT_TRUE(metric->distance_matrix(16, 16));
+  EXPECT_TRUE(metric->distance_matrix(32, 1));
+  EXPECT_TRUE(metric->distance_matrix(32, 2));
+  EXPECT_TRUE(metric->distance_matrix(32, 4));
+  EXPECT_TRUE(metric->distance_matrix(32, 8));
+  EXPECT_TRUE(metric->distance_matrix(32, 16));
+  EXPECT_TRUE(metric->distance_matrix(32, 32));
+
+  EXPECT_TRUE(metric->support_normalize());
+  float result = 1.0f;
+  metric->normalize(&result);
+  EXPECT_FLOAT_EQ(-1.0f, result);
+}

tests/core/quantizer/CMakeLists.txt

@@ -0,0 +1,14 @@
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+      NAME ${CC_TARGET}
+      STRICT
+      LIBS zvec_ailego core_framework core_utility core_quantizer
+      SRCS ${CC_SRCS}
+      INCS . ${PROJECT_ROOT_DIR}/src/core/
+    )
+endforeach()

tests/core/quantizer/half_float_reformer_test.cc

@@ -0,0 +1,119 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <random>
+
+// #include <zvec/ailego/container/vector.h>
+// #include <zvec/ailego/container/params.h>
+
+#include <gtest/gtest.h>
+#include "zvec/core/framework/index_factory.h"
+#include "zvec/core/framework/index_holder.h"
+
+using namespace zvec::core;
+
+TEST(HalfFloatReformer, General) {
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<float> dist(-1.0, 1.0);
+
+  const size_t COUNT = 1000;
+  const size_t DIMENSION = 128;
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+
+  auto converter = IndexFactory::CreateConverter("HalfFloatConverter");
+  ASSERT_TRUE(converter);
+  ASSERT_EQ(0u, converter->init(meta, zvec::ailego::Params()));
+
+  auto reformer = IndexFactory::CreateReformer("HalfFloatReformer");
+  ASSERT_TRUE(reformer);
+  ASSERT_EQ(0u, reformer->init(zvec::ailego::Params()));
+
+  auto holder =
+      std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+          DIMENSION);
+  for (size_t i = 0; i < COUNT; ++i) {
+    zvec::ailego::NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = dist(gen);
+    }
+    holder->emplace(i + 1, vec);
+  }
+  EXPECT_EQ(COUNT, holder->count());
+  EXPECT_EQ(IndexMeta::DataType::DT_FP32, holder->data_type());
+  ASSERT_EQ(0u, IndexConverter::TrainAndTransform(converter, holder));
+
+  auto holder2 = converter->result();
+  EXPECT_EQ(COUNT, holder2->count());
+  EXPECT_EQ(IndexMeta::DataType::DT_FP16, holder2->data_type());
+  EXPECT_EQ(holder->dimension(), holder2->dimension());
+  EXPECT_EQ(holder->element_size(), holder2->element_size() * 2);
+
+  auto iter = holder->create_iterator();
+  auto iter2 = holder2->create_iterator();
+  std::string buffer;
+
+  for (; iter->is_valid(); iter->next(), iter2->next()) {
+    EXPECT_TRUE(iter2->is_valid());
+    EXPECT_TRUE(iter->data());
+    EXPECT_TRUE(iter2->data());
+
+    const float *f32 = (const float *)iter->data();
+    const zvec::ailego::Float16 *f16 =
+        (const zvec::ailego::Float16 *)iter2->data();
+    printf("%f %f\n", f32[0], (float)f16[0]);
+
+    std::string buffer2(
+        std::string((const char *)iter2->data(), holder2->element_size()));
+
+    IndexQueryMeta qmeta;
+    EXPECT_EQ(0, reformer->transform(
+                     iter->data(),
+                     IndexQueryMeta(holder->data_type(), holder->dimension()),
+                     &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_FP16, qmeta.data_type());
+    EXPECT_EQ(holder->dimension(), qmeta.dimension());
+    EXPECT_EQ(buffer, buffer2);
+
+    EXPECT_EQ(0, reformer->transform(iter->data(),
+                                     IndexQueryMeta(holder->data_type(),
+                                                    holder->dimension() / 4),
+                                     4, &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_FP16, qmeta.data_type());
+    EXPECT_EQ(holder->dimension() / 4, qmeta.dimension());
+    EXPECT_EQ(buffer, buffer2);
+
+    // Test reformer convert
+    buffer.clear();
+    EXPECT_EQ(0, reformer->convert(
+                     iter->data(),
+                     IndexQueryMeta(holder->data_type(), holder->dimension()),
+                     &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_FP16, qmeta.data_type());
+    EXPECT_EQ(holder->dimension(), qmeta.dimension());
+    EXPECT_EQ(buffer, buffer2);
+
+    buffer.clear();
+    EXPECT_EQ(0, reformer->convert(iter->data(),
+                                   IndexQueryMeta(holder->data_type(),
+                                                  holder->dimension() / 4),
+                                   4, &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_FP16, qmeta.data_type());
+    EXPECT_EQ(holder->dimension() / 4, qmeta.dimension());
+    EXPECT_EQ(buffer, buffer2);
+  }
+}

tests/core/quantizer/uniform_int8_reformer_test.cc

@@ -0,0 +1,539 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <iostream>
+#include <limits>
+#include <random>
+#include <string>
+#include <vector>
+#include <gtest/gtest.h>
+#include <zvec/ailego/container/vector.h>
+#include "zvec/core/framework/index_factory.h"
+#include "zvec/core/framework/index_holder.h"
+
+using namespace zvec::core;
+
+// ---------------------------------------------------------------------------
+// UniformInt8 Converter + Reformer: General (MultiPassHolder, uniform dist)
+// ---------------------------------------------------------------------------
+TEST(UniformInt8Reformer, General) {
+  std::mt19937 gen(42);
+  std::uniform_real_distribution<float> dist(0.0f, 1.0f);
+
+  const size_t COUNT = 5000;
+  const size_t DIMENSION = 64;
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+
+  auto converter =
+      IndexFactory::CreateConverter("UniformInt8StreamingConverter");
+  ASSERT_TRUE(converter);
+  ASSERT_EQ(0u, converter->init(meta, zvec::ailego::Params()));
+
+  auto holder =
+      std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+          DIMENSION);
+  for (size_t i = 0; i < COUNT; ++i) {
+    zvec::ailego::NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = dist(gen);
+    }
+    holder->emplace(i + 1, vec);
+  }
+  EXPECT_EQ(COUNT, holder->count());
+  EXPECT_EQ(IndexMeta::DataType::DT_FP32, holder->data_type());
+
+  ASSERT_EQ(0u, IndexConverter::TrainAndTransform(converter, holder));
+
+  auto &stats = converter->stats();
+  EXPECT_EQ(COUNT, stats.trained_count());
+  EXPECT_EQ(COUNT, stats.transformed_count());
+
+  auto holder2 = converter->result();
+  ASSERT_TRUE(holder2);
+  EXPECT_EQ(COUNT, holder2->count());
+  EXPECT_EQ(IndexMeta::DataType::DT_INT8, holder2->data_type());
+  EXPECT_EQ(DIMENSION, holder2->dimension());
+  // INT8: 1 byte per dim; FP32: 4 bytes per dim
+  EXPECT_EQ(holder->element_size(), holder2->element_size() * 4);
+
+  // Verify quantized values are in [0, 127]
+  auto iter_check = holder2->create_iterator();
+  for (; iter_check->is_valid(); iter_check->next()) {
+    const int8_t *quantized =
+        reinterpret_cast<const int8_t *>(iter_check->data());
+    for (size_t d = 0; d < DIMENSION; ++d) {
+      EXPECT_GE(quantized[d], 0) << "dim=" << d;
+      EXPECT_LE(quantized[d], 127) << "dim=" << d;
+    }
+  }
+
+  // Create reformer from converter's trained params
+  auto reformer = IndexFactory::CreateReformer("UniformInt8StreamingReformer");
+  ASSERT_TRUE(reformer);
+  ASSERT_EQ(0u, reformer->init(converter->meta().reformer_params()));
+
+  // Verify transform() produces the same int8 as the converter
+  auto iter = holder->create_iterator();
+  auto iter2 = holder2->create_iterator();
+  std::string buffer;
+
+  for (; iter->is_valid(); iter->next(), iter2->next()) {
+    ASSERT_TRUE(iter2->is_valid());
+    ASSERT_TRUE(iter->data());
+    ASSERT_TRUE(iter2->data());
+
+    std::string expected(reinterpret_cast<const char *>(iter2->data()),
+                         holder2->element_size());
+
+    IndexQueryMeta qmeta;
+    EXPECT_EQ(0, reformer->transform(
+                     iter->data(),
+                     IndexQueryMeta(holder->data_type(), holder->dimension()),
+                     &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_INT8, qmeta.data_type());
+    EXPECT_EQ(DIMENSION, qmeta.dimension());
+    EXPECT_EQ(expected, buffer);
+
+    // Batch transform (count=4, dimension/4 per sub-vector)
+    EXPECT_EQ(0, reformer->transform(iter->data(),
+                                     IndexQueryMeta(holder->data_type(),
+                                                    holder->dimension() / 4),
+                                     4, &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_INT8, qmeta.data_type());
+    EXPECT_EQ(DIMENSION / 4, qmeta.dimension());
+    EXPECT_EQ(expected, buffer);
+
+    // convert() should produce the same result
+    buffer.clear();
+    EXPECT_EQ(0, reformer->convert(
+                     iter->data(),
+                     IndexQueryMeta(holder->data_type(), holder->dimension()),
+                     &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_INT8, qmeta.data_type());
+    EXPECT_EQ(DIMENSION, qmeta.dimension());
+    EXPECT_EQ(expected, buffer);
+
+    // Batch convert
+    buffer.clear();
+    EXPECT_EQ(0, reformer->convert(iter->data(),
+                                   IndexQueryMeta(holder->data_type(),
+                                                  holder->dimension() / 4),
+                                   4, &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_INT8, qmeta.data_type());
+    EXPECT_EQ(DIMENSION / 4, qmeta.dimension());
+    EXPECT_EQ(expected, buffer);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// OnePassHolder: verify converter works with single-pass holders
+// ---------------------------------------------------------------------------
+TEST(UniformInt8Reformer, OnePassHolder) {
+  std::mt19937 gen(123);
+  std::normal_distribution<float> dist(5.0f, 2.0f);
+
+  const size_t COUNT = 5000;
+  const size_t DIMENSION = 128;
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+
+  auto converter =
+      IndexFactory::CreateConverter("UniformInt8StreamingConverter");
+  ASSERT_TRUE(converter);
+  ASSERT_EQ(0u, converter->init(meta, zvec::ailego::Params()));
+
+  auto holder =
+      std::make_shared<OnePassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+          DIMENSION);
+  auto holder_mirror =
+      std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+          DIMENSION);
+  for (size_t i = 0; i < COUNT; ++i) {
+    zvec::ailego::NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = dist(gen);
+    }
+    holder->emplace(i + 1, vec);
+    holder_mirror->emplace(i + 1, vec);
+  }
+
+  ASSERT_EQ(0u, IndexConverter::TrainAndTransform(converter, holder));
+
+  auto holder2 = converter->result();
+  ASSERT_TRUE(holder2);
+  EXPECT_EQ(COUNT, holder2->count());
+  EXPECT_EQ(IndexMeta::DataType::DT_INT8, holder2->data_type());
+  EXPECT_EQ(DIMENSION, holder2->dimension());
+
+  auto reformer = IndexFactory::CreateReformer("UniformInt8StreamingReformer");
+  ASSERT_TRUE(reformer);
+  ASSERT_EQ(0u, reformer->init(converter->meta().reformer_params()));
+
+  auto iter = holder_mirror->create_iterator();
+  auto iter2 = holder2->create_iterator();
+  std::string buffer;
+
+  for (; iter->is_valid(); iter->next(), iter2->next()) {
+    ASSERT_TRUE(iter2->is_valid());
+    std::string expected(reinterpret_cast<const char *>(iter2->data()),
+                         holder2->element_size());
+
+    IndexQueryMeta qmeta;
+    EXPECT_EQ(0, reformer->transform(
+                     iter->data(),
+                     IndexQueryMeta(holder->data_type(), holder->dimension()),
+                     &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_INT8, qmeta.data_type());
+    EXPECT_EQ(expected, buffer);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// TrainedParams: verify scale/bias are persisted correctly after train
+// ---------------------------------------------------------------------------
+TEST(UniformInt8Reformer, TrainedParams) {
+  std::mt19937 gen(99);
+  std::uniform_real_distribution<float> dist(-3.0f, 7.0f);
+
+  const size_t COUNT = 5000;
+  const size_t DIMENSION = 32;
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+
+  auto converter =
+      IndexFactory::CreateConverter("UniformInt8StreamingConverter");
+  ASSERT_TRUE(converter);
+  ASSERT_EQ(0u, converter->init(meta, zvec::ailego::Params()));
+
+  auto holder =
+      std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+          DIMENSION);
+  for (size_t i = 0; i < COUNT; ++i) {
+    zvec::ailego::NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = dist(gen);
+    }
+    holder->emplace(i + 1, vec);
+  }
+
+  ASSERT_EQ(0u, IndexConverter::TrainAndTransform(converter, holder));
+  EXPECT_EQ(COUNT, converter->stats().trained_count());
+
+  // Verify reformer params contain scale and bias
+  auto reformer_params = converter->meta().reformer_params();
+  float scale = 0.0f, bias = 0.0f;
+  EXPECT_TRUE(reformer_params.get("uniform_int8.reformer.scale", &scale));
+  EXPECT_TRUE(reformer_params.get("uniform_int8.reformer.bias", &bias));
+  EXPECT_GT(scale, 0.0f);
+  EXPECT_TRUE(std::isfinite(scale));
+  EXPECT_TRUE(std::isfinite(bias));
+
+  // Verify converter params also contain scale/bias (for persistence)
+  auto conv_params = converter->meta().converter_params();
+  float conv_scale = 0.0f, conv_bias = 0.0f;
+  EXPECT_TRUE(conv_params.get("uniform_int8.reformer.scale", &conv_scale));
+  EXPECT_TRUE(conv_params.get("uniform_int8.reformer.bias", &conv_bias));
+  EXPECT_FLOAT_EQ(scale, conv_scale);
+  EXPECT_FLOAT_EQ(bias, conv_bias);
+
+  // Verify meta reflects the correct reformer and metric
+  EXPECT_EQ("UniformInt8StreamingReformer", converter->meta().reformer_name());
+  EXPECT_EQ("UniformInt8", converter->meta().metric_name());
+}
+
+// ---------------------------------------------------------------------------
+// Revert: verify int8 → float dequantization round-trip quality
+// ---------------------------------------------------------------------------
+TEST(UniformInt8Reformer, Revert) {
+  std::mt19937 gen(77);
+  std::uniform_real_distribution<float> dist(0.0f, 10.0f);
+
+  const size_t COUNT = 100;
+  const size_t DIMENSION = 16;
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+
+  auto converter =
+      IndexFactory::CreateConverter("UniformInt8StreamingConverter");
+  ASSERT_TRUE(converter);
+  ASSERT_EQ(0u, converter->init(meta, zvec::ailego::Params()));
+
+  auto holder =
+      std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+          DIMENSION);
+  for (size_t i = 0; i < COUNT; ++i) {
+    zvec::ailego::NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = dist(gen);
+    }
+    holder->emplace(i + 1, vec);
+  }
+
+  ASSERT_EQ(0u, IndexConverter::TrainAndTransform(converter, holder));
+
+  auto reformer = IndexFactory::CreateReformer("UniformInt8StreamingReformer");
+  ASSERT_TRUE(reformer);
+  ASSERT_EQ(0u, reformer->init(converter->meta().reformer_params()));
+
+  // Verify round-trip: float → int8 → float
+  auto iter = holder->create_iterator();
+  std::string quantized_buf, reverted_buf;
+
+  for (; iter->is_valid(); iter->next()) {
+    const float *original = reinterpret_cast<const float *>(iter->data());
+
+    IndexQueryMeta qmeta;
+    ASSERT_EQ(0, reformer->transform(
+                     iter->data(),
+                     IndexQueryMeta(holder->data_type(), holder->dimension()),
+                     &quantized_buf, &qmeta));
+
+    ASSERT_EQ(0, reformer->revert(quantized_buf.data(), qmeta, &reverted_buf));
+
+    const float *reverted =
+        reinterpret_cast<const float *>(reverted_buf.data());
+
+    // Quantization error should be bounded by step_size / 2
+    // step_size ≈ range / 127
+    float range = 10.0f;  // approximate
+    float max_error = range / 127.0f;
+    for (size_t d = 0; d < DIMENSION; ++d) {
+      EXPECT_NEAR(original[d], reverted[d], max_error * 1.5f)
+          << "dim=" << d << " original=" << original[d]
+          << " reverted=" << reverted[d];
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Normalize: verify score rescaling from int8 L2 to float L2
+// ---------------------------------------------------------------------------
+TEST(UniformInt8Reformer, Normalize) {
+  const size_t COUNT = 1000;
+  const size_t DIMENSION = 32;
+
+  std::mt19937 gen(55);
+  std::uniform_real_distribution<float> dist(0.0f, 5.0f);
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+
+  auto converter =
+      IndexFactory::CreateConverter("UniformInt8StreamingConverter");
+  ASSERT_TRUE(converter);
+  ASSERT_EQ(0u, converter->init(meta, zvec::ailego::Params()));
+
+  auto holder =
+      std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+          DIMENSION);
+  for (size_t i = 0; i < COUNT; ++i) {
+    zvec::ailego::NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = dist(gen);
+    }
+    holder->emplace(i + 1, vec);
+  }
+
+  ASSERT_EQ(0u, IndexConverter::TrainAndTransform(converter, holder));
+
+  auto reformer_params = converter->meta().reformer_params();
+  float scale = 0.0f;
+  ASSERT_TRUE(reformer_params.get("uniform_int8.reformer.scale", &scale));
+
+  auto reformer = IndexFactory::CreateReformer("UniformInt8StreamingReformer");
+  ASSERT_TRUE(reformer);
+  ASSERT_EQ(0u, reformer->init(reformer_params));
+
+  // Create mock results and verify normalize rescales by 1/scale^2
+  IndexDocumentList results;
+  float int8_score = 100.0f;
+  IndexDocument doc;
+  *doc.mutable_score() = int8_score;
+  results.push_back(doc);
+
+  // normalize is independent of query, pass nullptr
+  ASSERT_EQ(
+      0, reformer->normalize(
+             nullptr, IndexQueryMeta(IndexMeta::DataType::DT_FP32, DIMENSION),
+             results));
+
+  float expected_score = int8_score / (scale * scale);
+  EXPECT_NEAR(results[0].score(), expected_score, expected_score * 1e-5f);
+}
+
+// ---------------------------------------------------------------------------
+// InitConverterWithTrainedParams: simulate the search-only path where
+// scale/bias come from persisted converter params (no re-train needed)
+// ---------------------------------------------------------------------------
+TEST(UniformInt8Reformer, InitConverterWithTrainedParams) {
+  std::mt19937 gen(42);
+  std::uniform_real_distribution<float> dist(0.0f, 1.0f);
+
+  const size_t COUNT = 5000;
+  const size_t DIMENSION = 12;
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+
+  // First pass: train to get params
+  auto converter =
+      IndexFactory::CreateConverter("UniformInt8StreamingConverter");
+  ASSERT_TRUE(converter);
+  ASSERT_EQ(0u, converter->init(meta, zvec::ailego::Params()));
+
+  auto holder =
+      std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+          DIMENSION);
+  for (size_t i = 0; i < COUNT; ++i) {
+    zvec::ailego::NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = dist(gen);
+    }
+    holder->emplace(i + 1, vec);
+  }
+
+  ASSERT_EQ(0, converter->train(holder));
+  auto reformer_params = converter->meta().reformer_params();
+  auto converter_params = converter->meta().converter_params();
+
+  // Second pass: create a new converter with trained params (skip train)
+  auto converter2 =
+      IndexFactory::CreateConverter("UniformInt8StreamingConverter");
+  ASSERT_TRUE(converter2);
+  ASSERT_EQ(0, converter2->init(meta, converter_params));
+  ASSERT_EQ(0, converter2->transform(holder));
+
+  auto &stats = converter2->stats();
+  EXPECT_EQ(0u, stats.trained_count());
+  EXPECT_EQ(COUNT, stats.transformed_count());
+
+  auto holder2 = converter2->result();
+  ASSERT_TRUE(holder2);
+  EXPECT_EQ(COUNT, holder2->count());
+  EXPECT_EQ(IndexMeta::DataType::DT_INT8, holder2->data_type());
+  EXPECT_EQ(DIMENSION, holder2->dimension());
+
+  // Verify reformer with persisted params produces same results
+  auto reformer = IndexFactory::CreateReformer("UniformInt8StreamingReformer");
+  ASSERT_TRUE(reformer);
+  ASSERT_EQ(0u, reformer->init(reformer_params));
+
+  auto iter = holder->create_iterator();
+  auto iter2 = holder2->create_iterator();
+  std::string buffer;
+
+  for (; iter->is_valid(); iter->next(), iter2->next()) {
+    ASSERT_TRUE(iter2->is_valid());
+    std::string expected(reinterpret_cast<const char *>(iter2->data()),
+                         holder2->element_size());
+
+    IndexQueryMeta qmeta;
+    EXPECT_EQ(0, reformer->transform(
+                     iter->data(),
+                     IndexQueryMeta(holder->data_type(), holder->dimension()),
+                     &buffer, &qmeta));
+    EXPECT_EQ(IndexMeta::DataType::DT_INT8, qmeta.data_type());
+    EXPECT_EQ(DIMENSION, qmeta.dimension());
+    EXPECT_EQ(expected, buffer);
+
+    // convert() path
+    buffer.clear();
+    EXPECT_EQ(0, reformer->convert(
+                     iter->data(),
+                     IndexQueryMeta(holder->data_type(), holder->dimension()),
+                     &buffer, &qmeta));
+    EXPECT_EQ(expected, buffer);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// LosslessIntegerFastPath: when all training values are integers within
+// [0, 127], scale should be 1.0 for exact mapping
+// ---------------------------------------------------------------------------
+TEST(UniformInt8Reformer, LosslessIntegerFastPath) {
+  const size_t COUNT = 100;
+  const size_t DIMENSION = 8;
+
+  IndexMeta meta;
+  meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION);
+
+  auto converter =
+      IndexFactory::CreateConverter("UniformInt8StreamingConverter");
+  ASSERT_TRUE(converter);
+  ASSERT_EQ(0u, converter->init(meta, zvec::ailego::Params()));
+
+  auto holder =
+      std::make_shared<MultiPassIndexHolder<IndexMeta::DataType::DT_FP32>>(
+          DIMENSION);
+
+  // Fill with integer values in [0, 50]
+  std::mt19937 gen(10);
+  std::uniform_int_distribution<int> idist(0, 50);
+  for (size_t i = 0; i < COUNT; ++i) {
+    zvec::ailego::NumericalVector<float> vec(DIMENSION);
+    for (size_t j = 0; j < DIMENSION; ++j) {
+      vec[j] = static_cast<float>(idist(gen));
+    }
+    holder->emplace(i + 1, vec);
+  }
+
+  ASSERT_EQ(0u, IndexConverter::TrainAndTransform(converter, holder));
+
+  // scale should be 1.0 for lossless integer path
+  auto reformer_params = converter->meta().reformer_params();
+  float scale = 0.0f;
+  ASSERT_TRUE(reformer_params.get("uniform_int8.reformer.scale", &scale));
+  EXPECT_FLOAT_EQ(1.0f, scale);
+
+  // Verify exact round-trip for integer values
+  auto reformer = IndexFactory::CreateReformer("UniformInt8StreamingReformer");
+  ASSERT_TRUE(reformer);
+  ASSERT_EQ(0u, reformer->init(reformer_params));
+
+  auto iter = holder->create_iterator();
+  std::string quantized_buf, reverted_buf;
+
+  for (; iter->is_valid(); iter->next()) {
+    const float *original = reinterpret_cast<const float *>(iter->data());
+
+    IndexQueryMeta qmeta;
+    ASSERT_EQ(0, reformer->transform(
+                     iter->data(),
+                     IndexQueryMeta(holder->data_type(), holder->dimension()),
+                     &quantized_buf, &qmeta));
+
+    // Verify quantized values match original integers
+    const int8_t *quantized =
+        reinterpret_cast<const int8_t *>(quantized_buf.data());
+    for (size_t d = 0; d < DIMENSION; ++d) {
+      EXPECT_EQ(static_cast<int8_t>(original[d] - 0 /* global_min offset */),
+                quantized[d])
+          << "dim=" << d;
+    }
+
+    // Revert should give exact values back
+    ASSERT_EQ(0, reformer->revert(quantized_buf.data(), qmeta, &reverted_buf));
+    const float *reverted =
+        reinterpret_cast<const float *>(reverted_buf.data());
+    for (size_t d = 0; d < DIMENSION; ++d) {
+      EXPECT_FLOAT_EQ(original[d], reverted[d]) << "dim=" << d;
+    }
+  }
+}

tests/core/utility/CMakeLists.txt

@@ -0,0 +1,16 @@
+
+include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
+include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
+
+file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc)
+
+foreach(CC_SRCS ${ALL_TEST_SRCS})
+  get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE)
+  cc_gtest(
+    NAME ${CC_TARGET} STRICT
+    LIBS zvec_ailego core_framework core_utility
+    SRCS ${CC_SRCS}
+    INCS . ${PROJECT_ROOT_DIR}/src/core/
+  )
+  cc_test_suite(zvec_ailego ${CC_TARGET})
+endforeach()

tests/core/utility/buffer_storage_test.cpp

@@ -0,0 +1,70 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <gtest/gtest.h>
+#include <zvec/core/framework/index_factory.h>
+#include <zvec/core/framework/index_helper.h>
+
+using namespace zvec;
+using namespace zvec::core;
+
+TEST(BufferStorage, General) {
+  std::string file_path = "buffer_storage_test_file";
+  ailego::File::Delete(file_path);
+
+  auto write_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_TRUE(write_storage);
+  std::cout << file_path << std::endl;
+  EXPECT_NE(0, write_storage->open(file_path, false));
+
+  ailego::Params params;
+  EXPECT_EQ(0, write_storage->init(params));
+  std::cout << file_path << std::endl;
+  EXPECT_EQ(0, write_storage->open(file_path, true));
+
+  IndexMeta meta;
+  meta.set_trainer("trainer", 111, ailego::Params());
+  meta.set_searcher("searcher", 222, ailego::Params());
+  meta.set_builder("builder", 333, ailego::Params());
+
+  EXPECT_EQ(0, IndexHelper::SerializeToStorage(meta, write_storage.get()));
+  EXPECT_EQ(0, write_storage->append("AAAA", 1234));
+  EXPECT_EQ(0, write_storage->append("BBBB", 1234));
+  auto aaaa = write_storage->get("AAAA");
+  ASSERT_TRUE(aaaa);
+  auto aaaa1 = aaaa->clone();
+  ASSERT_TRUE(aaaa1);
+  std::string hello = "Hello world!!!";
+  EXPECT_EQ(hello.size(), aaaa1->write(0, hello.data(), hello.size()));
+  EXPECT_EQ(0, write_storage->close());
+
+  // Reopen it
+  auto read_storage = IndexFactory::CreateStorage("BufferStorage");
+  EXPECT_EQ(0, read_storage->open(file_path, false));
+
+  IndexMeta meta2;
+  EXPECT_EQ(0, IndexHelper::DeserializeFromStorage(read_storage.get(), &meta2));
+  EXPECT_EQ("trainer", meta2.trainer_name());
+  EXPECT_EQ("searcher", meta2.searcher_name());
+  EXPECT_EQ("builder", meta2.builder_name());
+  auto aaaa2 = read_storage->get("AAAA");
+  ASSERT_TRUE(aaaa2);
+  const void *data;
+  EXPECT_EQ(hello.size(), aaaa2->read(0, &data, hello.size()));
+  auto aaaa3 = aaaa2->clone();
+  ASSERT_TRUE(aaaa3);
+  EXPECT_EQ(hello.size(), aaaa3->read(0, &data, hello.size()));
+  EXPECT_EQ(hello, std::string((const char *)data, hello.size()));
+}

tests/core/utility/file_dumper_test.cc

@@ -0,0 +1,130 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <iostream>
+#include <gtest/gtest.h>
+#include "zvec/core/framework/index_factory.h"
+#include "zvec/core/framework/index_helper.h"
+#include "zvec/core/framework/index_segment_storage.h"
+
+using namespace zvec;
+using namespace zvec::core;
+
+TEST(FileDumper, General) {
+  std::string file_path = "file_dumper_test_file";
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_TRUE(dumper);
+
+  IndexMeta meta1;
+  meta1.set_trainer("index_trainer", 0, ailego::Params());
+  ASSERT_EQ(0, dumper->create(file_path));
+  EXPECT_EQ(0, IndexHelper::SerializeToDumper(meta1, dumper.get()));
+
+  for (size_t i = 0; i < 10; ++i) {
+    std::string hello = "Hello world!!! #" + std::to_string(i);
+    EXPECT_EQ(hello.size(), dumper->write(hello.data(), hello.size()));
+    EXPECT_EQ(0, dumper->append(std::to_string(i), hello.size(), 0, 0));
+  }
+  ASSERT_EQ(0, dumper->close());
+
+  auto container = IndexFactory::CreateStorage("MMapFileReadStorage");
+  ASSERT_TRUE(container);
+
+  ailego::Params params;
+  params.set("proxima.mmap_file.container.memory_locking", true);
+  params.set("proxima.mmap_file.container.memory_warmup", true);
+  params.set("proxima.mmap_file.container.checksum_validation", true);
+  ASSERT_EQ(0, container->init(params));
+
+  IndexMeta meta2;
+  EXPECT_EQ("", meta2.trainer_name());
+  ASSERT_EQ(0, container->open(file_path, false));
+  EXPECT_EQ(0, IndexHelper::DeserializeFromStorage(container.get(), &meta2));
+  EXPECT_EQ("index_trainer", meta2.trainer_name());
+
+  for (size_t i = 0; i < 10; ++i) {
+    auto seg = container->get(std::to_string(i));
+    const void *data = nullptr;
+    EXPECT_EQ(seg->data_size(), seg->read(0, &data, seg->data_size()));
+
+    std::string hello = "Hello world!!! #" + std::to_string(i);
+    EXPECT_EQ(hello, std::string((const char *)data, seg->data_size()));
+  }
+}
+
+TEST(IndexSegmentDumper, General) {
+  std::string file_path = "index_segment_dumper_test_file";
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_TRUE(dumper);
+  ASSERT_EQ(0, dumper->create(file_path));
+
+  {
+    IndexDumper::Pointer dumper2 =
+        std::make_shared<IndexSegmentDumper>(dumper, "AAAAA");
+
+    IndexMeta meta1;
+    meta1.set_trainer("index_trainer", 0, ailego::Params());
+    EXPECT_EQ(0, IndexHelper::SerializeToDumper(meta1, dumper2.get()));
+
+    for (size_t i = 0; i < 10; ++i) {
+      std::string hello = "A: Hello world!!! #" + std::to_string(i);
+      EXPECT_EQ(hello.size(), dumper2->write(hello.data(), hello.size()));
+      EXPECT_EQ(0, dumper2->append(std::to_string(i), hello.size(), 0, 0));
+    }
+  }
+
+  {
+    IndexDumper::Pointer dumper2 =
+        std::make_shared<IndexSegmentDumper>(dumper, "BBBBB");
+
+    IndexMeta meta1;
+    meta1.set_builder("index_builder", 0, ailego::Params());
+    EXPECT_EQ(0, IndexHelper::SerializeToDumper(meta1, dumper2.get()));
+
+    for (size_t i = 100; i < 110; ++i) {
+      std::string hello = "B: Hello world!!! #" + std::to_string(i);
+      EXPECT_EQ(hello.size(), dumper2->write(hello.data(), hello.size()));
+      EXPECT_EQ(0, dumper2->append(std::to_string(i), hello.size(), 0, 0));
+    }
+  }
+
+  {
+    IndexDumper::Pointer dumper2 =
+        std::make_shared<IndexSegmentDumper>(dumper, "CCCCC");
+
+    IndexMeta meta1;
+    meta1.set_converter("index_converter", 0, ailego::Params());
+    EXPECT_EQ(0, IndexHelper::SerializeToDumper(meta1, dumper2.get()));
+
+    for (size_t i = 1000; i < 1010; ++i) {
+      std::string hello = "C: Hello world!!! #" + std::to_string(i);
+      EXPECT_EQ(hello.size(), dumper2->write(hello.data(), hello.size()));
+      EXPECT_EQ(0, dumper2->append(std::to_string(i), hello.size(), 0, 0));
+    }
+  }
+  ASSERT_EQ(0, dumper->close());
+
+  ///// Read data with container
+
+  auto container = IndexFactory::CreateStorage("MMapFileReadStorage");
+  ASSERT_TRUE(container);
+
+  ailego::Params params;
+  params.set("proxima.mmap_file.container.memory_locking", true);
+  params.set("proxima.mmap_file.container.memory_warmup", true);
+  params.set("proxima.mmap_file.container.checksum_validation", true);
+  ASSERT_EQ(0, container->init(params));
+  ASSERT_EQ(0, container->open(file_path, false));
+}

tests/core/utility/memory_dumper_test.cc

@@ -0,0 +1,62 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <iostream>
+#include <gtest/gtest.h>
+#include "zvec/core/framework/index_factory.h"
+#include "zvec/core/framework/index_helper.h"
+
+using namespace zvec;
+using namespace zvec::core;
+
+TEST(MemoryDumper, General) {
+  std::string file_path = "memory_dumper_test_file";
+
+  auto dumper = IndexFactory::CreateDumper("MemoryDumper");
+  ASSERT_TRUE(dumper);
+
+  IndexMeta meta1;
+  meta1.set_trainer("index_trainer", 0, ailego::Params());
+  ASSERT_EQ(0, dumper->create(file_path));
+  EXPECT_EQ(0, IndexHelper::SerializeToDumper(meta1, dumper.get()));
+
+  for (size_t i = 0; i < 10; ++i) {
+    std::string hello = "Hello world!!! #" + std::to_string(i);
+    EXPECT_EQ(hello.size(), dumper->write(hello.data(), hello.size()));
+    EXPECT_EQ(0, dumper->append(std::to_string(i), hello.size(), 0, 0));
+  }
+  ASSERT_EQ(0, dumper->close());
+
+  auto container = IndexFactory::CreateStorage("MemoryReadStorage");
+  ASSERT_TRUE(container);
+
+  ailego::Params params;
+  params.set("memory.container.checksum_validation", true);
+  ASSERT_EQ(0, container->init(params));
+
+  IndexMeta meta2;
+  EXPECT_EQ("", meta2.trainer_name());
+  ASSERT_EQ(0, container->open(file_path, false));
+
+  EXPECT_EQ(0, IndexHelper::DeserializeFromStorage(container.get(), &meta2));
+  EXPECT_EQ("index_trainer", meta2.trainer_name());
+
+  for (size_t i = 0; i < 10; ++i) {
+    auto seg = container->get(std::to_string(i));
+    const void *data = nullptr;
+    EXPECT_EQ(seg->data_size(), seg->read(0, &data, seg->data_size()));
+
+    std::string hello = "Hello world!!! #" + std::to_string(i);
+    EXPECT_EQ(hello, std::string((const char *)data, seg->data_size()));
+  }
+}

tests/core/utility/mmap_file_container_test.cc

@@ -0,0 +1,109 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <iostream>
+#include <gtest/gtest.h>
+#include "zvec/core/framework/index_factory.h"
+#include "zvec/core/framework/index_helper.h"
+
+using namespace zvec;
+using namespace zvec::core;
+
+static int GenRandInt(int m, int n) {
+  static std::mt19937 gen((std::random_device())());
+  return std::uniform_int_distribution<int>(m, n)(gen);
+}
+
+static void AddRandomPadding(const std::string &in, const std::string &out,
+                             size_t header_padding_size,
+                             size_t footer_padding_size) {
+  ailego::File out_file;
+  out_file.create(out, 0);
+  for (size_t i = 0; i < header_padding_size; ++i) {
+    uint8_t r = GenRandInt(0, 255);
+    out_file.write(&r, 1);
+  }
+
+  ailego::File in_file;
+  ASSERT_TRUE(in_file.open(in, true));
+  std::string buf(in_file.size(), '\0');
+  ASSERT_EQ(buf.size(), in_file.read(&buf[0], buf.size()));
+  out_file.write(buf.data(), buf.size());
+
+  for (size_t i = 0; i < footer_padding_size; ++i) {
+    uint8_t r = GenRandInt(0, 255);
+    out_file.write(&r, 1);
+  }
+}
+
+TEST(MMapFileReadStorage, General) {
+  std::string file_path = "mmap_file_container_test_file";
+  std::string file_path_padding = "mmap_file_container_test_file_padding";
+
+  auto dumper = IndexFactory::CreateDumper("FileDumper");
+  ASSERT_TRUE(dumper);
+
+  IndexMeta meta1;
+  meta1.set_trainer("index_trainer", 0, ailego::Params());
+  ASSERT_EQ(0, dumper->create(file_path));
+  EXPECT_EQ(0, IndexHelper::SerializeToDumper(meta1, dumper.get()));
+
+  for (size_t i = 0; i < 21; ++i) {
+    std::string hello = "Hello world!!! #" + std::to_string(i);
+    EXPECT_EQ(hello.size(), dumper->write(hello.data(), hello.size()));
+    EXPECT_EQ(0, dumper->append(std::to_string(i), hello.size(), 0, 0));
+  }
+  ASSERT_EQ(0, dumper->close());
+  size_t header_paddings = GenRandInt(0, 1024);
+  size_t footer_paddings = GenRandInt(0, 1024);
+  AddRandomPadding(file_path, file_path_padding, header_paddings,
+                   footer_paddings);
+  ailego::File file;
+  file.open(file_path_padding, true);
+  int64_t header_offset =
+      GenRandInt(0, 1) ? header_paddings : header_paddings - file.size();
+  int64_t footer_offset =
+      (GenRandInt(0, 1) ? file.size() : 0) - footer_paddings;
+
+  auto container = IndexFactory::CreateStorage("MMapFileReadStorage");
+  ASSERT_TRUE(container);
+
+  ailego::Params params;
+  params.set("proxima.mmap_file.container.memory_locking", true);
+  params.set("proxima.mmap_file.container.memory_warmup", true);
+  params.set("proxima.mmap_file.container.checksum_validation", true);
+  params.set("proxima.mmap_file.container.header_offset", header_offset);
+  params.set("proxima.mmap_file.container.footer_offset", footer_offset);
+  ASSERT_EQ(0, container->init(params));
+
+  IndexMeta meta2;
+  EXPECT_EQ(0u, container->get_all().size());
+  EXPECT_EQ("", meta2.trainer_name());
+  EXPECT_EQ("", meta2.searcher_name());
+  ASSERT_EQ(0, container->open(file_path_padding, false));
+  EXPECT_EQ(0, IndexHelper::DeserializeFromStorage(container.get(), &meta2));
+  EXPECT_EQ(23u, container->get_all().size());
+  EXPECT_EQ("index_trainer", meta2.trainer_name());
+  EXPECT_EQ("", meta2.searcher_name());
+
+  for (size_t i = 0; i < 21; ++i) {
+    auto seg = container->get(std::to_string(i));
+    auto seg1 = seg->clone();
+
+    const void *data = nullptr;
+    EXPECT_EQ(seg1->data_size(), seg1->read(0, &data, seg1->data_size()));
+    std::string hello = "Hello world!!! #" + std::to_string(i);
+    EXPECT_EQ(hello, std::string((const char *)data, seg1->data_size()));
+  }
+  container->cleanup();
+}

tests/core/utility/mmap_file_storage_test.cpp

@@ -0,0 +1,99 @@
+// Copyright 2025-present the zvec project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fstream>
+#include <iostream>
+#include <gtest/gtest.h>
+#include <zvec/core/framework/index_factory.h>
+#include <zvec/core/framework/index_helper.h>
+
+using namespace zvec;
+using namespace zvec::core;
+
+TEST(MMapFileStorage, TestHugePage) {
+  std::string file_path = "/mnt/huge/mmap_file_storage_test_file";
+  // std::string file_path = "mmap_file_storage_test_file";
+  ailego::File::Delete(file_path);
+
+  auto write_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  ASSERT_TRUE(write_storage);
+
+  ailego::Params params;
+  params.set("proxima.mmap_file.storage.huge_page", true);
+  EXPECT_EQ(0, write_storage->init(params));
+  EXPECT_EQ(0, write_storage->open(file_path, true));
+
+  IndexMeta meta;
+  meta.set_trainer("trainer", 111, ailego::Params());
+  meta.set_searcher("searcher", 222, ailego::Params());
+  meta.set_builder("builder", 333, ailego::Params());
+
+  EXPECT_EQ(0, IndexHelper::SerializeToStorage(meta, write_storage.get()));
+  EXPECT_EQ(0, write_storage->append("AAAA", 1234));
+  EXPECT_EQ(0, write_storage->append("BBBB", 1234));
+  auto aaaa = write_storage->get("AAAA");
+  ASSERT_TRUE(aaaa);
+  auto aaaa1 = aaaa->clone();
+  ASSERT_TRUE(aaaa1);
+  std::string hello = "Hello world!!!";
+  EXPECT_EQ(hello.size(), aaaa1->write(0, hello.data(), hello.size()));
+  auto hasHugePageInUse = [&]() {
+    std::ifstream smaps("/proc/self/smaps");
+    if (!smaps.is_open()) {
+      std::cerr << "Cannot open /proc/self/smaps\n";
+      return false;
+    }
+
+    std::string line;
+    while (std::getline(smaps, line)) {
+      // 查找 KernelPageSize 行
+      if (line.find("KernelPageSize:") != std::string::npos) {
+        // 提取页大小（单位 kB）
+        size_t pos = line.find_first_of("0123456789");
+        if (pos != std::string::npos) {
+          uint64_t pageSizeKB = std::stoull(line.substr(pos));
+          // std::cerr << pageSizeKB << std::endl;
+          if (pageSizeKB > 4) {  // 普通页是 4kB，大于即为 HugePage
+            std::cout << "Found HugePage region with KernelPageSize: "
+                      << pageSizeKB << " kB\n";
+            return true;
+          }
+        }
+      }
+    }
+    return false;
+  };
+  if (!hasHugePageInUse()) {
+    EXPECT_EQ(0, 1);
+  }
+  EXPECT_EQ(0, write_storage->close());
+  // Reopen it
+  auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
+  EXPECT_EQ(0, write_storage->init(params));
+  EXPECT_EQ(0, read_storage->open(file_path, false));
+
+  IndexMeta meta2;
+  EXPECT_EQ(0, IndexHelper::DeserializeFromStorage(read_storage.get(), &meta2));
+  EXPECT_EQ("trainer", meta2.trainer_name());
+  EXPECT_EQ("searcher", meta2.searcher_name());
+  EXPECT_EQ("builder", meta2.builder_name());
+  auto aaaa2 = read_storage->get("AAAA");
+  ASSERT_TRUE(aaaa2);
+  const void *data;
+  EXPECT_EQ(hello.size(), aaaa2->read(0, &data, hello.size()));
+  auto aaaa3 = aaaa2->clone();
+  ASSERT_TRUE(aaaa3);
+  EXPECT_EQ(hello.size(), aaaa3->read(0, &data, hello.size()));
+  EXPECT_EQ(hello, std::string((const char *)data, hello.size()));
+}