chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,128 @@
|
||||
cmake_minimum_required(VERSION 3.13)
|
||||
cmake_policy(SET CMP0077 NEW)
|
||||
project(zvec-example-c++)
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
# Enable compile_commands.json
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
# --- Paths to Zvec ---
|
||||
# Allow custom host build directory, default to "build"
|
||||
if(NOT DEFINED HOST_BUILD_DIR)
|
||||
set(HOST_BUILD_DIR "build")
|
||||
endif()
|
||||
|
||||
get_filename_component(ZVEC_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE)
|
||||
set(ZVEC_INCLUDE_DIR ${ZVEC_ROOT_DIR}/src/include)
|
||||
set(ZVEC_LIB_DIR ${ZVEC_ROOT_DIR}/${HOST_BUILD_DIR}/lib)
|
||||
|
||||
# Add include and library search paths
|
||||
include_directories(${ZVEC_INCLUDE_DIR})
|
||||
set(ZVEC_LIB_SEARCH_DIRS ${ZVEC_LIB_DIR})
|
||||
|
||||
# Support multi-config builds (MSVC puts libs in Debug/Release subdirectories)
|
||||
if(CMAKE_BUILD_TYPE)
|
||||
set(ZVEC_CONFIG_LIB_DIR ${ZVEC_LIB_DIR}/${CMAKE_BUILD_TYPE})
|
||||
if(EXISTS "${ZVEC_CONFIG_LIB_DIR}")
|
||||
list(APPEND ZVEC_LIB_SEARCH_DIRS ${ZVEC_CONFIG_LIB_DIR})
|
||||
endif()
|
||||
endif()
|
||||
if(WIN32)
|
||||
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
|
||||
endif()
|
||||
|
||||
function(zvec_find_shared_library OUT_VAR LIB_NAME)
|
||||
unset(${OUT_VAR} CACHE)
|
||||
if(WIN32)
|
||||
find_library(${OUT_VAR}
|
||||
NAMES ${LIB_NAME}_shared ${LIB_NAME}
|
||||
PATHS ${ZVEC_LIB_SEARCH_DIRS}
|
||||
NO_DEFAULT_PATH
|
||||
NO_CMAKE_FIND_ROOT_PATH
|
||||
)
|
||||
else()
|
||||
set(ZVEC_ORIGINAL_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
|
||||
if(APPLE)
|
||||
set(CMAKE_FIND_LIBRARY_SUFFIXES ".dylib")
|
||||
else()
|
||||
set(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
|
||||
endif()
|
||||
find_library(${OUT_VAR}
|
||||
NAMES ${LIB_NAME}
|
||||
PATHS ${ZVEC_LIB_SEARCH_DIRS}
|
||||
NO_DEFAULT_PATH
|
||||
NO_CMAKE_FIND_ROOT_PATH
|
||||
)
|
||||
set(CMAKE_FIND_LIBRARY_SUFFIXES "${ZVEC_ORIGINAL_LIBRARY_SUFFIXES}")
|
||||
endif()
|
||||
set(${OUT_VAR} "${${OUT_VAR}}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(zvec_require_shared_library OUT_VAR LIB_NAME)
|
||||
zvec_find_shared_library(${OUT_VAR} ${LIB_NAME})
|
||||
if(NOT ${OUT_VAR})
|
||||
message(FATAL_ERROR
|
||||
"lib${LIB_NAME} shared library was not found in ${ZVEC_LIB_SEARCH_DIRS}. "
|
||||
"Build zvec first, or pass -DHOST_BUILD_DIR=<build-dir>.")
|
||||
endif()
|
||||
set(${OUT_VAR} "${${OUT_VAR}}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
zvec_require_shared_library(ZVEC_SHARED_LIBRARY zvec)
|
||||
zvec_require_shared_library(ZVEC_AILEGO_SHARED_LIBRARY zvec_ailego)
|
||||
zvec_require_shared_library(ZVEC_CORE_SHARED_LIBRARY zvec_core)
|
||||
|
||||
# --- Create INTERFACE target for libzvec (all-in-one C++ shared library) ---
|
||||
# libzvec.so/.dylib/.dll already bundles all zvec internal components
|
||||
# (zvec, zvec_core, zvec_ailego, zvec_turbo), so no individual dependency
|
||||
# libraries need to be specified by the consumer.
|
||||
add_library(zvec-lib INTERFACE)
|
||||
target_link_libraries(zvec-lib INTERFACE "${ZVEC_SHARED_LIBRARY}")
|
||||
|
||||
# --- Create INTERFACE target for libzvec_ailego (ailego-only all-in-one library) ---
|
||||
# The ailego example intentionally depends only on libzvec_ailego.
|
||||
add_library(zvec-ailego-lib INTERFACE)
|
||||
target_link_libraries(zvec-ailego-lib INTERFACE "${ZVEC_AILEGO_SHARED_LIBRARY}")
|
||||
|
||||
# --- Create INTERFACE target for libzvec_core (core-only all-in-one library) ---
|
||||
# The core example intentionally depends only on libzvec_core.
|
||||
add_library(zvec-core-lib INTERFACE)
|
||||
target_link_libraries(zvec-core-lib INTERFACE "${ZVEC_CORE_SHARED_LIBRARY}")
|
||||
|
||||
# --- Executables ---
|
||||
set(ZVEC_EXAMPLE_TARGETS)
|
||||
|
||||
add_executable(db-example db/main.cc)
|
||||
target_link_libraries(db-example PRIVATE zvec-lib)
|
||||
if(ANDROID)
|
||||
target_link_libraries(db-example PRIVATE log)
|
||||
endif()
|
||||
list(APPEND ZVEC_EXAMPLE_TARGETS db-example)
|
||||
|
||||
add_executable(ailego-example ailego/main.cc)
|
||||
target_link_libraries(ailego-example PRIVATE zvec-ailego-lib)
|
||||
list(APPEND ZVEC_EXAMPLE_TARGETS ailego-example)
|
||||
|
||||
add_executable(core-example core/main.cc)
|
||||
target_link_libraries(core-example PRIVATE zvec-core-lib)
|
||||
list(APPEND ZVEC_EXAMPLE_TARGETS core-example)
|
||||
|
||||
add_executable(external-vector-example core/external_vector_example.cc)
|
||||
target_link_libraries(external-vector-example PRIVATE zvec-core-lib)
|
||||
list(APPEND ZVEC_EXAMPLE_TARGETS external-vector-example)
|
||||
|
||||
# Strip symbols to reduce executable size
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "Release" AND ANDROID)
|
||||
foreach(ZVEC_EXAMPLE_TARGET ${ZVEC_EXAMPLE_TARGETS})
|
||||
add_custom_command(TARGET ${ZVEC_EXAMPLE_TARGET} POST_BUILD
|
||||
COMMAND ${CMAKE_STRIP} "$<TARGET_FILE:${ZVEC_EXAMPLE_TARGET}>"
|
||||
COMMENT "Stripping symbols from ${ZVEC_EXAMPLE_TARGET}")
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
# Optimize for size
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "Release" AND ANDROID AND ZVEC_EXAMPLE_TARGETS)
|
||||
set_property(TARGET ${ZVEC_EXAMPLE_TARGETS} PROPERTY COMPILE_FLAGS "-Os")
|
||||
set_property(TARGET ${ZVEC_EXAMPLE_TARGETS} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
|
||||
endif()
|
||||
@@ -0,0 +1,11 @@
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <zvec/ailego/utility/string_helper.h>
|
||||
|
||||
using namespace zvec;
|
||||
|
||||
int main() {
|
||||
std::string a{"hello world"};
|
||||
|
||||
std::cout << ailego::StringHelper::StartsWith(a, "hello") << std::endl;
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/// @file external_vector_example.cc
|
||||
/// @brief Demonstrates using HNSW index in external-vector mode.
|
||||
///
|
||||
/// In external-vector mode the index does NOT store raw vectors internally.
|
||||
/// Instead, a user-provided VectorSource is passed on every Add/Search call
|
||||
/// so the index can fetch vectors on demand. This is useful when vectors are
|
||||
/// already stored elsewhere (e.g. a columnar store, mmap file, remote storage)
|
||||
/// and you want to avoid duplicating them inside the index.
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <numeric>
|
||||
#include <vector>
|
||||
#include <zvec/core/interface/index.h>
|
||||
#include <zvec/core/interface/index_factory.h>
|
||||
#include <zvec/core/interface/index_param.h>
|
||||
#include <zvec/core/interface/index_param_builders.h>
|
||||
#include <zvec/core/interface/vector_source.h>
|
||||
|
||||
using namespace zvec::core_interface;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// A simple VectorSource backed by an in-memory float matrix.
|
||||
// In production this could be backed by mmap, a database, or remote storage.
|
||||
// ---------------------------------------------------------------------------
|
||||
class InMemoryVectorSource : public zvec::core::VectorSource {
|
||||
public:
|
||||
/// @param base Pointer to a contiguous float array of shape [n, dim].
|
||||
/// @param dim Dimensionality of each vector.
|
||||
InMemoryVectorSource(const float *base, uint32_t dim)
|
||||
: base_(base), dim_(dim) {}
|
||||
|
||||
const void *get_vector(uint32_t node_id) const override {
|
||||
return base_ + static_cast<size_t>(node_id) * dim_;
|
||||
}
|
||||
|
||||
private:
|
||||
const float *base_;
|
||||
uint32_t dim_;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper: generate random float vectors in [0, 1)
|
||||
// ---------------------------------------------------------------------------
|
||||
static std::vector<float> generate_random_vectors(uint32_t count,
|
||||
uint32_t dim) {
|
||||
std::vector<float> data(static_cast<size_t>(count) * dim);
|
||||
for (auto &v : data) {
|
||||
v = static_cast<float>(std::rand()) / static_cast<float>(RAND_MAX);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper: brute-force kNN (L2) for recall verification
|
||||
// ---------------------------------------------------------------------------
|
||||
static std::vector<uint32_t> brute_force_knn(const float *base, uint32_t n,
|
||||
uint32_t dim, const float *query,
|
||||
uint32_t topk) {
|
||||
std::vector<std::pair<float, uint32_t>> dists(n);
|
||||
for (uint32_t i = 0; i < n; ++i) {
|
||||
float dist = 0.0f;
|
||||
for (uint32_t d = 0; d < dim; ++d) {
|
||||
float diff = base[static_cast<size_t>(i) * dim + d] - query[d];
|
||||
dist += diff * diff;
|
||||
}
|
||||
dists[i] = {dist, i};
|
||||
}
|
||||
std::partial_sort(dists.begin(), dists.begin() + topk, dists.end());
|
||||
std::vector<uint32_t> result(topk);
|
||||
for (uint32_t i = 0; i < topk; ++i) {
|
||||
result[i] = dists[i].second;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
int main() {
|
||||
constexpr uint32_t kDimension = 32;
|
||||
constexpr uint32_t kDocCount = 200;
|
||||
constexpr uint32_t kTopK = 5;
|
||||
const std::string index_path = "external_vector_example.index";
|
||||
|
||||
// Clean up any previous run
|
||||
std::filesystem::remove_all(index_path);
|
||||
|
||||
// ------ Step 1: Generate random vector data (simulating external storage)
|
||||
std::srand(42);
|
||||
auto vectors = generate_random_vectors(kDocCount, kDimension);
|
||||
|
||||
// Wrap data in our VectorSource
|
||||
InMemoryVectorSource source(vectors.data(), kDimension);
|
||||
|
||||
// ------ Step 2: Build HNSW index with external-vector mode enabled
|
||||
auto param = HNSWIndexParamBuilder()
|
||||
.WithMetricType(MetricType::kL2sq)
|
||||
.WithDataType(DataType::DT_FP32)
|
||||
.WithDimension(kDimension)
|
||||
.WithIsSparse(false)
|
||||
.WithUseExternalVector(true) // <-- key setting
|
||||
.Build();
|
||||
|
||||
auto index = IndexFactory::CreateAndInitIndex(*param);
|
||||
if (!index) {
|
||||
std::cerr << "Failed to create index." << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ret = index->Open(
|
||||
index_path, StorageOptions{StorageOptions::StorageType::kMMAP, true});
|
||||
if (ret != 0) {
|
||||
std::cerr << "Failed to open index." << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// ------ Step 3: Add vectors using AddWithSource
|
||||
for (uint32_t i = 0; i < kDocCount; ++i) {
|
||||
VectorData vd;
|
||||
vd.vector =
|
||||
DenseVector{vectors.data() + static_cast<size_t>(i) * kDimension};
|
||||
ret = index->AddWithSource(vd, i, source);
|
||||
if (ret != 0) {
|
||||
std::cerr << "Failed to add doc " << i << std::endl;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
std::cout << "[OK] Added " << kDocCount << " vectors in external mode."
|
||||
<< std::endl;
|
||||
|
||||
// ------ Step 4: Search using SearchWithSource
|
||||
auto query_param =
|
||||
HNSWQueryParamBuilder().with_topk(kTopK).with_ef_search(64).build();
|
||||
|
||||
// Use the first vector as query
|
||||
const float *query_vec = vectors.data();
|
||||
VectorData query;
|
||||
query.vector = DenseVector{query_vec};
|
||||
|
||||
SearchResult result;
|
||||
ret = index->SearchWithSource(query, query_param, source, &result);
|
||||
if (ret != 0) {
|
||||
std::cerr << "Search failed." << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::cout << "[OK] Search returned " << result.doc_list_.size() << " results."
|
||||
<< std::endl;
|
||||
|
||||
// The closest vector to vectors[0] should be itself (doc_id=0)
|
||||
if (!result.doc_list_.empty() && result.doc_list_[0].key() == 0) {
|
||||
std::cout << "[OK] Nearest neighbor is doc_id=0 (self), score="
|
||||
<< result.doc_list_[0].score() << std::endl;
|
||||
}
|
||||
|
||||
// ------ Step 5: Verify recall against brute-force
|
||||
auto gt =
|
||||
brute_force_knn(vectors.data(), kDocCount, kDimension, query_vec, kTopK);
|
||||
uint32_t hits = 0;
|
||||
for (const auto &doc : result.doc_list_) {
|
||||
if (std::find(gt.begin(), gt.end(), static_cast<uint32_t>(doc.key())) !=
|
||||
gt.end()) {
|
||||
++hits;
|
||||
}
|
||||
}
|
||||
float recall = static_cast<float>(hits) / static_cast<float>(kTopK);
|
||||
std::cout << "[OK] Recall@" << kTopK << " = " << recall * 100.0f << "%"
|
||||
<< std::endl;
|
||||
|
||||
// ------ Step 6: Reopen index and search again (persistence verification)
|
||||
index->Close();
|
||||
std::cout << "[OK] Index closed." << std::endl;
|
||||
|
||||
// Must re-create index instance with same params before reopening
|
||||
index = IndexFactory::CreateAndInitIndex(*param);
|
||||
if (!index) {
|
||||
std::cerr << "Failed to re-create index for reopen." << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
ret = index->Open(index_path,
|
||||
StorageOptions{StorageOptions::StorageType::kMMAP, false});
|
||||
if (ret != 0) {
|
||||
std::cerr << "Failed to reopen index." << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
SearchResult result2;
|
||||
ret = index->SearchWithSource(query, query_param, source, &result2);
|
||||
if (ret != 0) {
|
||||
std::cerr << "Search after reopen failed." << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::cout << "[OK] After reopen: search returned " << result2.doc_list_.size()
|
||||
<< " results, top1 doc_id=" << result2.doc_list_[0].key()
|
||||
<< std::endl;
|
||||
|
||||
// Cleanup
|
||||
index->Close();
|
||||
std::filesystem::remove_all(index_path);
|
||||
std::cout << "\n=== External Vector Example Complete ===" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <zvec/core/interface/index.h>
|
||||
#include <zvec/core/interface/index_factory.h>
|
||||
#include <zvec/core/interface/index_param.h>
|
||||
#include <zvec/core/interface/index_param_builders.h>
|
||||
|
||||
using namespace zvec::core_interface;
|
||||
|
||||
constexpr uint32_t kDimension = 64;
|
||||
const std::string index_name{"test.index"};
|
||||
|
||||
Index::Pointer create_index(const BaseIndexParam::Pointer ¶m,
|
||||
int doc_num = 10) {
|
||||
auto index = IndexFactory::CreateAndInitIndex(*param);
|
||||
if (!index) {
|
||||
std::cout << "Failed to create index." << std::endl;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
int ret = index->Open(
|
||||
index_name, StorageOptions{StorageOptions::StorageType::kMMAP, true});
|
||||
if (ret != 0) {
|
||||
std::cout << "Failed to open index." << std::endl;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
for (int i = 0; i < doc_num; ++i) {
|
||||
std::vector<float> vector(kDimension, i / 10.0f + 0.1f);
|
||||
VectorData vector_data;
|
||||
vector_data.vector = DenseVector{vector.data()};
|
||||
ret = index->Add(vector_data, i);
|
||||
if (ret != 0) {
|
||||
std::cout << "Failed to add to index." << std::endl;
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
ret = index->Train();
|
||||
if (ret != 0) {
|
||||
std::cout << "Failed to train index." << std::endl;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return index;
|
||||
}
|
||||
|
||||
int main() {
|
||||
std::filesystem::remove(index_name);
|
||||
|
||||
auto param = HNSWIndexParamBuilder()
|
||||
.WithMetricType(MetricType::kInnerProduct)
|
||||
.WithDataType(DataType::DT_FP32)
|
||||
.WithDimension(kDimension)
|
||||
.WithIsSparse(false)
|
||||
.Build();
|
||||
auto index = create_index(param, 1);
|
||||
std::cout << "index stats: " << index->GetDocCount() << std::endl;
|
||||
|
||||
// query
|
||||
auto query_param = HNSWQueryParamBuilder()
|
||||
.with_topk(10)
|
||||
.with_fetch_vector(true)
|
||||
.with_ef_search(20)
|
||||
.build();
|
||||
|
||||
SearchResult result;
|
||||
VectorData query;
|
||||
std::vector<float> vector(kDimension, 0.1f);
|
||||
query.vector = DenseVector{vector.data()};
|
||||
int ret = index->Search(query, query_param, &result);
|
||||
if (ret != 0) {
|
||||
std::cout << "Failed to search index." << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::cout << "query results: " << result.doc_list_.size() << std::endl;
|
||||
if (result.doc_list_.size() == 0) {
|
||||
std::cout << "No results found." << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::cout << "key: " << result.doc_list_[0].key()
|
||||
<< ", score: " << result.doc_list_[0].score() << std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <zvec/db/collection.h>
|
||||
#include <zvec/db/doc.h>
|
||||
#include <zvec/db/schema.h>
|
||||
#include <zvec/db/status.h>
|
||||
|
||||
using namespace zvec;
|
||||
|
||||
Doc create_doc(const uint64_t doc_id, const CollectionSchema &schema,
|
||||
std::string pk = "") {
|
||||
Doc new_doc;
|
||||
if (pk.empty()) {
|
||||
pk = "pk_" + std::to_string(doc_id);
|
||||
}
|
||||
new_doc.set_pk(pk);
|
||||
|
||||
for (auto &field : schema.fields()) {
|
||||
switch (field->data_type()) {
|
||||
case DataType::BINARY: {
|
||||
std::string binary_str("binary_" + std::to_string(doc_id));
|
||||
new_doc.set<std::string>(field->name(), binary_str);
|
||||
break;
|
||||
}
|
||||
case DataType::BOOL:
|
||||
new_doc.set<bool>(field->name(), doc_id % 10 == 0);
|
||||
break;
|
||||
case DataType::INT32:
|
||||
new_doc.set<int32_t>(field->name(), (int32_t)doc_id);
|
||||
break;
|
||||
case DataType::INT64:
|
||||
new_doc.set<int64_t>(field->name(), (int64_t)doc_id);
|
||||
break;
|
||||
case DataType::UINT32:
|
||||
new_doc.set<uint32_t>(field->name(), (uint32_t)doc_id);
|
||||
break;
|
||||
case DataType::UINT64:
|
||||
new_doc.set<uint64_t>(field->name(), (uint64_t)doc_id);
|
||||
break;
|
||||
case DataType::FLOAT:
|
||||
new_doc.set<float>(field->name(), (float)doc_id);
|
||||
break;
|
||||
case DataType::DOUBLE:
|
||||
new_doc.set<double>(field->name(), (double)doc_id);
|
||||
break;
|
||||
case DataType::STRING:
|
||||
new_doc.set<std::string>(field->name(),
|
||||
"value_" + std::to_string(doc_id));
|
||||
break;
|
||||
case DataType::ARRAY_BINARY: {
|
||||
std::vector<std::string> bin_vec;
|
||||
for (size_t i = 0; i < (doc_id % 10); i++) {
|
||||
bin_vec.push_back("bin_" + std::to_string(i));
|
||||
}
|
||||
new_doc.set<std::vector<std::string>>(field->name(), bin_vec);
|
||||
break;
|
||||
}
|
||||
case DataType::ARRAY_BOOL:
|
||||
new_doc.set<std::vector<bool>>(field->name(),
|
||||
std::vector<bool>(10, doc_id % 10 == 0));
|
||||
break;
|
||||
case DataType::ARRAY_INT32:
|
||||
new_doc.set<std::vector<int32_t>>(
|
||||
field->name(), std::vector<int32_t>(10, (int32_t)doc_id));
|
||||
break;
|
||||
case DataType::ARRAY_INT64:
|
||||
new_doc.set<std::vector<int64_t>>(
|
||||
field->name(), std::vector<int64_t>(10, (int64_t)doc_id));
|
||||
break;
|
||||
case DataType::ARRAY_UINT32:
|
||||
new_doc.set<std::vector<uint32_t>>(
|
||||
field->name(), std::vector<uint32_t>(10, (uint32_t)doc_id));
|
||||
break;
|
||||
case DataType::ARRAY_UINT64:
|
||||
new_doc.set<std::vector<uint64_t>>(
|
||||
field->name(), std::vector<uint64_t>(10, (uint64_t)doc_id));
|
||||
break;
|
||||
case DataType::ARRAY_FLOAT:
|
||||
new_doc.set<std::vector<float>>(field->name(),
|
||||
std::vector<float>(10, (float)doc_id));
|
||||
break;
|
||||
case DataType::ARRAY_DOUBLE:
|
||||
new_doc.set<std::vector<double>>(
|
||||
field->name(), std::vector<double>(10, (double)doc_id));
|
||||
break;
|
||||
case DataType::ARRAY_STRING:
|
||||
new_doc.set<std::vector<std::string>>(
|
||||
field->name(),
|
||||
std::vector<std::string>(10, "value_" + std::to_string(doc_id)));
|
||||
break;
|
||||
case DataType::VECTOR_BINARY32:
|
||||
new_doc.set<std::vector<uint32_t>>(
|
||||
field->name(),
|
||||
std::vector<uint32_t>(field->dimension(), uint32_t(doc_id + 0.1)));
|
||||
break;
|
||||
case DataType::VECTOR_BINARY64:
|
||||
new_doc.set<std::vector<uint64_t>>(
|
||||
field->name(),
|
||||
std::vector<uint64_t>(field->dimension(), uint64_t(doc_id + 0.1)));
|
||||
break;
|
||||
case DataType::VECTOR_FP32:
|
||||
new_doc.set<std::vector<float>>(
|
||||
field->name(),
|
||||
std::vector<float>(field->dimension(), float(doc_id + 0.1)));
|
||||
break;
|
||||
case DataType::VECTOR_FP64:
|
||||
new_doc.set<std::vector<double>>(
|
||||
field->name(),
|
||||
std::vector<double>(field->dimension(), double(doc_id + 0.1)));
|
||||
break;
|
||||
case DataType::VECTOR_FP16:
|
||||
new_doc.set<std::vector<zvec::float16_t>>(
|
||||
field->name(), std::vector<zvec::float16_t>(
|
||||
field->dimension(), static_cast<zvec::float16_t>(
|
||||
float(doc_id + 0.1))));
|
||||
break;
|
||||
case DataType::VECTOR_INT8:
|
||||
new_doc.set<std::vector<int8_t>>(
|
||||
field->name(),
|
||||
std::vector<int8_t>(field->dimension(), (int8_t)doc_id));
|
||||
break;
|
||||
case DataType::VECTOR_INT16:
|
||||
new_doc.set<std::vector<int16_t>>(
|
||||
field->name(),
|
||||
std::vector<int16_t>(field->dimension(), (int16_t)doc_id));
|
||||
break;
|
||||
case DataType::SPARSE_VECTOR_FP16: {
|
||||
std::vector<uint32_t> indices;
|
||||
std::vector<zvec::float16_t> values;
|
||||
for (uint32_t i = 0; i < 100; i++) {
|
||||
indices.push_back(i);
|
||||
values.push_back(zvec::float16_t(float(doc_id + 0.1)));
|
||||
}
|
||||
std::pair<std::vector<uint32_t>, std::vector<zvec::float16_t>>
|
||||
sparse_float_vec;
|
||||
sparse_float_vec.first = indices;
|
||||
sparse_float_vec.second = values;
|
||||
new_doc.set<
|
||||
std::pair<std::vector<uint32_t>, std::vector<zvec::float16_t>>>(
|
||||
field->name(), sparse_float_vec);
|
||||
break;
|
||||
}
|
||||
case DataType::SPARSE_VECTOR_FP32: {
|
||||
std::vector<uint32_t> indices;
|
||||
std::vector<float> values;
|
||||
for (uint32_t i = 0; i < 100; i++) {
|
||||
indices.push_back(i);
|
||||
values.push_back(float(doc_id + 0.1));
|
||||
}
|
||||
std::pair<std::vector<uint32_t>, std::vector<float>> sparse_float_vec;
|
||||
sparse_float_vec.first = indices;
|
||||
sparse_float_vec.second = values;
|
||||
new_doc.set<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
||||
field->name(), sparse_float_vec);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
std::cout << "Unsupported data type: " << field->name() << std::endl;
|
||||
throw std::runtime_error("Unsupported vector data type");
|
||||
}
|
||||
}
|
||||
|
||||
return new_doc;
|
||||
}
|
||||
|
||||
CollectionSchema::Ptr create_schema() {
|
||||
auto schema = std::make_shared<CollectionSchema>("demo");
|
||||
schema->set_max_doc_count_per_segment(1000);
|
||||
|
||||
schema->add_field(std::make_shared<FieldSchema>(
|
||||
"id", DataType::INT64, false, std::make_shared<InvertIndexParams>(true)));
|
||||
schema->add_field(std::make_shared<FieldSchema>(
|
||||
"name", DataType::STRING, false,
|
||||
std::make_shared<InvertIndexParams>(false)));
|
||||
schema->add_field(
|
||||
std::make_shared<FieldSchema>("weight", DataType::FLOAT, true));
|
||||
|
||||
schema->add_field(std::make_shared<FieldSchema>(
|
||||
"dense", DataType::VECTOR_FP32, 128, false,
|
||||
std::make_shared<HnswIndexParams>(MetricType::IP)));
|
||||
schema->add_field(std::make_shared<FieldSchema>(
|
||||
"sparse", DataType::SPARSE_VECTOR_FP32, 0, false,
|
||||
std::make_shared<HnswIndexParams>(MetricType::IP)));
|
||||
|
||||
return schema;
|
||||
}
|
||||
|
||||
int main() {
|
||||
std::string path = "./demo";
|
||||
std::filesystem::remove_all(path);
|
||||
|
||||
auto schema = create_schema();
|
||||
CollectionOptions options{false, true};
|
||||
|
||||
auto result = Collection::CreateAndOpen(path, *schema, options);
|
||||
if (!result.has_value()) {
|
||||
std::cout << result.error().message() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::cout << "init stats: " << result.value()->Stats().value().to_string()
|
||||
<< std::endl;
|
||||
|
||||
auto coll = std::move(result).value();
|
||||
|
||||
// insert docs
|
||||
{
|
||||
auto doc1 = create_doc(0, *schema);
|
||||
std::vector<Doc> docs{doc1};
|
||||
auto res = coll->Insert(docs);
|
||||
if (!res.has_value()) {
|
||||
std::cout << res.error().message() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
std::cout << "after insert stats " << coll->Stats().value().to_string()
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
// optimize
|
||||
{
|
||||
auto res = coll->Optimize();
|
||||
if (!res.ok()) {
|
||||
std::cout << res.message() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
std::cout << "after optimize stats " << coll->Stats().value().to_string()
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
// query
|
||||
{
|
||||
SearchQuery query;
|
||||
query.topk_ = 10;
|
||||
query.target_.field_name_ = "dense";
|
||||
query.include_vector_ = true;
|
||||
std::vector<float> query_vector = std::vector<float>(128, 0.1);
|
||||
query.target_.set_vector(std::string((char *)query_vector.data(),
|
||||
query_vector.size() * sizeof(float)));
|
||||
auto res = coll->Query(query);
|
||||
if (!res.has_value()) {
|
||||
std::cout << res.error().message() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
std::cout << "query result: doc_count[" << res.value().size() << "]"
|
||||
<< std::endl;
|
||||
std::cout << "first doc: " << res.value()[0]->to_detail_string()
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
// close and reopen
|
||||
coll.reset();
|
||||
options.read_only_ = true;
|
||||
result = Collection::Open(path, options);
|
||||
if (!result.has_value()) {
|
||||
std::cout << result.error().message() << std::endl;
|
||||
return -1;
|
||||
}
|
||||
std::cout << "reopen stats: " << result.value()->Stats().value().to_string()
|
||||
<< std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
cmake_minimum_required(VERSION 3.13)
|
||||
cmake_policy(SET CMP0077 NEW)
|
||||
project(zvec-example-c)
|
||||
set(CMAKE_C_STANDARD 99)
|
||||
set(CMAKE_C_STANDARD_REQUIRED ON)
|
||||
|
||||
# Enable compile_commands.json
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
# --- Paths to Zvec and dependencies ---
|
||||
# Allow custom host build directory, default to "build"
|
||||
if(NOT DEFINED HOST_BUILD_DIR)
|
||||
set(HOST_BUILD_DIR "build")
|
||||
endif()
|
||||
|
||||
set(ZVEC_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../../../src/include)
|
||||
set(ZVEC_GENERATED_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../../../${HOST_BUILD_DIR}/src/generated)
|
||||
set(ZVEC_LIB_DIR ${CMAKE_BINARY_DIR}/../../../${HOST_BUILD_DIR}/lib)
|
||||
|
||||
# Add include and library search paths
|
||||
include_directories(${ZVEC_INCLUDE_DIR} ${ZVEC_GENERATED_INCLUDE_DIR})
|
||||
link_directories(${ZVEC_LIB_DIR})
|
||||
|
||||
# Support multi-config builds (MSVC puts libs in Debug/Release subdirectories)
|
||||
if(CMAKE_BUILD_TYPE)
|
||||
link_directories(${ZVEC_LIB_DIR}/${CMAKE_BUILD_TYPE})
|
||||
endif()
|
||||
|
||||
# Find required packages
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
# Create INTERFACE target for zvec_c_api (fat shared library)
|
||||
# No whole-archive flags needed — all symbols are already resolved in the .so/.dylib
|
||||
add_library(zvec-c-api INTERFACE)
|
||||
target_link_libraries(zvec-c-api INTERFACE
|
||||
zvec_c_api
|
||||
Threads::Threads
|
||||
)
|
||||
if(NOT WIN32)
|
||||
target_link_libraries(zvec-c-api INTERFACE ${CMAKE_DL_LIBS})
|
||||
endif()
|
||||
if(APPLE)
|
||||
target_link_options(zvec-c-api INTERFACE -Wl,-rpath,${ZVEC_LIB_DIR})
|
||||
endif()
|
||||
|
||||
# Basic example
|
||||
add_executable(c_api_basic_example basic_example.c)
|
||||
target_link_libraries(c_api_basic_example PRIVATE
|
||||
zvec-c-api
|
||||
)
|
||||
|
||||
# Schema example
|
||||
add_executable(c_api_collection_schema_example collection_schema_example.c)
|
||||
target_link_libraries(c_api_collection_schema_example PRIVATE
|
||||
zvec-c-api
|
||||
)
|
||||
|
||||
# Struct document example
|
||||
add_executable(c_api_doc_example doc_example.c)
|
||||
target_link_libraries(c_api_doc_example PRIVATE
|
||||
zvec-c-api
|
||||
)
|
||||
|
||||
# Index example
|
||||
add_executable(c_api_index_example index_example.c)
|
||||
target_link_libraries(c_api_index_example PRIVATE
|
||||
zvec-c-api
|
||||
)
|
||||
|
||||
# Field schema example
|
||||
add_executable(c_api_field_schema_example field_schema_example.c)
|
||||
target_link_libraries(c_api_field_schema_example PRIVATE
|
||||
zvec-c-api
|
||||
)
|
||||
|
||||
# Optimized example
|
||||
add_executable(c_api_optimized_example optimized_example.c)
|
||||
target_link_libraries(c_api_optimized_example PRIVATE
|
||||
zvec-c-api
|
||||
)
|
||||
|
||||
# DiskANN example
|
||||
add_executable(c_api_diskann_example diskann_example.c)
|
||||
target_link_libraries(c_api_diskann_example PRIVATE
|
||||
zvec-c-api
|
||||
)
|
||||
|
||||
# Strip symbols to reduce executable size
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "Release" AND (ANDROID OR (CMAKE_SYSTEM_NAME STREQUAL "Linux")))
|
||||
add_custom_command(TARGET c_api_basic_example POST_BUILD
|
||||
COMMAND ${CMAKE_STRIP} "$<TARGET_FILE:c_api_basic_example>"
|
||||
COMMENT "Stripping symbols from c_api_basic_example")
|
||||
add_custom_command(TARGET c_api_collection_schema_example POST_BUILD
|
||||
COMMAND ${CMAKE_STRIP} "$<TARGET_FILE:c_api_collection_schema_example>"
|
||||
COMMENT "Stripping symbols from c_api_collection_schema_example")
|
||||
add_custom_command(TARGET c_api_doc_example POST_BUILD
|
||||
COMMAND ${CMAKE_STRIP} "$<TARGET_FILE:c_api_doc_example>"
|
||||
COMMENT "Stripping symbols from c_api_doc_example")
|
||||
add_custom_command(TARGET c_api_index_example POST_BUILD
|
||||
COMMAND ${CMAKE_STRIP} "$<TARGET_FILE:c_api_index_example>"
|
||||
COMMENT "Stripping symbols from c_api_index_example")
|
||||
add_custom_command(TARGET c_api_field_schema_example POST_BUILD
|
||||
COMMAND ${CMAKE_STRIP} "$<TARGET_FILE:c_api_field_schema_example>"
|
||||
COMMENT "Stripping symbols from c_api_field_schema_example")
|
||||
add_custom_command(TARGET c_api_optimized_example POST_BUILD
|
||||
COMMAND ${CMAKE_STRIP} "$<TARGET_FILE:c_api_optimized_example>"
|
||||
COMMENT "Stripping symbols from c_api_optimized_example")
|
||||
add_custom_command(TARGET c_api_diskann_example POST_BUILD
|
||||
COMMAND ${CMAKE_STRIP} "$<TARGET_FILE:c_api_diskann_example>"
|
||||
COMMENT "Stripping symbols from c_api_diskann_example")
|
||||
endif()
|
||||
|
||||
# Optimize for size
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "Release" AND ANDROID)
|
||||
set_property(TARGET c_api_basic_example c_api_collection_schema_example c_api_doc_example
|
||||
c_api_index_example c_api_field_schema_example c_api_optimized_example
|
||||
c_api_diskann_example
|
||||
PROPERTY COMPILE_FLAGS "-Os")
|
||||
set_property(TARGET c_api_basic_example c_api_collection_schema_example c_api_doc_example
|
||||
c_api_index_example c_api_field_schema_example c_api_optimized_example
|
||||
c_api_diskann_example
|
||||
PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
|
||||
endif()
|
||||
@@ -0,0 +1,267 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "zvec/c_api.h"
|
||||
|
||||
/**
|
||||
* @brief Print error message and return error code
|
||||
*/
|
||||
static zvec_error_code_t handle_error(zvec_error_code_t error,
|
||||
const char *context) {
|
||||
if (error != ZVEC_OK) {
|
||||
char *error_msg = NULL;
|
||||
zvec_get_last_error(&error_msg);
|
||||
fprintf(stderr, "Error in %s: %d - %s\n", context, error,
|
||||
error_msg ? error_msg : "Unknown error");
|
||||
zvec_free(error_msg);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Create a simple test collection using CollectionSchema
|
||||
*/
|
||||
static zvec_error_code_t create_simple_test_collection(
|
||||
zvec_collection_t **collection) {
|
||||
// Create collection schema using C API
|
||||
zvec_collection_schema_t *schema =
|
||||
zvec_collection_schema_create("test_collection");
|
||||
if (!schema) {
|
||||
return ZVEC_ERROR_INTERNAL_ERROR;
|
||||
}
|
||||
|
||||
zvec_error_code_t error = ZVEC_OK;
|
||||
|
||||
// Create index parameters using new API
|
||||
zvec_index_params_t *invert_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_INVERT);
|
||||
if (!invert_params) {
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return ZVEC_ERROR_RESOURCE_EXHAUSTED;
|
||||
}
|
||||
zvec_index_params_set_invert_params(invert_params, true, false);
|
||||
|
||||
zvec_index_params_t *hnsw_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_HNSW);
|
||||
if (!hnsw_params) {
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return ZVEC_ERROR_RESOURCE_EXHAUSTED;
|
||||
}
|
||||
zvec_index_params_set_metric_type(hnsw_params, ZVEC_METRIC_TYPE_COSINE);
|
||||
zvec_index_params_set_hnsw_params(hnsw_params, 16, 200);
|
||||
|
||||
// Create and add ID field (primary key)
|
||||
zvec_field_schema_t *id_field =
|
||||
zvec_field_schema_create("id", ZVEC_DATA_TYPE_STRING, false, 0);
|
||||
zvec_field_schema_set_index_params(id_field, invert_params);
|
||||
error = zvec_collection_schema_add_field(schema, id_field);
|
||||
if (error != ZVEC_OK) {
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_index_params_destroy(hnsw_params);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return error;
|
||||
}
|
||||
|
||||
// Create text field (inverted index)
|
||||
zvec_field_schema_t *text_field =
|
||||
zvec_field_schema_create("text", ZVEC_DATA_TYPE_STRING, true, 0);
|
||||
zvec_field_schema_set_index_params(text_field, invert_params);
|
||||
error = zvec_collection_schema_add_field(schema, text_field);
|
||||
if (error != ZVEC_OK) {
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_index_params_destroy(hnsw_params);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return error;
|
||||
}
|
||||
|
||||
// Create embedding field (HNSW index)
|
||||
zvec_field_schema_t *embedding_field = zvec_field_schema_create(
|
||||
"embedding", ZVEC_DATA_TYPE_VECTOR_FP32, false, 3);
|
||||
zvec_field_schema_set_index_params(embedding_field, hnsw_params);
|
||||
error = zvec_collection_schema_add_field(schema, embedding_field);
|
||||
if (error != ZVEC_OK) {
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_index_params_destroy(hnsw_params);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return error;
|
||||
}
|
||||
|
||||
// Cleanup index parameters (they have been copied to the field schemas)
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_index_params_destroy(hnsw_params);
|
||||
|
||||
// Use default options
|
||||
zvec_collection_options_t *options = zvec_collection_options_create();
|
||||
if (!options) {
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return ZVEC_ERROR_RESOURCE_EXHAUSTED;
|
||||
}
|
||||
|
||||
// Create collection using the new API
|
||||
error = zvec_collection_create_and_open("./test_collection", schema, options,
|
||||
collection);
|
||||
|
||||
// Cleanup resources
|
||||
zvec_collection_options_destroy(options);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Basic C API usage example
|
||||
*/
|
||||
int main() {
|
||||
printf("=== ZVec C API Basic Example ===\n\n");
|
||||
|
||||
zvec_error_code_t error;
|
||||
|
||||
// Create collection using simplified function
|
||||
zvec_collection_t *collection = NULL;
|
||||
error = create_simple_test_collection(&collection);
|
||||
if (handle_error(error, "creating collection") != ZVEC_OK) {
|
||||
return 1;
|
||||
}
|
||||
printf("✓ Collection created successfully\n");
|
||||
|
||||
// Prepare test data
|
||||
float vector1[] = {0.1f, 0.2f, 0.3f};
|
||||
float vector2[] = {0.4f, 0.5f, 0.6f};
|
||||
|
||||
zvec_doc_t *docs[2];
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
docs[i] = zvec_doc_create();
|
||||
if (!docs[i]) {
|
||||
fprintf(stderr, "Failed to create document %d\n", i);
|
||||
// Cleanup allocated resources
|
||||
for (int j = 0; j < i; ++j) {
|
||||
zvec_doc_destroy(docs[j]);
|
||||
}
|
||||
return ZVEC_ERROR_INTERNAL_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
// Manually add fields to document 1
|
||||
zvec_doc_set_pk(docs[0], "doc1");
|
||||
zvec_doc_add_field_by_value(docs[0], "id", ZVEC_DATA_TYPE_STRING, "doc1",
|
||||
strlen("doc1"));
|
||||
zvec_doc_add_field_by_value(docs[0], "text", ZVEC_DATA_TYPE_STRING,
|
||||
"First document", strlen("First document"));
|
||||
zvec_doc_add_field_by_value(docs[0], "embedding", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
vector1, 3 * sizeof(float));
|
||||
|
||||
// Manually add fields to document 2
|
||||
zvec_doc_set_pk(docs[1], "doc2");
|
||||
zvec_doc_add_field_by_value(docs[1], "id", ZVEC_DATA_TYPE_STRING, "doc2",
|
||||
strlen("doc2"));
|
||||
zvec_doc_add_field_by_value(docs[1], "text", ZVEC_DATA_TYPE_STRING,
|
||||
"Second document", strlen("Second document"));
|
||||
zvec_doc_add_field_by_value(docs[1], "embedding", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
vector2, 3 * sizeof(float));
|
||||
|
||||
// Insert documents
|
||||
size_t success_count = 0;
|
||||
size_t error_count = 0;
|
||||
error = zvec_collection_insert(collection, (const zvec_doc_t **)docs, 2,
|
||||
&success_count, &error_count);
|
||||
if (handle_error(error, "inserting documents") != ZVEC_OK) {
|
||||
zvec_collection_destroy(collection);
|
||||
return 1;
|
||||
}
|
||||
printf("✓ Documents inserted - Success: %zu, Failed: %zu\n", success_count,
|
||||
error_count);
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
zvec_doc_destroy(docs[i]);
|
||||
}
|
||||
|
||||
// Flush collection
|
||||
error = zvec_collection_flush(collection);
|
||||
if (handle_error(error, "flushing collection") != ZVEC_OK) {
|
||||
printf("Collection flush failed\n");
|
||||
} else {
|
||||
printf("✓ Collection flushed successfully\n");
|
||||
}
|
||||
|
||||
// Get collection statistics
|
||||
zvec_collection_stats_t *stats = NULL;
|
||||
error = zvec_collection_get_stats(collection, &stats);
|
||||
if (handle_error(error, "getting collection stats") == ZVEC_OK) {
|
||||
printf("✓ Collection stats - Document count: %llu\n",
|
||||
(unsigned long long)zvec_collection_stats_get_doc_count(stats));
|
||||
// Free statistics memory
|
||||
zvec_collection_stats_destroy(stats);
|
||||
}
|
||||
|
||||
printf("Testing vector query...\n");
|
||||
// Query documents
|
||||
zvec_vector_query_t *query = zvec_vector_query_create();
|
||||
if (!query) {
|
||||
fprintf(stderr, "Failed to create vector query\n");
|
||||
zvec_collection_destroy(collection);
|
||||
return 1;
|
||||
}
|
||||
|
||||
zvec_vector_query_set_field_name(query, "embedding");
|
||||
zvec_vector_query_set_query_vector(query, vector1, 3 * sizeof(float));
|
||||
zvec_vector_query_set_topk(query, 10);
|
||||
zvec_vector_query_set_filter(query, "");
|
||||
zvec_vector_query_set_include_vector(query, true);
|
||||
zvec_vector_query_set_include_doc_id(query, true);
|
||||
|
||||
zvec_doc_t **results = NULL;
|
||||
size_t result_count = 0;
|
||||
error = zvec_collection_query(collection, (const zvec_vector_query_t *)query,
|
||||
&results, &result_count);
|
||||
|
||||
if (error != ZVEC_OK) {
|
||||
char *error_msg = NULL;
|
||||
zvec_get_last_error(&error_msg);
|
||||
printf("[ERROR] Query failed: %s\n",
|
||||
error_msg ? error_msg : "Unknown error");
|
||||
zvec_free(error_msg);
|
||||
zvec_vector_query_destroy(query);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
zvec_vector_query_destroy(query);
|
||||
|
||||
printf("✓ Query successful - Returned %zu results\n", result_count);
|
||||
|
||||
// Process query results
|
||||
for (size_t i = 0; i < result_count && i < 5; ++i) {
|
||||
const zvec_doc_t *doc = results[i];
|
||||
const char *pk = zvec_doc_get_pk_copy(doc);
|
||||
|
||||
printf(" Result %zu: PK=%s, DocID=%llu, Score=%.4f\n", i + 1,
|
||||
pk ? pk : "NULL", (unsigned long long)zvec_doc_get_doc_id(doc),
|
||||
zvec_doc_get_score(doc));
|
||||
|
||||
if (pk) {
|
||||
zvec_free((void *)pk);
|
||||
}
|
||||
}
|
||||
|
||||
// Free query results memory
|
||||
zvec_docs_free(results, result_count);
|
||||
|
||||
cleanup:
|
||||
// Cleanup resources
|
||||
zvec_collection_destroy(collection);
|
||||
printf("✓ Example completed\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "zvec/c_api.h"
|
||||
|
||||
/**
|
||||
* @brief Print error message and return error code
|
||||
*/
|
||||
static zvec_error_code_t handle_error(zvec_error_code_t error,
|
||||
const char *context) {
|
||||
if (error != ZVEC_OK) {
|
||||
char *error_msg = NULL;
|
||||
zvec_get_last_error(&error_msg);
|
||||
fprintf(stderr, "Error in %s: %d - %s\n", context, error,
|
||||
error_msg ? error_msg : "Unknown error");
|
||||
zvec_free(error_msg);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Collection schema creation and management example
|
||||
*/
|
||||
int main() {
|
||||
printf("=== ZVec Collection Schema Example ===\n\n");
|
||||
|
||||
zvec_error_code_t error;
|
||||
|
||||
// 1. Create collection schema
|
||||
zvec_collection_schema_t *schema =
|
||||
zvec_collection_schema_create("schema_example_collection");
|
||||
if (!schema) {
|
||||
fprintf(stderr, "Failed to create collection schema\n");
|
||||
return 1;
|
||||
}
|
||||
printf("✓ Collection schema created successfully\n");
|
||||
|
||||
// 2. Set schema properties
|
||||
zvec_collection_schema_set_max_doc_count_per_segment(schema, 1000000);
|
||||
printf("✓ Set max documents per segment: %llu\n",
|
||||
(unsigned long long)
|
||||
zvec_collection_schema_get_max_doc_count_per_segment(schema));
|
||||
|
||||
// 3. Create index parameters
|
||||
zvec_index_params_t *invert_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_INVERT);
|
||||
if (!invert_params) {
|
||||
fprintf(stderr, "Failed to create invert index parameters\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
zvec_index_params_set_invert_params(invert_params, true, false);
|
||||
|
||||
zvec_index_params_t *hnsw_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_HNSW);
|
||||
if (!hnsw_params) {
|
||||
fprintf(stderr, "Failed to create HNSW index parameters\n");
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(hnsw_params, ZVEC_METRIC_TYPE_L2);
|
||||
zvec_index_params_set_hnsw_params(hnsw_params, 16, 200);
|
||||
|
||||
// 4. Create and add ID field (primary key)
|
||||
zvec_field_schema_t *id_field =
|
||||
zvec_field_schema_create("id", ZVEC_DATA_TYPE_STRING, false, 0);
|
||||
if (!id_field) {
|
||||
fprintf(stderr, "Failed to create ID field\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
|
||||
error = zvec_collection_schema_add_field(schema, id_field);
|
||||
if (handle_error(error, "adding ID field") != ZVEC_OK) {
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
printf("✓ ID field added successfully\n");
|
||||
|
||||
// 5. Create and add text field with inverted index
|
||||
zvec_field_schema_t *text_field =
|
||||
zvec_field_schema_create("content", ZVEC_DATA_TYPE_STRING, true, 0);
|
||||
if (!text_field) {
|
||||
fprintf(stderr, "Failed to create text field\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
|
||||
zvec_field_schema_set_index_params(text_field, invert_params);
|
||||
error = zvec_collection_schema_add_field(schema, text_field);
|
||||
if (handle_error(error, "adding text field") != ZVEC_OK) {
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
printf("✓ Text field with inverted index added successfully\n");
|
||||
|
||||
// 6. Create and add vector field with HNSW index
|
||||
zvec_field_schema_t *vector_field = zvec_field_schema_create(
|
||||
"embedding", ZVEC_DATA_TYPE_VECTOR_FP32, false, 128);
|
||||
if (!vector_field) {
|
||||
fprintf(stderr, "Failed to create vector field\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
|
||||
zvec_field_schema_set_index_params(vector_field, hnsw_params);
|
||||
error = zvec_collection_schema_add_field(schema, vector_field);
|
||||
if (handle_error(error, "adding vector field") != ZVEC_OK) {
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
printf("✓ Vector field with HNSW index added successfully\n");
|
||||
|
||||
// 7. Check field count
|
||||
// Note: This function may not exist in current API, commenting out for now
|
||||
// size_t field_count = zvec_collection_schema_get_field_count(schema);
|
||||
// printf("✓ Total field count: %zu\n", field_count);
|
||||
|
||||
// 8. Create collection with schema
|
||||
zvec_collection_options_t *options = zvec_collection_options_create();
|
||||
if (!options) {
|
||||
fprintf(stderr, "Failed to create collection options\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
zvec_collection_t *collection = NULL;
|
||||
|
||||
error = zvec_collection_create_and_open("./schema_example_collection", schema,
|
||||
options, &collection);
|
||||
if (handle_error(error, "creating collection with schema") != ZVEC_OK) {
|
||||
zvec_collection_options_destroy(options);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
zvec_collection_options_destroy(options);
|
||||
printf("✓ Collection created successfully with schema\n");
|
||||
|
||||
// 9. Prepare test data
|
||||
float vector1[128];
|
||||
float vector2[128];
|
||||
for (int i = 0; i < 128; i++) {
|
||||
vector1[i] = (float)(i + 1) / 128.0f;
|
||||
vector2[i] = (float)(i + 2) / 128.0f;
|
||||
}
|
||||
|
||||
// 10. Create documents
|
||||
zvec_doc_t *docs[2];
|
||||
for (int i = 0; i < 2; i++) {
|
||||
docs[i] = zvec_doc_create();
|
||||
if (!docs[i]) {
|
||||
fprintf(stderr, "Failed to create document %d\n", i);
|
||||
// Cleanup
|
||||
for (int j = 0; j < i; j++) {
|
||||
zvec_doc_destroy(docs[j]);
|
||||
}
|
||||
zvec_collection_destroy(collection);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Add fields to document 1
|
||||
zvec_doc_set_pk(docs[0], "doc1");
|
||||
zvec_doc_add_field_by_value(docs[0], "id", ZVEC_DATA_TYPE_STRING, "doc1",
|
||||
strlen("doc1"));
|
||||
zvec_doc_add_field_by_value(docs[0], "content", ZVEC_DATA_TYPE_STRING,
|
||||
"First test document",
|
||||
strlen("First test document"));
|
||||
zvec_doc_add_field_by_value(docs[0], "embedding", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
vector1, 128 * sizeof(float));
|
||||
|
||||
// Add fields to document 2
|
||||
zvec_doc_set_pk(docs[1], "doc2");
|
||||
zvec_doc_add_field_by_value(docs[1], "id", ZVEC_DATA_TYPE_STRING, "doc2",
|
||||
strlen("doc2"));
|
||||
zvec_doc_add_field_by_value(docs[1], "content", ZVEC_DATA_TYPE_STRING,
|
||||
"Second test document",
|
||||
strlen("Second test document"));
|
||||
zvec_doc_add_field_by_value(docs[1], "embedding", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
vector2, 128 * sizeof(float));
|
||||
|
||||
// 11. Insert documents
|
||||
size_t success_count = 0, error_count = 0;
|
||||
error = zvec_collection_insert(collection, (const zvec_doc_t **)docs, 2,
|
||||
&success_count, &error_count);
|
||||
if (handle_error(error, "inserting documents") != ZVEC_OK) {
|
||||
// Cleanup
|
||||
for (int i = 0; i < 2; i++) {
|
||||
zvec_doc_destroy(docs[i]);
|
||||
}
|
||||
zvec_collection_destroy(collection);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
printf("✓ Documents inserted - Success: %zu, Failed: %zu\n", success_count,
|
||||
error_count);
|
||||
|
||||
// Cleanup documents
|
||||
for (int i = 0; i < 2; i++) {
|
||||
zvec_doc_destroy(docs[i]);
|
||||
}
|
||||
|
||||
// 12. Flush collection
|
||||
error = zvec_collection_flush(collection);
|
||||
if (handle_error(error, "flushing collection") == ZVEC_OK) {
|
||||
printf("✓ Collection flushed successfully\n");
|
||||
}
|
||||
|
||||
// 13. Query test
|
||||
zvec_vector_query_t *query = zvec_vector_query_create();
|
||||
if (!query) {
|
||||
fprintf(stderr, "Failed to create vector query\n");
|
||||
zvec_collection_destroy(collection);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return 1;
|
||||
}
|
||||
zvec_vector_query_set_field_name(query, "embedding");
|
||||
zvec_vector_query_set_query_vector(query, vector1, 128 * sizeof(float));
|
||||
zvec_vector_query_set_topk(query, 5);
|
||||
zvec_vector_query_set_filter(query, "");
|
||||
zvec_vector_query_set_include_vector(query, true);
|
||||
zvec_vector_query_set_include_doc_id(query, true);
|
||||
|
||||
zvec_doc_t **results = NULL;
|
||||
size_t result_count = 0;
|
||||
error = zvec_collection_query(collection, (const zvec_vector_query_t *)query,
|
||||
&results, &result_count);
|
||||
if (error == ZVEC_OK) {
|
||||
printf("✓ Vector query successful - Returned %zu results\n", result_count);
|
||||
zvec_docs_free(results, result_count);
|
||||
}
|
||||
zvec_vector_query_destroy(query);
|
||||
|
||||
// 14. Cleanup resources
|
||||
zvec_collection_destroy(collection);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
printf("✓ Schema example completed\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/**
|
||||
* @file diskann_example.c
|
||||
* @brief End-to-end example demonstrating DiskANN index usage via the C API.
|
||||
*
|
||||
* DiskANN is a disk-based approximate nearest neighbor search algorithm
|
||||
* optimized for large-scale datasets that exceed available memory. It uses
|
||||
* a Vamana graph structure combined with product quantization (PQ) to
|
||||
* achieve high recall with efficient disk I/O.
|
||||
*
|
||||
* NOTE: DiskANN requires Linux x86_64 with libaio. On other platforms the
|
||||
* example will compile but the runtime plugin will fail to load.
|
||||
*
|
||||
* Workflow demonstrated:
|
||||
* 1. Create collection schema with DiskANN-indexed vector field
|
||||
* 2. Insert documents with high-dimensional vectors
|
||||
* 3. Flush collection (triggers PQ training + graph build)
|
||||
* 4. Search using DiskANN query parameters (list_size controls recall)
|
||||
* 5. Clean up all resources
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "zvec/c_api.h"
|
||||
|
||||
/* --------------------------------------------------------------------------
|
||||
* Helpers
|
||||
* -------------------------------------------------------------------------- */
|
||||
|
||||
static zvec_error_code_t handle_error(zvec_error_code_t error,
|
||||
const char *context) {
|
||||
if (error != ZVEC_OK) {
|
||||
char *error_msg = NULL;
|
||||
zvec_get_last_error(&error_msg);
|
||||
fprintf(stderr, "Error in %s: %d - %s\n", context, error,
|
||||
error_msg ? error_msg : "Unknown error");
|
||||
zvec_free(error_msg);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
#define VECTOR_DIM 64
|
||||
#define NUM_DOCS 100
|
||||
#define COLLECTION_DIR "./diskann_example_collection"
|
||||
|
||||
/* --------------------------------------------------------------------------
|
||||
* Main
|
||||
* -------------------------------------------------------------------------- */
|
||||
|
||||
int main(void) {
|
||||
printf("=== ZVec DiskANN Index Example ===\n\n");
|
||||
|
||||
zvec_error_code_t error;
|
||||
int i;
|
||||
|
||||
/* ------------------------------------------------------------------
|
||||
* Step 1: Create collection schema
|
||||
* ------------------------------------------------------------------ */
|
||||
printf("[Step 1] Creating collection schema...\n");
|
||||
|
||||
zvec_collection_schema_t *schema =
|
||||
zvec_collection_schema_create("diskann_example");
|
||||
if (!schema) {
|
||||
fprintf(stderr, "Failed to create schema\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Index params — declared up-front and NULL-initialized so the
|
||||
* cleanup_schema path never touches an uninitialized pointer even if an
|
||||
* early field addition fails. */
|
||||
zvec_index_params_t *invert_params = NULL;
|
||||
zvec_index_params_t *diskann_params = NULL;
|
||||
|
||||
/* Scalar field with inverted index (for primary key / filtering) */
|
||||
invert_params = zvec_index_params_create(ZVEC_INDEX_TYPE_INVERT);
|
||||
zvec_index_params_set_invert_params(invert_params, true, false);
|
||||
|
||||
zvec_field_schema_t *id_field =
|
||||
zvec_field_schema_create("id", ZVEC_DATA_TYPE_STRING, false, 0);
|
||||
zvec_field_schema_set_index_params(id_field, invert_params);
|
||||
error = zvec_collection_schema_add_field(schema, id_field);
|
||||
if (handle_error(error, "adding id field") != ZVEC_OK) {
|
||||
goto cleanup_schema;
|
||||
}
|
||||
printf(" + id field (STRING, inverted index)\n");
|
||||
|
||||
/* Vector field with DiskANN index */
|
||||
diskann_params = zvec_index_params_create(ZVEC_INDEX_TYPE_DISKANN);
|
||||
if (!diskann_params) {
|
||||
fprintf(stderr, "Failed to create DiskANN index parameters\n");
|
||||
goto cleanup_schema;
|
||||
}
|
||||
zvec_index_params_set_metric_type(diskann_params, ZVEC_METRIC_TYPE_L2);
|
||||
zvec_index_params_set_diskann_params(
|
||||
diskann_params, 64, /* max_degree: graph connectivity */
|
||||
100, /* list_size: build-time candidates */
|
||||
8); /* pq_chunk_num: PQ chunks (0=auto) */
|
||||
|
||||
printf(
|
||||
" DiskANN index params: max_degree=%d, list_size=%d, pq_chunk_num=%d\n",
|
||||
zvec_index_params_get_diskann_max_degree(diskann_params),
|
||||
zvec_index_params_get_diskann_list_size(diskann_params),
|
||||
zvec_index_params_get_diskann_pq_chunk_num(diskann_params));
|
||||
|
||||
zvec_field_schema_t *embedding_field = zvec_field_schema_create(
|
||||
"embedding", ZVEC_DATA_TYPE_VECTOR_FP32, false, VECTOR_DIM);
|
||||
zvec_field_schema_set_index_params(embedding_field, diskann_params);
|
||||
error = zvec_collection_schema_add_field(schema, embedding_field);
|
||||
if (handle_error(error, "adding embedding field") != ZVEC_OK) {
|
||||
goto cleanup_schema;
|
||||
}
|
||||
printf(" + embedding field (VECTOR_FP32, %dD, DiskANN index)\n", VECTOR_DIM);
|
||||
|
||||
/* Index params are copied into field schemas; safe to destroy now */
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_index_params_destroy(diskann_params);
|
||||
invert_params = NULL;
|
||||
diskann_params = NULL;
|
||||
|
||||
/* ------------------------------------------------------------------
|
||||
* Step 2: Create and open collection
|
||||
* ------------------------------------------------------------------ */
|
||||
printf("\n[Step 2] Creating collection...\n");
|
||||
|
||||
zvec_collection_options_t *options = zvec_collection_options_create();
|
||||
zvec_collection_t *collection = NULL;
|
||||
error = zvec_collection_create_and_open(COLLECTION_DIR, schema, options,
|
||||
&collection);
|
||||
zvec_collection_options_destroy(options);
|
||||
if (handle_error(error, "creating collection") != ZVEC_OK) {
|
||||
goto cleanup_schema;
|
||||
}
|
||||
printf(" Collection created at %s\n", COLLECTION_DIR);
|
||||
|
||||
/* ------------------------------------------------------------------
|
||||
* Step 3: Generate and insert documents
|
||||
* ------------------------------------------------------------------ */
|
||||
printf("\n[Step 3] Inserting %d documents with %dD vectors...\n", NUM_DOCS,
|
||||
VECTOR_DIM);
|
||||
|
||||
/* Allocate vector storage */
|
||||
float(*vectors)[VECTOR_DIM] =
|
||||
(float(*)[VECTOR_DIM])malloc(NUM_DOCS * VECTOR_DIM * sizeof(float));
|
||||
if (!vectors) {
|
||||
fprintf(stderr, "Failed to allocate vector storage\n");
|
||||
goto cleanup_collection;
|
||||
}
|
||||
|
||||
/* Generate deterministic vector data */
|
||||
for (i = 0; i < NUM_DOCS; i++) {
|
||||
for (int d = 0; d < VECTOR_DIM; d++) {
|
||||
vectors[i][d] = (float)((i * VECTOR_DIM + d) % 1000) / 1000.0f;
|
||||
}
|
||||
}
|
||||
|
||||
/* Insert in batches */
|
||||
int batch_size = 20;
|
||||
size_t total_success = 0, total_error = 0;
|
||||
|
||||
for (int batch_start = 0; batch_start < NUM_DOCS; batch_start += batch_size) {
|
||||
int count = batch_start + batch_size > NUM_DOCS ? NUM_DOCS - batch_start
|
||||
: batch_size;
|
||||
|
||||
zvec_doc_t **docs =
|
||||
(zvec_doc_t **)malloc((size_t)count * sizeof(zvec_doc_t *));
|
||||
for (i = 0; i < count; i++) {
|
||||
int idx = batch_start + i;
|
||||
docs[i] = zvec_doc_create();
|
||||
|
||||
char pk[32];
|
||||
snprintf(pk, sizeof(pk), "doc_%04d", idx);
|
||||
zvec_doc_set_pk(docs[i], pk);
|
||||
|
||||
zvec_doc_add_field_by_value(docs[i], "id", ZVEC_DATA_TYPE_STRING, pk,
|
||||
strlen(pk));
|
||||
zvec_doc_add_field_by_value(docs[i], "embedding",
|
||||
ZVEC_DATA_TYPE_VECTOR_FP32, vectors[idx],
|
||||
VECTOR_DIM * sizeof(float));
|
||||
}
|
||||
|
||||
size_t success_count = 0, error_count = 0;
|
||||
error = zvec_collection_insert(collection, (const zvec_doc_t **)docs,
|
||||
(size_t)count, &success_count, &error_count);
|
||||
if (error != ZVEC_OK) {
|
||||
handle_error(error, "inserting batch");
|
||||
}
|
||||
total_success += success_count;
|
||||
total_error += error_count;
|
||||
|
||||
for (i = 0; i < count; i++) {
|
||||
zvec_doc_destroy(docs[i]);
|
||||
}
|
||||
free(docs);
|
||||
}
|
||||
printf(" Inserted: %zu succeeded, %zu failed\n", total_success, total_error);
|
||||
|
||||
/* ------------------------------------------------------------------
|
||||
* Step 4: Flush to trigger index build (PQ training + graph construction)
|
||||
* ------------------------------------------------------------------ */
|
||||
printf("\n[Step 4] Flushing collection (triggers DiskANN index build)...\n");
|
||||
|
||||
error = zvec_collection_flush(collection);
|
||||
if (handle_error(error, "flushing collection") != ZVEC_OK) {
|
||||
goto cleanup_vectors;
|
||||
}
|
||||
|
||||
zvec_collection_stats_t *stats = NULL;
|
||||
error = zvec_collection_get_stats(collection, &stats);
|
||||
if (error == ZVEC_OK && stats) {
|
||||
printf(" Document count after flush: %llu\n",
|
||||
(unsigned long long)zvec_collection_stats_get_doc_count(stats));
|
||||
zvec_collection_stats_destroy(stats);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------
|
||||
* Step 5: Search with DiskANN query parameters
|
||||
* ------------------------------------------------------------------ */
|
||||
printf("\n[Step 5] Searching with DiskANN query parameters...\n");
|
||||
|
||||
/* Create DiskANN query params — list_size controls the search frontier
|
||||
* (beam width). Larger values improve recall at the cost of latency. */
|
||||
zvec_diskann_query_params_t *da_qp = zvec_query_params_diskann_create(200);
|
||||
if (!da_qp) {
|
||||
fprintf(stderr, "Failed to create DiskANN query params\n");
|
||||
goto cleanup_vectors;
|
||||
}
|
||||
printf(" DiskANN query params: list_size=%d\n",
|
||||
zvec_query_params_diskann_get_list_size(da_qp));
|
||||
|
||||
/* Build the vector query */
|
||||
zvec_vector_query_t *query = zvec_vector_query_create();
|
||||
zvec_vector_query_set_field_name(query, "embedding");
|
||||
zvec_vector_query_set_query_vector(query, vectors[0],
|
||||
VECTOR_DIM * sizeof(float));
|
||||
zvec_vector_query_set_topk(query, 10);
|
||||
zvec_vector_query_set_include_vector(query, false);
|
||||
zvec_vector_query_set_include_doc_id(query, true);
|
||||
|
||||
/* Attach DiskANN query params (ownership transfers to query) */
|
||||
error = zvec_vector_query_set_diskann_params(query, da_qp);
|
||||
if (handle_error(error, "setting DiskANN query params") != ZVEC_OK) {
|
||||
zvec_vector_query_destroy(query);
|
||||
goto cleanup_vectors;
|
||||
}
|
||||
/* da_qp is now owned by query — do NOT call diskann_destroy on it */
|
||||
|
||||
/* Execute the query */
|
||||
zvec_doc_t **results = NULL;
|
||||
size_t result_count = 0;
|
||||
error = zvec_collection_query(collection, (const zvec_vector_query_t *)query,
|
||||
&results, &result_count);
|
||||
if (error != ZVEC_OK) {
|
||||
handle_error(error, "executing DiskANN query");
|
||||
printf(
|
||||
" (This is expected on non-Linux platforms — DiskANN requires "
|
||||
"libaio)\n");
|
||||
} else {
|
||||
printf(" Query returned %zu results:\n", result_count);
|
||||
for (size_t r = 0; r < result_count && r < 5; r++) {
|
||||
const char *pk = zvec_doc_get_pk_copy(results[r]);
|
||||
printf(" [%zu] pk=%s doc_id=%llu score=%.6f\n", r + 1,
|
||||
pk ? pk : "NULL",
|
||||
(unsigned long long)zvec_doc_get_doc_id(results[r]),
|
||||
zvec_doc_get_score(results[r]));
|
||||
if (pk) {
|
||||
zvec_free((void *)pk);
|
||||
}
|
||||
}
|
||||
if (result_count > 5) {
|
||||
printf(" ... and %zu more\n", result_count - 5);
|
||||
}
|
||||
zvec_docs_free(results, result_count);
|
||||
}
|
||||
zvec_vector_query_destroy(query);
|
||||
|
||||
/* ------------------------------------------------------------------
|
||||
* Step 6: Demonstrate list_size tuning (higher recall vs. lower latency)
|
||||
* ------------------------------------------------------------------ */
|
||||
printf("\n[Step 6] Tuning list_size for recall/latency trade-off...\n");
|
||||
|
||||
int list_sizes[] = {50, 100, 300};
|
||||
for (int li = 0; li < 3; li++) {
|
||||
zvec_diskann_query_params_t *tune_qp =
|
||||
zvec_query_params_diskann_create(list_sizes[li]);
|
||||
|
||||
zvec_vector_query_t *tune_query = zvec_vector_query_create();
|
||||
zvec_vector_query_set_field_name(tune_query, "embedding");
|
||||
zvec_vector_query_set_query_vector(tune_query, vectors[0],
|
||||
VECTOR_DIM * sizeof(float));
|
||||
zvec_vector_query_set_topk(tune_query, 10);
|
||||
zvec_vector_query_set_include_doc_id(tune_query, true);
|
||||
zvec_vector_query_set_diskann_params(tune_query, tune_qp);
|
||||
|
||||
zvec_doc_t **tune_results = NULL;
|
||||
size_t tune_count = 0;
|
||||
error = zvec_collection_query(collection,
|
||||
(const zvec_vector_query_t *)tune_query,
|
||||
&tune_results, &tune_count);
|
||||
if (error == ZVEC_OK) {
|
||||
printf(" list_size=%3d -> %zu results returned\n", list_sizes[li],
|
||||
tune_count);
|
||||
zvec_docs_free(tune_results, tune_count);
|
||||
} else {
|
||||
printf(" list_size=%3d -> query failed (expected on non-Linux)\n",
|
||||
list_sizes[li]);
|
||||
}
|
||||
zvec_vector_query_destroy(tune_query);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------
|
||||
* Cleanup
|
||||
* ------------------------------------------------------------------ */
|
||||
cleanup_vectors:
|
||||
free(vectors);
|
||||
|
||||
cleanup_collection:
|
||||
zvec_collection_destroy(collection);
|
||||
|
||||
cleanup_schema:
|
||||
zvec_collection_schema_destroy(schema);
|
||||
if (invert_params) {
|
||||
zvec_index_params_destroy(invert_params);
|
||||
}
|
||||
if (diskann_params) {
|
||||
zvec_index_params_destroy(diskann_params);
|
||||
}
|
||||
|
||||
printf("\n DiskANN index type string: %s\n",
|
||||
zvec_index_type_to_string(ZVEC_INDEX_TYPE_DISKANN));
|
||||
printf("=== Example completed ===\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,539 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <math.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "zvec/c_api.h"
|
||||
|
||||
/**
|
||||
* @brief Print error message and return error code
|
||||
*/
|
||||
static zvec_error_code_t handle_error(zvec_error_code_t error,
|
||||
const char *context) {
|
||||
if (error != ZVEC_OK) {
|
||||
char *error_msg = NULL;
|
||||
zvec_get_last_error(&error_msg);
|
||||
fprintf(stderr, "Error in %s: %d - %s\n", context, error,
|
||||
error_msg ? error_msg : "Unknown error");
|
||||
zvec_free(error_msg);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Create a test document with all data types
|
||||
* @param doc_index Document index for generating unique data
|
||||
* @return zvec_doc_t* Created document pointer
|
||||
*/
|
||||
static zvec_doc_t *create_full_type_test_doc(int doc_index) {
|
||||
zvec_doc_t *doc = zvec_doc_create();
|
||||
if (!doc) {
|
||||
fprintf(stderr, "Failed to create document\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Set primary key
|
||||
char pk_buffer[32];
|
||||
snprintf(pk_buffer, sizeof(pk_buffer), "doc_%d", doc_index);
|
||||
zvec_doc_set_pk(doc, pk_buffer);
|
||||
|
||||
// Add Id field with inverted index
|
||||
char id_buffer[32];
|
||||
snprintf(id_buffer, sizeof(id_buffer), "id_%d", doc_index);
|
||||
zvec_doc_add_field_by_value(doc, "id", ZVEC_DATA_TYPE_STRING, id_buffer,
|
||||
strlen(id_buffer));
|
||||
|
||||
// Add scalar fields with different data types
|
||||
// String field
|
||||
char string_value[64];
|
||||
snprintf(string_value, sizeof(string_value), "test_string_%d", doc_index);
|
||||
zvec_doc_add_field_by_value(doc, "string_field", ZVEC_DATA_TYPE_STRING,
|
||||
string_value, strlen(string_value));
|
||||
|
||||
// Boolean field
|
||||
bool bool_value = (doc_index % 2 == 0);
|
||||
zvec_doc_add_field_by_value(doc, "bool_field", ZVEC_DATA_TYPE_BOOL,
|
||||
&bool_value, sizeof(bool_value));
|
||||
|
||||
// Integer fields
|
||||
int32_t int32_value = doc_index * 1000;
|
||||
zvec_doc_add_field_by_value(doc, "int32_field", ZVEC_DATA_TYPE_INT32,
|
||||
&int32_value, sizeof(int32_value));
|
||||
|
||||
int64_t int64_value = (int64_t)doc_index * 1000000LL;
|
||||
zvec_doc_add_field_by_value(doc, "int64_field", ZVEC_DATA_TYPE_INT64,
|
||||
&int64_value, sizeof(int64_value));
|
||||
|
||||
// Floating point fields
|
||||
float float_value = (float)doc_index * 1.5f;
|
||||
zvec_doc_add_field_by_value(doc, "float_field", ZVEC_DATA_TYPE_FLOAT,
|
||||
&float_value, sizeof(float_value));
|
||||
|
||||
double double_value = (double)doc_index * 2.718281828;
|
||||
zvec_doc_add_field_by_value(doc, "double_field", ZVEC_DATA_TYPE_DOUBLE,
|
||||
&double_value, sizeof(double_value));
|
||||
|
||||
// Vector fields with different dimensions
|
||||
// FP32 vector (3D)
|
||||
float fp32_vector[3] = {(float)doc_index, (float)doc_index * 2.0f,
|
||||
(float)doc_index * 3.0f};
|
||||
zvec_doc_add_field_by_value(doc, "vector_fp32", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
fp32_vector, 3 * sizeof(float));
|
||||
|
||||
// Larger FP32 vector (16D)
|
||||
float large_vector[16];
|
||||
for (int i = 0; i < 16; i++) {
|
||||
large_vector[i] = (float)(doc_index * 16 + i) / 256.0f;
|
||||
}
|
||||
zvec_doc_add_field_by_value(doc, "large_vector", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
large_vector, 16 * sizeof(float));
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Compare two documents for equality
|
||||
*/
|
||||
static bool compare_documents(const zvec_doc_t *doc1, const zvec_doc_t *doc2) {
|
||||
if (!doc1 || !doc2) return false;
|
||||
|
||||
// Compare primary keys
|
||||
const char *pk1 = zvec_doc_get_pk_pointer(doc1);
|
||||
const char *pk2 = zvec_doc_get_pk_pointer(doc2);
|
||||
|
||||
if (!pk1 || !pk2 || strcmp(pk1, pk2) != 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// TODO: Compare other fields and values
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Print document fields and their values
|
||||
* @param doc The document to print
|
||||
* @param doc_index Document index for identification
|
||||
*/
|
||||
static void print_doc(const zvec_doc_t *doc, int doc_index) {
|
||||
if (!doc) {
|
||||
printf("Document %d: NULL document\n", doc_index);
|
||||
return;
|
||||
}
|
||||
|
||||
printf("\n=== Document %d ===\n", doc_index);
|
||||
|
||||
// Print primary key
|
||||
const char *pk = zvec_doc_get_pk_pointer(doc);
|
||||
printf("Primary Key: %s\n", pk ? pk : "NULL");
|
||||
|
||||
// Print document ID
|
||||
uint64_t doc_id = zvec_doc_get_doc_id(doc);
|
||||
printf("Document ID: %llu\n", (unsigned long long)doc_id);
|
||||
|
||||
// Print score
|
||||
float score = zvec_doc_get_score(doc);
|
||||
printf("Score: %.6f\n", score);
|
||||
|
||||
// Print scalar fields
|
||||
printf("\nScalar Fields:\n");
|
||||
|
||||
// ID field (using pointer function for strings)
|
||||
const void *id_value = NULL;
|
||||
size_t id_size = 0;
|
||||
zvec_error_code_t error = zvec_doc_get_field_value_pointer(
|
||||
doc, "id", ZVEC_DATA_TYPE_STRING, &id_value, &id_size);
|
||||
if (error == ZVEC_OK && id_value) {
|
||||
printf(" id: %.*s\n", (int)id_size, (const char *)id_value);
|
||||
}
|
||||
|
||||
// String field (using pointer function for strings)
|
||||
const void *string_value = NULL;
|
||||
size_t string_size = 0;
|
||||
error = zvec_doc_get_field_value_pointer(
|
||||
doc, "string_field", ZVEC_DATA_TYPE_STRING, &string_value, &string_size);
|
||||
if (error == ZVEC_OK && string_value) {
|
||||
printf(" string_field: %.*s\n", (int)string_size,
|
||||
(const char *)string_value);
|
||||
}
|
||||
|
||||
// Boolean field
|
||||
bool bool_value;
|
||||
error = zvec_doc_get_field_value_basic(doc, "bool_field", ZVEC_DATA_TYPE_BOOL,
|
||||
&bool_value, sizeof(bool_value));
|
||||
if (error == ZVEC_OK) {
|
||||
printf(" bool_field: %s\n", bool_value ? "true" : "false");
|
||||
}
|
||||
|
||||
// Int32 field
|
||||
int32_t int32_value;
|
||||
error =
|
||||
zvec_doc_get_field_value_basic(doc, "int32_field", ZVEC_DATA_TYPE_INT32,
|
||||
&int32_value, sizeof(int32_value));
|
||||
if (error == ZVEC_OK) {
|
||||
printf(" int32_field: %d\n", int32_value);
|
||||
}
|
||||
|
||||
// Int64 field
|
||||
int64_t int64_value;
|
||||
error =
|
||||
zvec_doc_get_field_value_basic(doc, "int64_field", ZVEC_DATA_TYPE_INT64,
|
||||
&int64_value, sizeof(int64_value));
|
||||
if (error == ZVEC_OK) {
|
||||
printf(" int64_field: %lld\n", (long long)int64_value);
|
||||
}
|
||||
|
||||
// Float field
|
||||
float float_value;
|
||||
error =
|
||||
zvec_doc_get_field_value_basic(doc, "float_field", ZVEC_DATA_TYPE_FLOAT,
|
||||
&float_value, sizeof(float_value));
|
||||
if (error == ZVEC_OK) {
|
||||
printf(" float_field: %.6f\n", float_value);
|
||||
}
|
||||
|
||||
// Double field
|
||||
double double_value;
|
||||
error =
|
||||
zvec_doc_get_field_value_basic(doc, "double_field", ZVEC_DATA_TYPE_DOUBLE,
|
||||
&double_value, sizeof(double_value));
|
||||
if (error == ZVEC_OK) {
|
||||
printf(" double_field: %.6f\n", double_value);
|
||||
}
|
||||
|
||||
// Print vector fields (using copy function for complex types)
|
||||
printf("\nVector Fields:\n");
|
||||
|
||||
// FP32 vector (3D)
|
||||
void *fp32_vector = NULL;
|
||||
size_t fp32_size = 0;
|
||||
error = zvec_doc_get_field_value_copy(
|
||||
doc, "vector_fp32", ZVEC_DATA_TYPE_VECTOR_FP32, &fp32_vector, &fp32_size);
|
||||
if (error == ZVEC_OK && fp32_vector) {
|
||||
const float *vec = (const float *)fp32_vector;
|
||||
size_t dim = fp32_size / sizeof(float);
|
||||
printf(" vector_fp32 (%zuD): [", dim);
|
||||
for (size_t i = 0; i < dim && i < 10; i++) { // Limit to first 10 elements
|
||||
printf("%.3f", vec[i]);
|
||||
if (i < dim - 1 && i < 9) printf(", ");
|
||||
}
|
||||
if (dim > 10) printf(", ...");
|
||||
printf("]\n");
|
||||
zvec_free(fp32_vector); // Free the allocated memory
|
||||
}
|
||||
|
||||
// Large vector (16D)
|
||||
void *large_vector = NULL;
|
||||
size_t large_size = 0;
|
||||
error = zvec_doc_get_field_value_copy(doc, "large_vector",
|
||||
ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
&large_vector, &large_size);
|
||||
if (error == ZVEC_OK && large_vector) {
|
||||
const float *vec = (const float *)large_vector;
|
||||
size_t dim = large_size / sizeof(float);
|
||||
printf(" large_vector (%zuD): [", dim);
|
||||
for (size_t i = 0; i < dim && i < 10; i++) { // Limit to first 10 elements
|
||||
printf("%.3f", vec[i]);
|
||||
if (i < dim - 1 && i < 9) printf(", ");
|
||||
}
|
||||
if (dim > 10) printf(", ...");
|
||||
printf("]\n");
|
||||
zvec_free(large_vector); // Free the allocated memory
|
||||
}
|
||||
|
||||
printf("==================\n\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Document creation, manipulation, and query example
|
||||
*/
|
||||
int main() {
|
||||
printf("=== ZVec Document Example ===\n\n");
|
||||
|
||||
zvec_error_code_t error;
|
||||
|
||||
// 1. Create collection schema for document testing
|
||||
zvec_collection_schema_t *schema =
|
||||
zvec_collection_schema_create("doc_example_collection");
|
||||
if (!schema) {
|
||||
fprintf(stderr, "Failed to create collection schema\n");
|
||||
return -1;
|
||||
}
|
||||
printf("✓ Collection schema created\n");
|
||||
|
||||
// 2. Create index parameters
|
||||
zvec_index_params_t *invert_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_INVERT);
|
||||
if (!invert_params) {
|
||||
fprintf(stderr, "Failed to create invert index parameters\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_invert_params(invert_params, true, false);
|
||||
|
||||
zvec_index_params_t *hnsw_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_HNSW);
|
||||
if (!hnsw_params) {
|
||||
fprintf(stderr, "Failed to create HNSW index parameters\n");
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(hnsw_params, ZVEC_METRIC_TYPE_L2);
|
||||
zvec_index_params_set_hnsw_params(hnsw_params, 16, 200);
|
||||
|
||||
// 3. Create fields for all data types
|
||||
printf("Creating fields for all data types...\n");
|
||||
|
||||
// Id field with inverted index
|
||||
zvec_field_schema_t *id_field =
|
||||
zvec_field_schema_create("id", ZVEC_DATA_TYPE_STRING, false, 0);
|
||||
if (id_field) {
|
||||
zvec_field_schema_set_index_params(id_field, invert_params);
|
||||
error = zvec_collection_schema_add_field(schema, id_field);
|
||||
if (handle_error(error, "adding ID field") == ZVEC_OK) {
|
||||
printf("✓ ID field with inverted index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Scalar fields
|
||||
zvec_field_schema_t *string_field =
|
||||
zvec_field_schema_create("string_field", ZVEC_DATA_TYPE_STRING, true, 0);
|
||||
zvec_field_schema_t *bool_field =
|
||||
zvec_field_schema_create("bool_field", ZVEC_DATA_TYPE_BOOL, true, 0);
|
||||
zvec_field_schema_t *int32_field =
|
||||
zvec_field_schema_create("int32_field", ZVEC_DATA_TYPE_INT32, true, 0);
|
||||
zvec_field_schema_t *int64_field =
|
||||
zvec_field_schema_create("int64_field", ZVEC_DATA_TYPE_INT64, true, 0);
|
||||
zvec_field_schema_t *float_field =
|
||||
zvec_field_schema_create("float_field", ZVEC_DATA_TYPE_FLOAT, true, 0);
|
||||
zvec_field_schema_t *double_field =
|
||||
zvec_field_schema_create("double_field", ZVEC_DATA_TYPE_DOUBLE, true, 0);
|
||||
|
||||
if (string_field) zvec_collection_schema_add_field(schema, string_field);
|
||||
if (bool_field) zvec_collection_schema_add_field(schema, bool_field);
|
||||
if (int32_field) zvec_collection_schema_add_field(schema, int32_field);
|
||||
if (int64_field) zvec_collection_schema_add_field(schema, int64_field);
|
||||
if (float_field) zvec_collection_schema_add_field(schema, float_field);
|
||||
if (double_field) zvec_collection_schema_add_field(schema, double_field);
|
||||
|
||||
// Vector fields
|
||||
zvec_field_schema_t *vector_fp32_field = zvec_field_schema_create(
|
||||
"vector_fp32", ZVEC_DATA_TYPE_VECTOR_FP32, false, 3);
|
||||
zvec_field_schema_t *large_vector_field = zvec_field_schema_create(
|
||||
"large_vector", ZVEC_DATA_TYPE_VECTOR_FP32, false, 16);
|
||||
|
||||
if (vector_fp32_field) {
|
||||
zvec_field_schema_set_index_params(vector_fp32_field, hnsw_params);
|
||||
error = zvec_collection_schema_add_field(schema, vector_fp32_field);
|
||||
if (handle_error(error, "adding vector FP32 field") == ZVEC_OK) {
|
||||
printf("✓ Vector FP32 field with HNSW index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
if (large_vector_field) {
|
||||
zvec_field_schema_set_index_params(large_vector_field, hnsw_params);
|
||||
error = zvec_collection_schema_add_field(schema, large_vector_field);
|
||||
if (handle_error(error, "adding large vector field") == ZVEC_OK) {
|
||||
printf("✓ Large vector field with HNSW index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Create collection
|
||||
zvec_collection_options_t *options = zvec_collection_options_create();
|
||||
if (!options) {
|
||||
fprintf(stderr, "Failed to create collection options\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_collection_t *collection = NULL;
|
||||
|
||||
error = zvec_collection_create_and_open("./doc_example_collection", schema,
|
||||
options, &collection);
|
||||
zvec_collection_options_destroy(options);
|
||||
if (handle_error(error, "creating collection") != ZVEC_OK) {
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
printf("✓ Collection created successfully\n");
|
||||
|
||||
// 5. Create and insert multiple test documents
|
||||
printf("Creating and inserting test documents...\n");
|
||||
|
||||
#define DOC_COUNT 5
|
||||
// Use dynamic allocation for MSVC compatibility (no VLA support)
|
||||
zvec_doc_t **test_docs =
|
||||
(zvec_doc_t **)malloc(DOC_COUNT * sizeof(zvec_doc_t *));
|
||||
if (!test_docs) {
|
||||
fprintf(stderr, "Failed to allocate test documents\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
for (int i = 0; i < DOC_COUNT; i++) {
|
||||
test_docs[i] = create_full_type_test_doc(i);
|
||||
if (!test_docs[i]) {
|
||||
fprintf(stderr, "Failed to create document %d\n", i);
|
||||
// Cleanup
|
||||
for (int j = 0; j < i; j++) {
|
||||
zvec_doc_destroy(test_docs[j]);
|
||||
}
|
||||
goto cleanup;
|
||||
}
|
||||
printf("✓ Created document %d with PK: %s\n", i,
|
||||
zvec_doc_get_pk_pointer(test_docs[i]));
|
||||
}
|
||||
|
||||
// Print all documents before insertion
|
||||
printf("\nDocuments before insertion:\n");
|
||||
for (int i = 0; i < DOC_COUNT; i++) {
|
||||
print_doc(test_docs[i], i);
|
||||
}
|
||||
|
||||
// Insert documents
|
||||
size_t success_count = 0, error_count = 0;
|
||||
error = zvec_collection_insert(collection, (const zvec_doc_t **)test_docs,
|
||||
DOC_COUNT, &success_count, &error_count);
|
||||
if (handle_error(error, "inserting documents") == ZVEC_OK) {
|
||||
printf("✓ Documents inserted - Success: %zu, Failed: %zu\n", success_count,
|
||||
error_count);
|
||||
}
|
||||
|
||||
// 6. Flush collection
|
||||
error = zvec_collection_flush(collection);
|
||||
if (handle_error(error, "flushing collection") != ZVEC_OK) {
|
||||
printf("Warning: Collection flush failed\n");
|
||||
} else {
|
||||
printf("✓ Collection flushed successfully\n");
|
||||
}
|
||||
|
||||
// Use the first document's vector for querying
|
||||
float query_vector[] = {0.0f, 0.0f, 0.0f};
|
||||
zvec_vector_query_t *query = zvec_vector_query_create();
|
||||
if (!query) {
|
||||
fprintf(stderr, "Failed to create vector query\n");
|
||||
zvec_collection_destroy(collection);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_vector_query_set_field_name(query, "vector_fp32");
|
||||
zvec_vector_query_set_query_vector(query, query_vector, 3 * sizeof(float));
|
||||
zvec_vector_query_set_topk(query, 5);
|
||||
zvec_vector_query_set_filter(query, "");
|
||||
zvec_vector_query_set_include_vector(query, true);
|
||||
zvec_vector_query_set_include_doc_id(query, true);
|
||||
|
||||
zvec_doc_t **query_results = NULL;
|
||||
size_t result_count = 0;
|
||||
|
||||
error = zvec_collection_query(collection, (const zvec_vector_query_t *)query,
|
||||
&query_results, &result_count);
|
||||
if (handle_error(error, "querying documents") != ZVEC_OK) {
|
||||
query_results = NULL;
|
||||
result_count = 0;
|
||||
}
|
||||
|
||||
printf("Query returned %zu results\n", result_count);
|
||||
|
||||
// Print query results
|
||||
printf("\nQuery Results:\n");
|
||||
for (size_t i = 0; i < result_count; i++) {
|
||||
print_doc(query_results[i], i);
|
||||
}
|
||||
|
||||
// Compare query results
|
||||
for (size_t i = 0; i < result_count && i < DOC_COUNT; i++) {
|
||||
const char *result_pk = zvec_doc_get_pk_pointer(query_results[i]);
|
||||
printf("Comparing query result[%zu]: %s\n", i, result_pk);
|
||||
|
||||
// Find matching original document
|
||||
bool found = false;
|
||||
for (int j = 0; j < DOC_COUNT; j++) {
|
||||
const char *original_pk = zvec_doc_get_pk_pointer(test_docs[j]);
|
||||
if (strcmp(result_pk, original_pk) == 0) {
|
||||
if (compare_documents(test_docs[j], query_results[i])) {
|
||||
printf("✓ Query result %s matches original document\n", result_pk);
|
||||
} else {
|
||||
printf("✗ Query result %s does not match original document\n",
|
||||
result_pk);
|
||||
}
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
printf("⚠ Original document not found for: %s\n", result_pk);
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Filter query test
|
||||
printf("\n=== Filter Query Test ===\n");
|
||||
|
||||
// Create filtered query
|
||||
zvec_vector_query_set_filter(query, "string_field = 'string_field_0'");
|
||||
|
||||
zvec_doc_t **filtered_results = NULL;
|
||||
size_t filtered_count = 0;
|
||||
|
||||
error = zvec_collection_query(collection, (const zvec_vector_query_t *)query,
|
||||
&filtered_results, &filtered_count);
|
||||
if (handle_error(error, "filtered querying") == ZVEC_OK) {
|
||||
printf("Filtered query returned %zu results\n", filtered_count);
|
||||
|
||||
// Verify filter results
|
||||
bool filter_correct = true;
|
||||
for (size_t i = 0; i < filtered_count; i++) {
|
||||
// Note: Field value access may require different API
|
||||
// For now, we'll just check that we got results
|
||||
const char *pk = zvec_doc_get_pk_pointer(filtered_results[i]);
|
||||
if (strstr(pk, "doc_") == NULL) {
|
||||
filter_correct = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (filter_correct) {
|
||||
printf("✓ Filter query results are correct\n");
|
||||
} else {
|
||||
printf("✗ Filter query results are incorrect\n");
|
||||
}
|
||||
|
||||
if (filtered_results) {
|
||||
zvec_docs_free(filtered_results, filtered_count);
|
||||
}
|
||||
}
|
||||
|
||||
// 8. Cleanup query results
|
||||
if (query_results) {
|
||||
zvec_docs_free(query_results, result_count);
|
||||
}
|
||||
|
||||
// 9. Cleanup documents
|
||||
for (int i = 0; i < DOC_COUNT; i++) {
|
||||
zvec_doc_destroy(test_docs[i]);
|
||||
}
|
||||
free(test_docs); // Free the array itself
|
||||
|
||||
// 10. Final cleanup
|
||||
cleanup:
|
||||
zvec_collection_destroy(collection);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
|
||||
printf("✓ Document example completed\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "zvec/c_api.h"
|
||||
|
||||
/**
|
||||
* @brief Print error message and return error code
|
||||
*/
|
||||
static zvec_error_code_t handle_error(zvec_error_code_t error,
|
||||
const char *context) {
|
||||
if (error != ZVEC_OK) {
|
||||
char *error_msg = NULL;
|
||||
zvec_get_last_error(&error_msg);
|
||||
fprintf(stderr, "Error in %s: %d - %s\n", context, error,
|
||||
error_msg ? error_msg : "Unknown error");
|
||||
zvec_free(error_msg);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Field schema creation and management example
|
||||
*/
|
||||
int main() {
|
||||
printf("=== ZVec Field Schema Example ===\n\n");
|
||||
|
||||
zvec_error_code_t error;
|
||||
|
||||
// 1. Create collection schema
|
||||
zvec_collection_schema_t *schema =
|
||||
zvec_collection_schema_create("field_example_collection");
|
||||
if (!schema) {
|
||||
fprintf(stderr, "Failed to create collection schema\n");
|
||||
return -1;
|
||||
}
|
||||
printf("✓ Collection schema created successfully\n");
|
||||
|
||||
// 2. Create different types of index parameters
|
||||
zvec_index_params_t *invert_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_INVERT);
|
||||
if (!invert_params) {
|
||||
fprintf(stderr, "Failed to create invert index parameters\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_invert_params(invert_params, true, false);
|
||||
|
||||
zvec_index_params_t *hnsw_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_HNSW);
|
||||
if (!hnsw_params) {
|
||||
fprintf(stderr, "Failed to create HNSW index parameters\n");
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(hnsw_params, ZVEC_METRIC_TYPE_COSINE);
|
||||
zvec_index_params_set_hnsw_params(hnsw_params, 16, 200);
|
||||
|
||||
zvec_index_params_t *flat_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_FLAT);
|
||||
if (!flat_params) {
|
||||
fprintf(stderr, "Failed to create Flat index parameters\n");
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_index_params_destroy(hnsw_params);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(flat_params, ZVEC_METRIC_TYPE_L2);
|
||||
|
||||
if (!invert_params || !hnsw_params || !flat_params) {
|
||||
fprintf(stderr, "Failed to create index parameters\n");
|
||||
zvec_index_params_destroy(invert_params);
|
||||
zvec_index_params_destroy(hnsw_params);
|
||||
zvec_index_params_destroy(flat_params);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// 3. Create scalar fields with different data types
|
||||
printf("Creating scalar fields...\n");
|
||||
|
||||
// String field with inverted index
|
||||
zvec_field_schema_t *name_field =
|
||||
zvec_field_schema_create("name", ZVEC_DATA_TYPE_STRING, false, 0);
|
||||
if (name_field) {
|
||||
zvec_field_schema_set_index_params(name_field, invert_params);
|
||||
error = zvec_collection_schema_add_field(schema, name_field);
|
||||
if (handle_error(error, "adding name field") == ZVEC_OK) {
|
||||
printf("✓ String field 'name' with inverted index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Integer field
|
||||
zvec_field_schema_t *age_field =
|
||||
zvec_field_schema_create("age", ZVEC_DATA_TYPE_INT32, true, 0);
|
||||
if (age_field) {
|
||||
error = zvec_collection_schema_add_field(schema, age_field);
|
||||
if (handle_error(error, "adding age field") == ZVEC_OK) {
|
||||
printf("✓ Integer field 'age' added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Float field
|
||||
zvec_field_schema_t *score_field =
|
||||
zvec_field_schema_create("score", ZVEC_DATA_TYPE_FLOAT, true, 0);
|
||||
if (score_field) {
|
||||
error = zvec_collection_schema_add_field(schema, score_field);
|
||||
if (handle_error(error, "adding score field") == ZVEC_OK) {
|
||||
printf("✓ Float field 'score' added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Boolean field
|
||||
zvec_field_schema_t *active_field =
|
||||
zvec_field_schema_create("active", ZVEC_DATA_TYPE_BOOL, false, 0);
|
||||
if (active_field) {
|
||||
error = zvec_collection_schema_add_field(schema, active_field);
|
||||
if (handle_error(error, "adding active field") == ZVEC_OK) {
|
||||
printf("✓ Boolean field 'active' added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Create vector fields with different dimensions and indexes
|
||||
printf("Creating vector fields...\n");
|
||||
|
||||
// Small dimension vector with HNSW index
|
||||
zvec_field_schema_t *small_vector_field = zvec_field_schema_create(
|
||||
"small_vector", ZVEC_DATA_TYPE_VECTOR_FP32, false, 32);
|
||||
if (small_vector_field) {
|
||||
zvec_field_schema_set_index_params(small_vector_field, hnsw_params);
|
||||
error = zvec_collection_schema_add_field(schema, small_vector_field);
|
||||
if (handle_error(error, "adding small vector field") == ZVEC_OK) {
|
||||
printf(
|
||||
"✓ Small vector field 'small_vector' (32D) with HNSW index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Medium dimension vector with Flat index
|
||||
zvec_field_schema_t *medium_vector_field = zvec_field_schema_create(
|
||||
"medium_vector", ZVEC_DATA_TYPE_VECTOR_FP32, false, 128);
|
||||
if (medium_vector_field) {
|
||||
zvec_field_schema_set_index_params(medium_vector_field, flat_params);
|
||||
error = zvec_collection_schema_add_field(schema, medium_vector_field);
|
||||
if (handle_error(error, "adding medium vector field") == ZVEC_OK) {
|
||||
printf(
|
||||
"✓ Medium vector field 'medium_vector' (128D) with Flat index "
|
||||
"added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Large dimension vector with HNSW index
|
||||
zvec_field_schema_t *large_vector_field = zvec_field_schema_create(
|
||||
"large_vector", ZVEC_DATA_TYPE_VECTOR_FP32, false, 512);
|
||||
if (large_vector_field) {
|
||||
zvec_field_schema_set_index_params(large_vector_field, hnsw_params);
|
||||
error = zvec_collection_schema_add_field(schema, large_vector_field);
|
||||
if (handle_error(error, "adding large vector field") == ZVEC_OK) {
|
||||
printf(
|
||||
"✓ Large vector field 'large_vector' (512D) with HNSW index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Create collection with the schema
|
||||
zvec_collection_options_t *options = zvec_collection_options_create();
|
||||
if (!options) {
|
||||
fprintf(stderr, "Failed to create collection options\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_collection_t *collection = NULL;
|
||||
|
||||
error = zvec_collection_create_and_open("./field_example_collection", schema,
|
||||
options, &collection);
|
||||
zvec_collection_options_destroy(options);
|
||||
if (handle_error(error, "creating collection") != ZVEC_OK) {
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
printf("✓ Collection created successfully\n");
|
||||
|
||||
// 6. Create test documents with various field types
|
||||
printf("Creating test documents...\n");
|
||||
|
||||
zvec_doc_t *doc1 = zvec_doc_create();
|
||||
zvec_doc_t *doc2 = zvec_doc_create();
|
||||
|
||||
if (!doc1 || !doc2) {
|
||||
fprintf(stderr, "Failed to create documents\n");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// Document 1
|
||||
zvec_doc_set_pk(doc1, "user1");
|
||||
zvec_doc_add_field_by_value(doc1, "name", ZVEC_DATA_TYPE_STRING,
|
||||
"Alice Johnson", strlen("Alice Johnson"));
|
||||
int32_t age1 = 28;
|
||||
zvec_doc_add_field_by_value(doc1, "age", ZVEC_DATA_TYPE_INT32, &age1,
|
||||
sizeof(age1));
|
||||
float score1 = 87.5f;
|
||||
zvec_doc_add_field_by_value(doc1, "score", ZVEC_DATA_TYPE_FLOAT, &score1,
|
||||
sizeof(score1));
|
||||
bool active1 = true;
|
||||
zvec_doc_add_field_by_value(doc1, "active", ZVEC_DATA_TYPE_BOOL, &active1,
|
||||
sizeof(active1));
|
||||
|
||||
// Add vector data
|
||||
float small_vec1[32];
|
||||
float medium_vec1[128];
|
||||
float large_vec1[512];
|
||||
|
||||
for (int i = 0; i < 32; i++) small_vec1[i] = (float)i / 32.0f;
|
||||
for (int i = 0; i < 128; i++) medium_vec1[i] = (float)i / 128.0f;
|
||||
for (int i = 0; i < 512; i++) large_vec1[i] = (float)i / 512.0f;
|
||||
|
||||
zvec_doc_add_field_by_value(doc1, "small_vector", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
small_vec1, 32 * sizeof(float));
|
||||
zvec_doc_add_field_by_value(doc1, "medium_vector", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
medium_vec1, 128 * sizeof(float));
|
||||
zvec_doc_add_field_by_value(doc1, "large_vector", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
large_vec1, 512 * sizeof(float));
|
||||
|
||||
// Document 2
|
||||
zvec_doc_set_pk(doc2, "user2");
|
||||
zvec_doc_add_field_by_value(doc2, "name", ZVEC_DATA_TYPE_STRING, "Bob Smith",
|
||||
strlen("Bob Smith"));
|
||||
int32_t age2 = 35;
|
||||
zvec_doc_add_field_by_value(doc2, "age", ZVEC_DATA_TYPE_INT32, &age2,
|
||||
sizeof(age2));
|
||||
float score2 = 92.0f;
|
||||
zvec_doc_add_field_by_value(doc2, "score", ZVEC_DATA_TYPE_FLOAT, &score2,
|
||||
sizeof(score2));
|
||||
bool active2 = false;
|
||||
zvec_doc_add_field_by_value(doc2, "active", ZVEC_DATA_TYPE_BOOL, &active2,
|
||||
sizeof(active2));
|
||||
|
||||
// Add vector data
|
||||
float small_vec2[32];
|
||||
float medium_vec2[128];
|
||||
float large_vec2[512];
|
||||
|
||||
for (int i = 0; i < 32; i++) small_vec2[i] = (float)(32 - i) / 32.0f;
|
||||
for (int i = 0; i < 128; i++) medium_vec2[i] = (float)(128 - i) / 128.0f;
|
||||
for (int i = 0; i < 512; i++) large_vec2[i] = (float)(512 - i) / 512.0f;
|
||||
|
||||
zvec_doc_add_field_by_value(doc2, "small_vector", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
small_vec2, 32 * sizeof(float));
|
||||
zvec_doc_add_field_by_value(doc2, "medium_vector", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
medium_vec2, 128 * sizeof(float));
|
||||
zvec_doc_add_field_by_value(doc2, "large_vector", ZVEC_DATA_TYPE_VECTOR_FP32,
|
||||
large_vec2, 512 * sizeof(float));
|
||||
|
||||
// 7. Insert documents
|
||||
zvec_doc_t *docs[] = {doc1, doc2};
|
||||
size_t success_count = 0, error_count = 0;
|
||||
error = zvec_collection_insert(collection, (const zvec_doc_t **)docs, 2,
|
||||
&success_count, &error_count);
|
||||
if (handle_error(error, "inserting documents") == ZVEC_OK) {
|
||||
printf("✓ Documents inserted - Success: %zu, Failed: %zu\n", success_count,
|
||||
error_count);
|
||||
}
|
||||
|
||||
// 8. Flush and test queries
|
||||
zvec_collection_flush(collection);
|
||||
printf("✓ Collection flushed\n");
|
||||
|
||||
// Test vector query on medium vector field
|
||||
zvec_vector_query_t *query = zvec_vector_query_create();
|
||||
if (!query) {
|
||||
fprintf(stderr, "Failed to create vector query\n");
|
||||
goto cleanup;
|
||||
}
|
||||
zvec_vector_query_set_field_name(query, "medium_vector");
|
||||
zvec_vector_query_set_query_vector(query, medium_vec1, 128 * sizeof(float));
|
||||
zvec_vector_query_set_topk(query, 2);
|
||||
zvec_vector_query_set_filter(query, "");
|
||||
zvec_vector_query_set_include_vector(query, false);
|
||||
zvec_vector_query_set_include_doc_id(query, true);
|
||||
|
||||
zvec_doc_t **results = NULL;
|
||||
size_t result_count = 0;
|
||||
error = zvec_collection_query(collection, (const zvec_vector_query_t *)query,
|
||||
&results, &result_count);
|
||||
if (error == ZVEC_OK) {
|
||||
printf("✓ Vector query successful - Found %zu results\n", result_count);
|
||||
zvec_docs_free(results, result_count);
|
||||
}
|
||||
zvec_vector_query_destroy(query);
|
||||
|
||||
// 9. Cleanup
|
||||
cleanup:
|
||||
if (doc1) zvec_doc_destroy(doc1);
|
||||
if (doc2) zvec_doc_destroy(doc2);
|
||||
zvec_collection_destroy(collection);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
|
||||
printf("✓ Field schema example completed\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "zvec/c_api.h"
|
||||
|
||||
/**
|
||||
* @brief Print error message and return error code
|
||||
*/
|
||||
static zvec_error_code_t handle_error(zvec_error_code_t error,
|
||||
const char *context) {
|
||||
if (error != ZVEC_OK) {
|
||||
char *error_msg = NULL;
|
||||
zvec_get_last_error(&error_msg);
|
||||
fprintf(stderr, "Error in %s: %d - %s\n", context, error,
|
||||
error_msg ? error_msg : "Unknown error");
|
||||
zvec_free(error_msg);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Index creation and management example
|
||||
*/
|
||||
int main() {
|
||||
printf("=== ZVec Index Example ===\n\n");
|
||||
|
||||
zvec_error_code_t error;
|
||||
|
||||
// 1. Create collection schema
|
||||
zvec_collection_schema_t *schema =
|
||||
zvec_collection_schema_create("index_example_collection");
|
||||
if (!schema) {
|
||||
fprintf(stderr, "Failed to create collection schema\n");
|
||||
return -1;
|
||||
}
|
||||
printf("✓ Collection schema created successfully\n");
|
||||
|
||||
// 2. Create different index parameter configurations
|
||||
printf("Creating index parameters...\n");
|
||||
|
||||
// Inverted index parameters
|
||||
zvec_index_params_t *invert_params_standard =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_INVERT);
|
||||
if (!invert_params_standard) {
|
||||
fprintf(stderr, "Failed to create invert index parameters (standard)\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_invert_params(invert_params_standard, true, false);
|
||||
|
||||
zvec_index_params_t *invert_params_extended =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_INVERT);
|
||||
if (!invert_params_extended) {
|
||||
fprintf(stderr, "Failed to create invert index parameters (extended)\n");
|
||||
zvec_index_params_destroy(invert_params_standard);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_invert_params(invert_params_extended, true, true);
|
||||
|
||||
// HNSW index parameters with different configurations
|
||||
zvec_index_params_t *hnsw_params_fast =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_HNSW);
|
||||
if (!hnsw_params_fast) {
|
||||
fprintf(stderr, "Failed to create HNSW index parameters (fast)\n");
|
||||
zvec_index_params_destroy(invert_params_standard);
|
||||
zvec_index_params_destroy(invert_params_extended);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(hnsw_params_fast, ZVEC_METRIC_TYPE_L2);
|
||||
zvec_index_params_set_hnsw_params(hnsw_params_fast, 16, 100);
|
||||
|
||||
// Demonstrate INT8 quantization with random rotation preprocessing
|
||||
// (enable_rotate rotates vectors before INT8 quantization to reduce error)
|
||||
zvec_index_params_set_quantize_type(hnsw_params_fast,
|
||||
ZVEC_QUANTIZE_TYPE_INT8);
|
||||
zvec_index_params_set_quantizer_enable_rotate(hnsw_params_fast, true);
|
||||
|
||||
zvec_index_params_t *hnsw_params_balanced =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_HNSW);
|
||||
if (!hnsw_params_balanced) {
|
||||
fprintf(stderr, "Failed to create HNSW index parameters (balanced)\n");
|
||||
zvec_index_params_destroy(invert_params_standard);
|
||||
zvec_index_params_destroy(invert_params_extended);
|
||||
zvec_index_params_destroy(hnsw_params_fast);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(hnsw_params_balanced,
|
||||
ZVEC_METRIC_TYPE_COSINE);
|
||||
zvec_index_params_set_hnsw_params(hnsw_params_balanced, 32, 200);
|
||||
|
||||
zvec_index_params_t *hnsw_params_accurate =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_HNSW);
|
||||
if (!hnsw_params_accurate) {
|
||||
fprintf(stderr, "Failed to create HNSW index parameters (accurate)\n");
|
||||
zvec_index_params_destroy(invert_params_standard);
|
||||
zvec_index_params_destroy(invert_params_extended);
|
||||
zvec_index_params_destroy(hnsw_params_fast);
|
||||
zvec_index_params_destroy(hnsw_params_balanced);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(hnsw_params_accurate, ZVEC_METRIC_TYPE_IP);
|
||||
zvec_index_params_set_hnsw_params(hnsw_params_accurate, 64, 400);
|
||||
|
||||
// Flat index parameters
|
||||
zvec_index_params_t *flat_params_l2 =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_FLAT);
|
||||
if (!flat_params_l2) {
|
||||
fprintf(stderr, "Failed to create Flat index parameters (L2)\n");
|
||||
zvec_index_params_destroy(invert_params_standard);
|
||||
zvec_index_params_destroy(invert_params_extended);
|
||||
zvec_index_params_destroy(hnsw_params_fast);
|
||||
zvec_index_params_destroy(hnsw_params_balanced);
|
||||
zvec_index_params_destroy(hnsw_params_accurate);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(flat_params_l2, ZVEC_METRIC_TYPE_L2);
|
||||
|
||||
zvec_index_params_t *flat_params_cosine =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_FLAT);
|
||||
if (!flat_params_cosine) {
|
||||
fprintf(stderr, "Failed to create Flat index parameters (cosine)\n");
|
||||
zvec_index_params_destroy(invert_params_standard);
|
||||
zvec_index_params_destroy(invert_params_extended);
|
||||
zvec_index_params_destroy(hnsw_params_fast);
|
||||
zvec_index_params_destroy(hnsw_params_balanced);
|
||||
zvec_index_params_destroy(hnsw_params_accurate);
|
||||
zvec_index_params_destroy(flat_params_l2);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(flat_params_cosine,
|
||||
ZVEC_METRIC_TYPE_COSINE);
|
||||
|
||||
// 3. Create fields with different index types
|
||||
printf("Creating fields with various index types...\n");
|
||||
|
||||
// Fields with inverted indexes
|
||||
zvec_field_schema_t *id_field =
|
||||
zvec_field_schema_create("id", ZVEC_DATA_TYPE_STRING, false, 0);
|
||||
if (id_field) {
|
||||
zvec_field_schema_set_index_params(id_field, invert_params_standard);
|
||||
error = zvec_collection_schema_add_field(schema, id_field);
|
||||
if (handle_error(error, "adding ID field") == ZVEC_OK) {
|
||||
printf("✓ ID field with standard inverted index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
zvec_field_schema_t *category_field =
|
||||
zvec_field_schema_create("category", ZVEC_DATA_TYPE_STRING, true, 0);
|
||||
if (category_field) {
|
||||
zvec_field_schema_set_index_params(category_field, invert_params_extended);
|
||||
error = zvec_collection_schema_add_field(schema, category_field);
|
||||
if (handle_error(error, "adding category field") == ZVEC_OK) {
|
||||
printf("✓ Category field with extended inverted index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Vector fields with HNSW indexes (different configurations)
|
||||
zvec_field_schema_t *fast_search_field = zvec_field_schema_create(
|
||||
"fast_vector", ZVEC_DATA_TYPE_VECTOR_FP32, false, 64);
|
||||
if (fast_search_field) {
|
||||
zvec_field_schema_set_index_params(fast_search_field, hnsw_params_fast);
|
||||
error = zvec_collection_schema_add_field(schema, fast_search_field);
|
||||
if (handle_error(error, "adding fast search field") == ZVEC_OK) {
|
||||
printf("✓ Fast search vector field (64D) with HNSW index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
zvec_field_schema_t *balanced_field = zvec_field_schema_create(
|
||||
"balanced_vector", ZVEC_DATA_TYPE_VECTOR_FP32, false, 128);
|
||||
if (balanced_field) {
|
||||
zvec_field_schema_set_index_params(balanced_field, hnsw_params_balanced);
|
||||
error = zvec_collection_schema_add_field(schema, balanced_field);
|
||||
if (handle_error(error, "adding balanced field") == ZVEC_OK) {
|
||||
printf("✓ Balanced vector field (128D) with HNSW index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
zvec_field_schema_t *accurate_field = zvec_field_schema_create(
|
||||
"accurate_vector", ZVEC_DATA_TYPE_VECTOR_FP32, false, 256);
|
||||
if (accurate_field) {
|
||||
zvec_field_schema_set_index_params(accurate_field, hnsw_params_accurate);
|
||||
error = zvec_collection_schema_add_field(schema, accurate_field);
|
||||
if (handle_error(error, "adding accurate field") == ZVEC_OK) {
|
||||
printf("✓ Accurate vector field (256D) with HNSW index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Vector field with Flat index
|
||||
zvec_field_schema_t *exact_field = zvec_field_schema_create(
|
||||
"exact_vector", ZVEC_DATA_TYPE_VECTOR_FP32, false, 32);
|
||||
if (exact_field) {
|
||||
zvec_field_schema_set_index_params(exact_field, flat_params_l2);
|
||||
error = zvec_collection_schema_add_field(schema, exact_field);
|
||||
if (handle_error(error, "adding exact field") == ZVEC_OK) {
|
||||
printf("✓ Exact search vector field (32D) with Flat index added\n");
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Create collection
|
||||
zvec_collection_options_t *options = zvec_collection_options_create();
|
||||
if (!options) {
|
||||
fprintf(stderr, "Failed to create collection options\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_collection_t *collection = NULL;
|
||||
|
||||
error = zvec_collection_create_and_open("./index_example_collection", schema,
|
||||
options, &collection);
|
||||
zvec_collection_options_destroy(options);
|
||||
if (handle_error(error, "creating collection") != ZVEC_OK) {
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
printf("✓ Collection created successfully\n");
|
||||
|
||||
// 5. Create test data
|
||||
printf("Creating test documents...\n");
|
||||
|
||||
zvec_doc_t *docs[3];
|
||||
for (int i = 0; i < 3; i++) {
|
||||
docs[i] = zvec_doc_create();
|
||||
if (!docs[i]) {
|
||||
fprintf(stderr, "Failed to create document %d\n", i);
|
||||
// Cleanup
|
||||
for (int j = 0; j < i; j++) {
|
||||
zvec_doc_destroy(docs[j]);
|
||||
}
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare vector data
|
||||
float fast_vec[3][64];
|
||||
float balanced_vec[3][128];
|
||||
float accurate_vec[3][256];
|
||||
float exact_vec[3][32];
|
||||
|
||||
// Generate different vector patterns for testing
|
||||
for (int doc_idx = 0; doc_idx < 3; doc_idx++) {
|
||||
for (int i = 0; i < 64; i++) {
|
||||
fast_vec[doc_idx][i] = (float)(doc_idx * 64 + i) / (64.0f * 3.0f);
|
||||
}
|
||||
for (int i = 0; i < 128; i++) {
|
||||
balanced_vec[doc_idx][i] = (float)(doc_idx * 128 + i) / (128.0f * 3.0f);
|
||||
}
|
||||
for (int i = 0; i < 256; i++) {
|
||||
accurate_vec[doc_idx][i] = (float)(doc_idx * 256 + i) / (256.0f * 3.0f);
|
||||
}
|
||||
for (int i = 0; i < 32; i++) {
|
||||
exact_vec[doc_idx][i] = (float)(doc_idx * 32 + i) / (32.0f * 3.0f);
|
||||
}
|
||||
}
|
||||
|
||||
// Populate documents
|
||||
for (int i = 0; i < 3; i++) {
|
||||
char pk[16];
|
||||
snprintf(pk, sizeof(pk), "doc%d", i + 1);
|
||||
zvec_doc_set_pk(docs[i], pk);
|
||||
|
||||
char id_val[16];
|
||||
snprintf(id_val, sizeof(id_val), "ID_%d", i + 1);
|
||||
zvec_doc_add_field_by_value(docs[i], "id", ZVEC_DATA_TYPE_STRING, id_val,
|
||||
strlen(id_val));
|
||||
|
||||
char category_val[16];
|
||||
snprintf(category_val, sizeof(category_val), "cat_%d", (i % 2) + 1);
|
||||
zvec_doc_add_field_by_value(docs[i], "category", ZVEC_DATA_TYPE_STRING,
|
||||
category_val, strlen(category_val));
|
||||
|
||||
zvec_doc_add_field_by_value(docs[i], "fast_vector",
|
||||
ZVEC_DATA_TYPE_VECTOR_FP32, fast_vec[i],
|
||||
64 * sizeof(float));
|
||||
zvec_doc_add_field_by_value(docs[i], "balanced_vector",
|
||||
ZVEC_DATA_TYPE_VECTOR_FP32, balanced_vec[i],
|
||||
128 * sizeof(float));
|
||||
zvec_doc_add_field_by_value(docs[i], "accurate_vector",
|
||||
ZVEC_DATA_TYPE_VECTOR_FP32, accurate_vec[i],
|
||||
256 * sizeof(float));
|
||||
zvec_doc_add_field_by_value(docs[i], "exact_vector",
|
||||
ZVEC_DATA_TYPE_VECTOR_FP32, exact_vec[i],
|
||||
32 * sizeof(float));
|
||||
}
|
||||
|
||||
// 6. Insert documents
|
||||
size_t success_count = 0, error_count = 0;
|
||||
error = zvec_collection_insert(collection, (const zvec_doc_t **)docs, 3,
|
||||
&success_count, &error_count);
|
||||
if (handle_error(error, "inserting documents") == ZVEC_OK) {
|
||||
printf("✓ Documents inserted - Success: %zu, Failed: %zu\n", success_count,
|
||||
error_count);
|
||||
}
|
||||
|
||||
// Cleanup documents
|
||||
for (int i = 0; i < 3; i++) {
|
||||
zvec_doc_destroy(docs[i]);
|
||||
}
|
||||
|
||||
// 7. Flush collection to build indexes
|
||||
error = zvec_collection_flush(collection);
|
||||
if (handle_error(error, "flushing collection") == ZVEC_OK) {
|
||||
printf("✓ Collection flushed - indexes built\n");
|
||||
}
|
||||
|
||||
// 8. Test different query types
|
||||
printf("Testing various index queries...\n");
|
||||
|
||||
// Test HNSW query (balanced)
|
||||
zvec_vector_query_t *hnsw_query = zvec_vector_query_create();
|
||||
if (!hnsw_query) {
|
||||
fprintf(stderr, "Failed to create HNSW query\n");
|
||||
goto cleanup;
|
||||
}
|
||||
zvec_vector_query_set_field_name(hnsw_query, "balanced_vector");
|
||||
zvec_vector_query_set_query_vector(hnsw_query, balanced_vec[0],
|
||||
128 * sizeof(float));
|
||||
zvec_vector_query_set_topk(hnsw_query, 2);
|
||||
zvec_vector_query_set_filter(hnsw_query, "");
|
||||
zvec_vector_query_set_include_vector(hnsw_query, false);
|
||||
zvec_vector_query_set_include_doc_id(hnsw_query, true);
|
||||
|
||||
zvec_doc_t **hnsw_results = NULL;
|
||||
size_t hnsw_result_count = 0;
|
||||
error =
|
||||
zvec_collection_query(collection, (const zvec_vector_query_t *)hnsw_query,
|
||||
&hnsw_results, &hnsw_result_count);
|
||||
if (error == ZVEC_OK) {
|
||||
printf("✓ HNSW query successful - Found %zu results\n", hnsw_result_count);
|
||||
zvec_docs_free(hnsw_results, hnsw_result_count);
|
||||
}
|
||||
zvec_vector_query_destroy(hnsw_query);
|
||||
|
||||
// Test Flat query (exact)
|
||||
zvec_vector_query_t *flat_query = zvec_vector_query_create();
|
||||
if (!flat_query) {
|
||||
fprintf(stderr, "Failed to create Flat query\n");
|
||||
goto cleanup;
|
||||
}
|
||||
zvec_vector_query_set_field_name(flat_query, "exact_vector");
|
||||
zvec_vector_query_set_query_vector(flat_query, exact_vec[0],
|
||||
32 * sizeof(float));
|
||||
zvec_vector_query_set_topk(flat_query, 2);
|
||||
zvec_vector_query_set_filter(flat_query, "");
|
||||
zvec_vector_query_set_include_vector(flat_query, false);
|
||||
zvec_vector_query_set_include_doc_id(flat_query, true);
|
||||
|
||||
zvec_doc_t **flat_results = NULL;
|
||||
size_t flat_result_count = 0;
|
||||
error =
|
||||
zvec_collection_query(collection, (const zvec_vector_query_t *)flat_query,
|
||||
&flat_results, &flat_result_count);
|
||||
if (error == ZVEC_OK) {
|
||||
printf("✓ Flat (exact) query successful - Found %zu results\n",
|
||||
flat_result_count);
|
||||
zvec_docs_free(flat_results, flat_result_count);
|
||||
}
|
||||
zvec_vector_query_destroy(flat_query);
|
||||
|
||||
// 9. Performance comparison information
|
||||
printf("\nIndex Performance Characteristics:\n");
|
||||
printf("- Inverted Index: Fast text search, supports filtering\n");
|
||||
printf(
|
||||
"- HNSW Index: Approximate nearest neighbor search, good balance of "
|
||||
"speed/accuracy\n");
|
||||
printf("- Flat Index: Exact search, slower but 100%% accurate\n");
|
||||
printf(
|
||||
"- Trade-off: Speed vs Accuracy - choose based on your requirements\n");
|
||||
|
||||
// 10. Cleanup
|
||||
cleanup:
|
||||
zvec_collection_destroy(collection);
|
||||
zvec_collection_schema_destroy(schema);
|
||||
|
||||
// Cleanup index parameters
|
||||
|
||||
printf("✓ Index example completed\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include "zvec/c_api.h"
|
||||
|
||||
/**
|
||||
* @brief Print error message and return error code
|
||||
*/
|
||||
static zvec_error_code_t handle_error(zvec_error_code_t error,
|
||||
const char *context) {
|
||||
if (error != ZVEC_OK) {
|
||||
char *error_msg = NULL;
|
||||
zvec_get_last_error(&error_msg);
|
||||
fprintf(stderr, "Error in %s: %d - %s\n", context, error,
|
||||
error_msg ? error_msg : "Unknown error");
|
||||
zvec_free(error_msg);
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Create test vector data
|
||||
*/
|
||||
static float *create_test_vector(size_t dimension) {
|
||||
float *vector = malloc(dimension * sizeof(float));
|
||||
if (!vector) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < dimension; i++) {
|
||||
vector[i] = (float)rand() / RAND_MAX;
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Optimized C API usage example with performance considerations
|
||||
*/
|
||||
int main() {
|
||||
printf("=== ZVec Optimized C API Example ===\n\n");
|
||||
|
||||
// Get version information
|
||||
const char *version = zvec_get_version();
|
||||
printf("ZVec Version: %s\n\n", version ? version : "Unknown");
|
||||
|
||||
zvec_error_code_t error;
|
||||
|
||||
// 1. Create optimized collection schema
|
||||
zvec_collection_schema_t *schema =
|
||||
zvec_collection_schema_create("optimized_example_collection");
|
||||
if (!schema) {
|
||||
fprintf(stderr, "Failed to create collection schema\n");
|
||||
return -1;
|
||||
}
|
||||
printf("✓ Collection schema created\n");
|
||||
|
||||
// 2. Create optimized index parameters
|
||||
zvec_index_params_t *hnsw_params =
|
||||
zvec_index_params_create(ZVEC_INDEX_TYPE_HNSW);
|
||||
if (!hnsw_params) {
|
||||
fprintf(stderr, "Failed to create HNSW index parameters\n");
|
||||
zvec_collection_schema_destroy(schema);
|
||||
return -1;
|
||||
}
|
||||
zvec_index_params_set_metric_type(hnsw_params, ZVEC_METRIC_TYPE_L2);
|
||||
zvec_index_params_set_hnsw_params(hnsw_params, 32, 200);
|
||||
|
||||
// 3. Create fields with optimized configuration
|
||||
zvec_field_schema_t *id_field =
|
||||
zvec_field_schema_create("id", ZVEC_DATA_TYPE_STRING, false, 0);
|
||||
zvec_field_schema_t *text_field =
|
||||
zvec_field_schema_create("text", ZVEC_DATA_TYPE_STRING, true, 0);
|
||||
zvec_field_schema_t *embedding_field = zvec_field_schema_create(
|
||||
"embedding", ZVEC_DATA_TYPE_VECTOR_FP32, false, 128);
|
||||
|
||||
if (!id_field || !text_field || !embedding_field) {
|
||||
fprintf(stderr, "Failed to create field schemas\n");
|
||||
goto cleanup_params;
|
||||
}
|
||||
|
||||
// Set indexes
|
||||
zvec_field_schema_set_index_params(embedding_field, hnsw_params);
|
||||
|
||||
// Add fields to schema
|
||||
error = zvec_collection_schema_add_field(schema, id_field);
|
||||
if (handle_error(error, "adding ID field") != ZVEC_OK) goto cleanup_fields;
|
||||
|
||||
error = zvec_collection_schema_add_field(schema, text_field);
|
||||
if (handle_error(error, "adding text field") != ZVEC_OK) goto cleanup_fields;
|
||||
|
||||
error = zvec_collection_schema_add_field(schema, embedding_field);
|
||||
if (handle_error(error, "adding embedding field") != ZVEC_OK)
|
||||
goto cleanup_fields;
|
||||
|
||||
printf("✓ Fields configured with indexes\n");
|
||||
|
||||
// 4. Create collection with optimized options
|
||||
zvec_collection_options_t *options = zvec_collection_options_create();
|
||||
if (!options) {
|
||||
fprintf(stderr, "Failed to create collection options\n");
|
||||
goto cleanup_fields;
|
||||
}
|
||||
zvec_collection_options_set_enable_mmap(
|
||||
options, true); // Enable memory mapping for better performance
|
||||
|
||||
zvec_collection_t *collection = NULL;
|
||||
error = zvec_collection_create_and_open("./optimized_example_collection",
|
||||
schema, options, &collection);
|
||||
zvec_collection_options_destroy(options);
|
||||
if (handle_error(error, "creating collection") != ZVEC_OK) {
|
||||
goto cleanup_fields;
|
||||
}
|
||||
printf("✓ Collection created with optimized settings\n");
|
||||
|
||||
// 5. Bulk insert test data
|
||||
const size_t DOC_COUNT = 1000;
|
||||
const size_t BATCH_SIZE = 100;
|
||||
|
||||
printf("Inserting %zu documents in batches of %zu...\n", DOC_COUNT,
|
||||
BATCH_SIZE);
|
||||
|
||||
clock_t start_time = clock();
|
||||
|
||||
for (size_t batch_start = 0; batch_start < DOC_COUNT;
|
||||
batch_start += BATCH_SIZE) {
|
||||
size_t current_batch_size = (batch_start + BATCH_SIZE > DOC_COUNT)
|
||||
? DOC_COUNT - batch_start
|
||||
: BATCH_SIZE;
|
||||
|
||||
zvec_doc_t **batch_docs = malloc(current_batch_size * sizeof(zvec_doc_t *));
|
||||
if (!batch_docs) {
|
||||
fprintf(stderr, "Failed to allocate batch documents\n");
|
||||
break;
|
||||
}
|
||||
|
||||
// Create batch documents
|
||||
for (size_t i = 0; i < current_batch_size; i++) {
|
||||
batch_docs[i] = zvec_doc_create();
|
||||
if (!batch_docs[i]) {
|
||||
fprintf(stderr, "Failed to create document\n");
|
||||
// Cleanup previous documents in batch
|
||||
for (size_t j = 0; j < i; j++) {
|
||||
zvec_doc_destroy(batch_docs[j]);
|
||||
}
|
||||
free(batch_docs);
|
||||
goto cleanup_collection;
|
||||
}
|
||||
|
||||
size_t doc_id = batch_start + i;
|
||||
char pk[32];
|
||||
snprintf(pk, sizeof(pk), "doc_%zu", doc_id);
|
||||
zvec_doc_set_pk(batch_docs[i], pk);
|
||||
|
||||
// Add ID field
|
||||
char id_str[32];
|
||||
snprintf(id_str, sizeof(id_str), "ID_%zu", doc_id);
|
||||
zvec_doc_add_field_by_value(batch_docs[i], "id", ZVEC_DATA_TYPE_STRING,
|
||||
id_str, strlen(id_str));
|
||||
|
||||
// Add text field
|
||||
char text_str[64];
|
||||
snprintf(text_str, sizeof(text_str),
|
||||
"Document number %zu with sample text", doc_id);
|
||||
zvec_doc_add_field_by_value(batch_docs[i], "text", ZVEC_DATA_TYPE_STRING,
|
||||
text_str, strlen(text_str));
|
||||
|
||||
// Add vector field
|
||||
float *vector = create_test_vector(128);
|
||||
if (vector) {
|
||||
zvec_doc_add_field_by_value(batch_docs[i], "embedding",
|
||||
ZVEC_DATA_TYPE_VECTOR_FP32, vector,
|
||||
128 * sizeof(float));
|
||||
free(vector);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert batch
|
||||
size_t success_count, error_count;
|
||||
error = zvec_collection_insert(collection, (const zvec_doc_t **)batch_docs,
|
||||
current_batch_size, &success_count,
|
||||
&error_count);
|
||||
if (handle_error(error, "inserting batch") != ZVEC_OK) {
|
||||
// Cleanup batch documents
|
||||
for (size_t i = 0; i < current_batch_size; i++) {
|
||||
zvec_doc_destroy(batch_docs[i]);
|
||||
}
|
||||
free(batch_docs);
|
||||
goto cleanup_collection;
|
||||
}
|
||||
|
||||
printf(" Batch %zu-%zu: %zu successful, %zu failed\n", batch_start,
|
||||
batch_start + current_batch_size - 1, success_count, error_count);
|
||||
|
||||
// Cleanup batch documents
|
||||
for (size_t i = 0; i < current_batch_size; i++) {
|
||||
zvec_doc_destroy(batch_docs[i]);
|
||||
}
|
||||
free(batch_docs);
|
||||
}
|
||||
|
||||
clock_t insert_end_time = clock();
|
||||
double insert_time =
|
||||
((double)(insert_end_time - start_time)) / CLOCKS_PER_SEC;
|
||||
printf("✓ Bulk insertion completed in %.3f seconds (%.0f docs/sec)\n",
|
||||
insert_time, DOC_COUNT / insert_time);
|
||||
|
||||
// 6. Flush and optimize collection
|
||||
printf("Flushing and optimizing collection...\n");
|
||||
zvec_collection_flush(collection);
|
||||
zvec_collection_optimize(collection);
|
||||
printf("✓ Collection optimized\n");
|
||||
|
||||
// 7. Performance query test
|
||||
printf("Testing query performance...\n");
|
||||
|
||||
float *query_vector = create_test_vector(128);
|
||||
if (!query_vector) {
|
||||
fprintf(stderr, "Failed to create query vector\n");
|
||||
goto cleanup_collection;
|
||||
}
|
||||
|
||||
zvec_vector_query_t *query = zvec_vector_query_create();
|
||||
if (!query) {
|
||||
fprintf(stderr, "Failed to create vector query\n");
|
||||
free(query_vector);
|
||||
goto cleanup_collection;
|
||||
}
|
||||
zvec_vector_query_set_field_name(query, "embedding");
|
||||
zvec_vector_query_set_query_vector(query, query_vector, 128 * sizeof(float));
|
||||
zvec_vector_query_set_topk(query, 10);
|
||||
zvec_vector_query_set_filter(query, "");
|
||||
zvec_vector_query_set_include_vector(query, false);
|
||||
zvec_vector_query_set_include_doc_id(query, true);
|
||||
|
||||
const int QUERY_COUNT = 100;
|
||||
start_time = clock();
|
||||
|
||||
for (int q = 0; q < QUERY_COUNT; q++) {
|
||||
zvec_doc_t **results = NULL;
|
||||
size_t result_count = 0;
|
||||
|
||||
error =
|
||||
zvec_collection_query(collection, (const zvec_vector_query_t *)query,
|
||||
&results, &result_count);
|
||||
if (error != ZVEC_OK) {
|
||||
char *error_msg = NULL;
|
||||
zvec_get_last_error(&error_msg);
|
||||
printf("Query %d failed: %s\n", q,
|
||||
error_msg ? error_msg : "Unknown error");
|
||||
zvec_free(error_msg);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (results) {
|
||||
zvec_docs_free(results, result_count);
|
||||
}
|
||||
}
|
||||
|
||||
clock_t query_end_time = clock();
|
||||
double query_time = ((double)(query_end_time - start_time)) / CLOCKS_PER_SEC;
|
||||
double avg_query_time = (query_time * 1000) / QUERY_COUNT;
|
||||
|
||||
printf("✓ Performance test completed\n");
|
||||
printf(" Average query time: %.2f ms\n", avg_query_time);
|
||||
printf(" Queries per second: %.0f\n", 1000.0 / avg_query_time);
|
||||
|
||||
free(query_vector);
|
||||
zvec_vector_query_destroy(query);
|
||||
|
||||
// 8. Memory usage information
|
||||
zvec_collection_stats_t *stats = NULL;
|
||||
error = zvec_collection_get_stats(collection, &stats);
|
||||
if (error == ZVEC_OK && stats) {
|
||||
printf("Collection Statistics:\n");
|
||||
printf(" Document count: %llu\n",
|
||||
(unsigned long long)zvec_collection_stats_get_doc_count(stats));
|
||||
zvec_collection_stats_destroy(stats);
|
||||
}
|
||||
|
||||
// 9. Cleanup
|
||||
cleanup_collection:
|
||||
zvec_collection_destroy(collection);
|
||||
|
||||
cleanup_fields:
|
||||
// Field schemas are managed by the collection schema, no need to destroy
|
||||
// individually
|
||||
|
||||
cleanup_params:
|
||||
zvec_collection_schema_destroy(schema);
|
||||
|
||||
printf("✓ Optimized example completed\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user