chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,30 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from .collection import Collection
|
||||
from .doc import Doc
|
||||
from .param.query import Fts, Query, VectorQuery
|
||||
from .schema.collection_schema import CollectionSchema
|
||||
from .schema.field_schema import FieldSchema
|
||||
|
||||
__all__ = [
|
||||
"Collection",
|
||||
"CollectionSchema",
|
||||
"Doc",
|
||||
"FieldSchema",
|
||||
"Fts",
|
||||
"Query",
|
||||
"VectorQuery",
|
||||
]
|
||||
@@ -0,0 +1,439 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from typing import Optional, Union, overload
|
||||
|
||||
from zvec._zvec import _Collection
|
||||
|
||||
from ..executor import QueryContext, QueryExecutor
|
||||
from ..extension import ReRanker
|
||||
from ..typing import Status
|
||||
from .convert import convert_to_cpp_doc, convert_to_py_doc
|
||||
from .doc import Doc, DocList
|
||||
from .param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
FlatIndexParam,
|
||||
FtsIndexParam,
|
||||
HnswIndexParam,
|
||||
HnswRabitqIndexParam,
|
||||
IndexOption,
|
||||
InvertIndexParam,
|
||||
IVFIndexParam,
|
||||
OptimizeOption,
|
||||
)
|
||||
from .param.query import Query
|
||||
from .schema import CollectionSchema, CollectionStats, FieldSchema
|
||||
|
||||
__all__ = ["Collection"]
|
||||
|
||||
|
||||
class Collection:
|
||||
"""Represents an opened collection in Zvec.
|
||||
|
||||
A `Collection` provides methods for data definition (DDL), data manipulation (DML),
|
||||
and querying (DQL). It is obtained via `create_and_open()` or `open()`.
|
||||
|
||||
This class is not meant to be instantiated directly; use factory functions instead.
|
||||
"""
|
||||
|
||||
def __init__(self, obj: _Collection):
|
||||
self._obj = obj
|
||||
self._schema = None
|
||||
self._querier = None
|
||||
|
||||
@classmethod
|
||||
def _from_core(cls, core_collection: _Collection) -> Collection:
|
||||
if not core_collection:
|
||||
raise ValueError("Collection is None")
|
||||
inst = cls.__new__(cls)
|
||||
inst._obj = core_collection
|
||||
schema = CollectionSchema._from_core(core_collection.Schema())
|
||||
inst._schema = schema
|
||||
inst._querier = QueryExecutor(schema)
|
||||
return inst
|
||||
|
||||
@property
|
||||
def path(self) -> str:
|
||||
"""str: The filesystem path of the collection."""
|
||||
return self._obj.Path()
|
||||
|
||||
@property
|
||||
def option(self) -> CollectionOption:
|
||||
"""CollectionOption: The options used to open the collection."""
|
||||
return self._obj.Options()
|
||||
|
||||
@property
|
||||
def schema(self) -> CollectionSchema:
|
||||
"""CollectionSchema: The schema defining the structure of the collection."""
|
||||
return self._schema
|
||||
|
||||
@property
|
||||
def stats(self) -> CollectionStats:
|
||||
"""CollectionStats: Runtime statistics about the collection (e.g., doc count, size)."""
|
||||
return self._obj.Stats()
|
||||
|
||||
# ========== Collection DDL Methods ==========
|
||||
def destroy(self) -> None:
|
||||
"""Permanently delete the collection from disk.
|
||||
|
||||
Warning:
|
||||
This operation is irreversible. All data will be lost.
|
||||
"""
|
||||
self._obj.Destroy()
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Force all pending writes to disk.
|
||||
|
||||
Ensures durability of recent inserts/updates.
|
||||
"""
|
||||
self._obj.Flush()
|
||||
|
||||
# ========== Index DDL Methods ==========
|
||||
def create_index(
|
||||
self,
|
||||
field_name: str,
|
||||
index_param: Union[
|
||||
HnswIndexParam,
|
||||
HnswRabitqIndexParam,
|
||||
IVFIndexParam,
|
||||
FlatIndexParam,
|
||||
InvertIndexParam,
|
||||
FtsIndexParam,
|
||||
],
|
||||
option: IndexOption = IndexOption(),
|
||||
) -> None:
|
||||
"""Create an index on a field.
|
||||
|
||||
Vector index types (HNSW, IVF, FLAT) can only be applied to vector fields.
|
||||
Inverted index (`InvertIndexParam`) is for scalar fields.
|
||||
FTS index (`FtsIndexParam`) is for full-text search on STRING fields.
|
||||
|
||||
Args:
|
||||
field_name (str): Name of the field to index.
|
||||
index_param (Union[HnswIndexParam, HnswRabitqIndexParam, IVFIndexParam, FlatIndexParam, InvertIndexParam, FtsIndexParam]):
|
||||
Index configuration.
|
||||
option (Optional[IndexOption], optional): Index creation options.
|
||||
Defaults to ``IndexOption()``.
|
||||
|
||||
"""
|
||||
self._obj.CreateIndex(field_name, index_param, option)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
self._querier._schema = self._schema
|
||||
|
||||
def drop_index(self, field_name: str) -> None:
|
||||
"""Remove the index from a field.
|
||||
|
||||
Args:
|
||||
field_name (str): Name of the indexed field.
|
||||
"""
|
||||
self._obj.DropIndex(field_name)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
self._querier._schema = self._schema
|
||||
|
||||
def optimize(self, option: OptimizeOption = OptimizeOption()) -> None:
|
||||
"""Optimize the collection (e.g., merge segments, rebuild index).
|
||||
|
||||
Args:
|
||||
option (Optional[OptimizeOption], optional): Optimization options.
|
||||
Defaults to ``OptimizeOption()``.
|
||||
"""
|
||||
self._obj.Optimize(option)
|
||||
|
||||
# ========== COLUMN DDL Methods ==========
|
||||
def add_column(
|
||||
self,
|
||||
field_schema: FieldSchema,
|
||||
expression: str = "",
|
||||
option: AddColumnOption = AddColumnOption(),
|
||||
) -> None:
|
||||
"""Add a new column to the collection.
|
||||
|
||||
The column is populated using the provided expression (e.g., SQL-like formula).
|
||||
|
||||
Args:
|
||||
field_schema (FieldSchema): Schema definition for the new column.
|
||||
expression (str): Expression to compute values for existing documents.
|
||||
option (Optional[AddColumnOption], optional): Options for the operation.
|
||||
Defaults to ``AddColumnOption()``.
|
||||
"""
|
||||
self._obj.AddColumn(field_schema._get_object(), expression, option)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
self._querier._schema = self._schema
|
||||
|
||||
def drop_column(self, field_name: str) -> None:
|
||||
"""Remove a column from the collection.
|
||||
|
||||
Args:
|
||||
field_name (str): Name of the column to drop.
|
||||
"""
|
||||
self._obj.DropColumn(field_name)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
self._querier._schema = self._schema
|
||||
|
||||
def alter_column(
|
||||
self,
|
||||
old_name: str,
|
||||
new_name: Optional[str] = None,
|
||||
field_schema: Optional[FieldSchema] = None,
|
||||
option: AlterColumnOption = AlterColumnOption(),
|
||||
) -> None:
|
||||
"""Rename a column, update its schema.
|
||||
|
||||
This method supports three atomic operations:
|
||||
1. Rename only (when `field_schema` is None).
|
||||
2. Modify schema only (when `new_name` is None or empty string).
|
||||
|
||||
Args:
|
||||
old_name (str): The current name of the column to be altered.
|
||||
new_name (Optional[str]): The new name for the column.
|
||||
- If provided and non-empty, the column will be renamed.
|
||||
- If `None` or empty string, no rename occurs.
|
||||
field_schema (Optional[FieldSchema]): The new schema definition.
|
||||
- If provided, the column's type, dimension, or other properties will be updated.
|
||||
- If `None`, only renaming (if requested) is performed.
|
||||
option (AlterColumnOption, optional): Options controlling the alteration behavior.
|
||||
Defaults to ``AlterColumnOption()``.
|
||||
|
||||
**Limitation**: This operation **only supports scalar numeric columns**. such as:
|
||||
- `DOUBLE`, `FLOAT`,
|
||||
- `INT32`, `INT64`, `UINT32`, `UINT64`
|
||||
|
||||
Note:
|
||||
- Schema modification may trigger data migration or index rebuild.
|
||||
|
||||
Examples:
|
||||
>>> # Rename column only
|
||||
>>> results = collection.alter_column(old_name="id", new_name="doc_id")
|
||||
|
||||
>>> # Modify schema only
|
||||
>>> new_schema = FieldSchema(name="doc_id", dtype=DataType.INT64)
|
||||
>>> collection.alter_column("id", field_schema=new_schema)
|
||||
"""
|
||||
self._obj.AlterColumn(
|
||||
old_name,
|
||||
new_name or "",
|
||||
field_schema._get_object() if field_schema else None,
|
||||
option,
|
||||
)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
self._querier._schema = self._schema
|
||||
|
||||
# ========== Collection DDL Methods ==========
|
||||
@overload
|
||||
def insert(self, docs: Doc) -> Status:
|
||||
pass
|
||||
|
||||
@overload
|
||||
def insert(self, docs: list[Doc]) -> list[Status]:
|
||||
pass
|
||||
|
||||
def insert(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
|
||||
"""Insert new documents into the collection.
|
||||
|
||||
Documents must have unique IDs and conform to the schema.
|
||||
|
||||
Args:
|
||||
docs (Union[Doc, list[Doc]]): One or more documents to insert.
|
||||
|
||||
Returns:
|
||||
Union[Status, list[Status]]: If a single Doc was given, returns its Status;
|
||||
if a list was given, returns a list of Status objects.
|
||||
"""
|
||||
is_single = isinstance(docs, Doc)
|
||||
doc_list = [docs] if is_single else docs
|
||||
results = self._obj.Insert(
|
||||
[convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
|
||||
)
|
||||
return results[0] if is_single else results
|
||||
|
||||
@overload
|
||||
def upsert(self, docs: Doc) -> Status:
|
||||
pass
|
||||
|
||||
@overload
|
||||
def upsert(self, docs: list[Doc]) -> list[Status]:
|
||||
pass
|
||||
|
||||
def upsert(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
|
||||
"""Insert new documents or update existing ones by ID.
|
||||
|
||||
Args:
|
||||
docs (Union[Doc, list[Doc]]): Documents to upsert.
|
||||
|
||||
Returns:
|
||||
Union[Status, list[Status]]: If a single Doc was given, returns its Status;
|
||||
if a list was given, returns a list of Status objects.
|
||||
"""
|
||||
is_single = isinstance(docs, Doc)
|
||||
doc_list = [docs] if is_single else docs
|
||||
results = self._obj.Upsert(
|
||||
[convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
|
||||
)
|
||||
return results[0] if is_single else results
|
||||
|
||||
@overload
|
||||
def update(self, docs: Doc) -> Status:
|
||||
pass
|
||||
|
||||
@overload
|
||||
def update(self, docs: list[Doc]) -> list[Status]:
|
||||
pass
|
||||
|
||||
def update(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
|
||||
"""Update existing documents by ID.
|
||||
|
||||
Only specified fields are updated; others remain unchanged.
|
||||
|
||||
Args:
|
||||
docs (Union[Doc, list[Doc]]): Documents containing updated fields.
|
||||
|
||||
Returns:
|
||||
Union[Status, list[Status]]: If a single Doc was given, returns its Status;
|
||||
if a list was given, returns a list of Status objects.
|
||||
"""
|
||||
is_single = isinstance(docs, Doc)
|
||||
doc_list = [docs] if is_single else docs
|
||||
results = self._obj.Update(
|
||||
[convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
|
||||
)
|
||||
return results[0] if is_single else results
|
||||
|
||||
@overload
|
||||
def delete(self, ids: str) -> Status:
|
||||
pass
|
||||
|
||||
@overload
|
||||
def delete(self, ids: list[str]) -> list[Status]:
|
||||
pass
|
||||
|
||||
def delete(self, ids: Union[str, list[str]]) -> Union[Status, list[Status]]:
|
||||
"""Delete documents by ID.
|
||||
|
||||
Args:
|
||||
ids (Union[str, list[str]]): One or more document IDs to delete.
|
||||
|
||||
Returns:
|
||||
Union[Status, list[Status]]: If a single id was given, returns its Status;
|
||||
if a list was given, returns a list of Status objects.
|
||||
"""
|
||||
is_single = isinstance(ids, str)
|
||||
id_list = [ids] if isinstance(ids, str) else ids
|
||||
results = self._obj.Delete(id_list)
|
||||
return results[0] if is_single else results
|
||||
|
||||
def delete_by_filter(self, filter: str) -> None:
|
||||
"""Delete documents matching a filter expression.
|
||||
|
||||
Args:
|
||||
filter (str): Boolean expression (e.g., ``"age > 30"``).
|
||||
"""
|
||||
self._obj.DeleteByFilter(filter)
|
||||
|
||||
# ========== Collection DQL-fetch Methods ==========
|
||||
def fetch(
|
||||
self,
|
||||
ids: Union[str, list[str]],
|
||||
*,
|
||||
output_fields: Optional[list[str]] = None,
|
||||
include_vector: bool = True,
|
||||
) -> dict[str, Doc]:
|
||||
"""Retrieve documents by ID.
|
||||
|
||||
Args:
|
||||
ids (Union[str, list[str]]): Document IDs to fetch.
|
||||
output_fields (Optional[list[str]], optional): Scalar fields to
|
||||
include. If None, all fields are returned. Defaults to None.
|
||||
include_vector (bool, optional): Whether to include vector data in
|
||||
results. Defaults to True.
|
||||
|
||||
Returns:
|
||||
dict[str, Doc]: Mapping from ID to document. Missing IDs are omitted.
|
||||
"""
|
||||
ids = [ids] if isinstance(ids, str) else ids
|
||||
docs = self._obj.Fetch(ids, output_fields, include_vector)
|
||||
return {
|
||||
doc_id: py_doc
|
||||
for doc_id, core_doc in docs.items()
|
||||
if (py_doc := convert_to_py_doc(core_doc, self.schema)) is not None
|
||||
}
|
||||
|
||||
# ========== Collection DQL-Query Methods ==========
|
||||
|
||||
def query(
|
||||
self,
|
||||
queries: Optional[Union[Query, list[Query]]] = None,
|
||||
*,
|
||||
vectors: Optional[Union[Query, list[Query]]] = None,
|
||||
topk: int = 10,
|
||||
filter: Optional[str] = None,
|
||||
include_vector: bool = False,
|
||||
output_fields: Optional[list[str]] = None,
|
||||
reranker: Optional[ReRanker] = None,
|
||||
) -> DocList:
|
||||
"""Perform vector similarity search with optional filtering and re-ranking.
|
||||
|
||||
At least one `Query` must be provided via `queries`.
|
||||
|
||||
Args:
|
||||
queries (Optional[Union[Query, list[Query]]], optional):
|
||||
One or more vector queries. Defaults to None.
|
||||
vectors (Optional[Union[Query, list[Query]]], optional):
|
||||
Deprecated. Use `queries` instead.
|
||||
topk (int, optional): Number of nearest neighbors to return.
|
||||
Defaults to 10.
|
||||
filter (Optional[str], optional): Boolean expression to pre-filter candidates.
|
||||
Defaults to None.
|
||||
include_vector (bool, optional): Whether to include vector data in results.
|
||||
Defaults to False.
|
||||
output_fields (Optional[list[str]], optional): Scalar fields to include.
|
||||
If None, all fields are returned. Defaults to None.
|
||||
reranker (Optional[ReRanker], optional): Re-ranker to refine results.
|
||||
Defaults to None.
|
||||
|
||||
Returns:
|
||||
DocList: Top-k matching documents, sorted by relevance score.
|
||||
|
||||
Examples:
|
||||
>>> from zvec import Query
|
||||
>>> results = collection.query(
|
||||
... queries=Query(field_name="embedding", vector=[0.1, 0.2]),
|
||||
... topk=5,
|
||||
... filter="category == 'tech'",
|
||||
... output_fields=["title", "url"]
|
||||
... )
|
||||
"""
|
||||
if vectors is not None:
|
||||
warnings.warn(
|
||||
"The 'vectors' parameter is deprecated and will be removed in a future version. "
|
||||
"Use 'queries' instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if queries is not None:
|
||||
raise ValueError("Cannot specify both 'queries' and 'vectors'.")
|
||||
queries = vectors
|
||||
|
||||
ctx = QueryContext(
|
||||
topk=topk,
|
||||
filter=filter,
|
||||
queries=[queries] if isinstance(queries, Query) else queries,
|
||||
include_vector=include_vector,
|
||||
output_fields=output_fields,
|
||||
reranker=reranker,
|
||||
)
|
||||
return self._querier.execute(ctx, self._obj)
|
||||
@@ -0,0 +1,54 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from zvec._zvec import _Doc
|
||||
|
||||
from .doc import Doc
|
||||
from .schema import CollectionSchema
|
||||
|
||||
|
||||
def convert_to_cpp_doc(doc: Doc, collection_schema: CollectionSchema) -> _Doc:
|
||||
if not doc or not collection_schema:
|
||||
return None
|
||||
|
||||
_doc = _Doc()
|
||||
|
||||
# set pk
|
||||
_doc.set_pk(doc.id)
|
||||
|
||||
# set scalar fields
|
||||
for k, v in doc.fields.items():
|
||||
field_schema = collection_schema.field(k)
|
||||
if not field_schema:
|
||||
raise ValueError(
|
||||
f"schema validate failed: {k} not found in collection schema"
|
||||
)
|
||||
_doc.set_any(k, field_schema._get_object(), v)
|
||||
|
||||
# set vector fields
|
||||
for k, v in doc.vectors.items():
|
||||
vector_schema = collection_schema.vector(k)
|
||||
if not vector_schema:
|
||||
raise ValueError(
|
||||
f"schema validate failed: {k} not found in collection schema"
|
||||
)
|
||||
_doc.set_any(k, vector_schema._get_object(), v)
|
||||
return _doc
|
||||
|
||||
|
||||
def convert_to_py_doc(doc: _Doc, collection_schema: CollectionSchema) -> Doc:
|
||||
if not doc or not collection_schema:
|
||||
return None
|
||||
|
||||
data_tuple = doc.get_all(collection_schema._get_object())
|
||||
return Doc._from_tuple(data_tuple)
|
||||
@@ -0,0 +1,178 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Optional
|
||||
|
||||
from ..common import VectorType
|
||||
|
||||
__all__ = [
|
||||
"Doc",
|
||||
"DocList",
|
||||
]
|
||||
|
||||
|
||||
class Doc:
|
||||
"""Represents a retrieved document with optional metadata, fields, and vectors.
|
||||
|
||||
This immutable data class encapsulates the result of a search or retrieval
|
||||
operation. It includes the document ID, relevance score (if applicable),
|
||||
scalar fields, and vector embeddings.
|
||||
|
||||
During initialization, any `numpy.ndarray` in `vectors` is automatically
|
||||
converted to a plain Python list for JSON serialization and immutability.
|
||||
|
||||
Attributes:
|
||||
id (str): Unique identifier of the document.
|
||||
score (Optional[float], optional): Relevance score from search.
|
||||
Defaults to None.
|
||||
vectors (Optional[dict[str, VectorType]], optional): Named vector
|
||||
embeddings associated with the document. Values are converted to
|
||||
lists if originally `np.ndarray`. Defaults to None.
|
||||
fields (Optional[dict[str, Any]], optional): Scalar metadata fields
|
||||
(e.g., title, timestamp). Defaults to None.
|
||||
|
||||
Examples:
|
||||
>>> import numpy as np
|
||||
>>> import zvec
|
||||
>>> doc = zvec.Doc(
|
||||
... id="doc1",
|
||||
... score=0.95,
|
||||
... vectors={"emb": np.array([0.1, 0.2, 0.3])},
|
||||
... fields={"title": "Hello World"}
|
||||
... )
|
||||
>>> print(doc.vector("emb"))
|
||||
[0.1, 0.2, 0.3]
|
||||
>>> print(doc.has_field("title"))
|
||||
True
|
||||
"""
|
||||
|
||||
__slots__ = ("id", "score", "vectors", "fields")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
id: str,
|
||||
score: Optional[float] = None,
|
||||
vectors: Optional[dict[str, VectorType]] = None,
|
||||
fields: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
self.id = id
|
||||
self.score = score
|
||||
self.vectors = vectors or {}
|
||||
self.fields = fields or {}
|
||||
|
||||
def has_field(self, name: str) -> bool:
|
||||
"""Check if the document contains a scalar field with the given name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the field to check.
|
||||
|
||||
Returns:
|
||||
bool: True if the field exists, False otherwise.
|
||||
"""
|
||||
return name in self.fields
|
||||
|
||||
def has_vector(self, name: str) -> bool:
|
||||
"""Check if the document contains a vector with the given name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the vector to check.
|
||||
|
||||
Returns:
|
||||
bool: True if the vector exists, False otherwise.
|
||||
"""
|
||||
return name in self.vectors
|
||||
|
||||
def vector(self, name: str):
|
||||
"""Get a vector by name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the vector.
|
||||
|
||||
Returns:
|
||||
Any: The vector (as a list) if it exists, otherwise None.
|
||||
"""
|
||||
return self.vectors and self.vectors.get(name)
|
||||
|
||||
def field(self, name: str):
|
||||
"""Get a scalar field by name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the field.
|
||||
|
||||
Returns:
|
||||
Any: The field value if it exists, otherwise None.
|
||||
"""
|
||||
return self.fields and self.fields.get(name)
|
||||
|
||||
def vector_names(self) -> list[str]:
|
||||
"""Get the list of all vector names in this document.
|
||||
|
||||
Returns:
|
||||
list[str]: A list of vector field names. Empty if no vectors.
|
||||
"""
|
||||
return [] if not self.vectors else list(self.vectors.keys())
|
||||
|
||||
def field_names(self) -> list[str]:
|
||||
"""Get the list of all scalar field names in this document.
|
||||
|
||||
Returns:
|
||||
list[str]: A list of field names. Empty if no fields.
|
||||
"""
|
||||
return [] if not self.fields else list(self.fields.keys())
|
||||
|
||||
def __repr__(self) -> str:
|
||||
try:
|
||||
schema = {
|
||||
"id": self.id,
|
||||
"score": self.score,
|
||||
"fields": self.fields,
|
||||
"vectors": self.vectors,
|
||||
}
|
||||
return json.dumps(schema, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"<Doc error during repr: {e}>"
|
||||
|
||||
def _replace(self, **changes):
|
||||
new_tuple = (
|
||||
changes.get("id", self.id),
|
||||
changes.get("score", self.score),
|
||||
changes.get("fields", self.fields.copy() if self.fields else None),
|
||||
changes.get("vectors", self.vectors.copy() if self.vectors else None),
|
||||
)
|
||||
return type(self)._from_tuple(new_tuple)
|
||||
|
||||
@classmethod
|
||||
def _from_tuple(
|
||||
cls, data_tuple: tuple[str, float, dict[str, Any], dict[str, VectorType]]
|
||||
):
|
||||
obj = object.__new__(cls)
|
||||
obj.id = data_tuple[0]
|
||||
obj.score = data_tuple[1]
|
||||
obj.fields = data_tuple[2] or {}
|
||||
|
||||
vectors = data_tuple[3]
|
||||
if vectors is not None:
|
||||
obj.vectors = {
|
||||
name: (vec.tolist() if hasattr(vec, "tolist") else vec)
|
||||
for name, vec in vectors.items()
|
||||
}
|
||||
else:
|
||||
obj.vectors = {}
|
||||
return obj
|
||||
|
||||
|
||||
#: Type alias for query results: a list of documents returned by a single query route.
|
||||
DocList = list[Doc]
|
||||
@@ -0,0 +1,60 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from zvec._zvec.param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
DiskAnnIndexParam,
|
||||
DiskAnnQueryParam,
|
||||
FlatIndexParam,
|
||||
FtsIndexParam,
|
||||
FtsQueryParam,
|
||||
HnswIndexParam,
|
||||
HnswQueryParam,
|
||||
HnswRabitqIndexParam,
|
||||
HnswRabitqQueryParam,
|
||||
IndexOption,
|
||||
InvertIndexParam,
|
||||
IVFIndexParam,
|
||||
IVFQueryParam,
|
||||
OptimizeOption,
|
||||
QuantizerParam,
|
||||
VamanaIndexParam,
|
||||
VamanaQueryParam,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AddColumnOption",
|
||||
"AlterColumnOption",
|
||||
"CollectionOption",
|
||||
"DiskAnnIndexParam",
|
||||
"DiskAnnQueryParam",
|
||||
"FlatIndexParam",
|
||||
"FtsIndexParam",
|
||||
"FtsQueryParam",
|
||||
"HnswIndexParam",
|
||||
"HnswQueryParam",
|
||||
"HnswRabitqIndexParam",
|
||||
"HnswRabitqQueryParam",
|
||||
"IVFIndexParam",
|
||||
"IVFQueryParam",
|
||||
"IndexOption",
|
||||
"InvertIndexParam",
|
||||
"OptimizeOption",
|
||||
"QuantizerParam",
|
||||
"VamanaIndexParam",
|
||||
"VamanaQueryParam",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,143 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Union
|
||||
|
||||
from ...common import VectorType
|
||||
from . import FtsQueryParam, HnswQueryParam, HnswRabitqQueryParam, IVFQueryParam
|
||||
|
||||
__all__ = ["Fts", "Query", "VectorQuery"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Fts:
|
||||
"""Full-text search query parameters.
|
||||
|
||||
Attributes:
|
||||
query_string (Optional[str]): FTS query expression
|
||||
(e.g. '+vector -slow "exact phrase"'). Mutually exclusive with match_string.
|
||||
match_string (Optional[str]): Natural language match string,
|
||||
tokenized and combined using the default operator.
|
||||
Mutually exclusive with query_string.
|
||||
"""
|
||||
|
||||
query_string: Optional[str] = None
|
||||
match_string: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Query:
|
||||
"""Represents a search query for a specific field in a collection.
|
||||
|
||||
A `Query` can be constructed for either vector search or full-text search,
|
||||
but not both simultaneously.
|
||||
|
||||
For vector search, provide `id` or `vector` (and optionally `param`).
|
||||
For FTS, provide `fts`.
|
||||
|
||||
Attributes:
|
||||
field_name (str): Name of the field to query.
|
||||
id (Optional[str], optional): Document ID to fetch vector from. Default is None.
|
||||
vector (VectorType, optional): Explicit query vector. Default is None.
|
||||
param (Optional[Union[HnswQueryParam, HnswRabitqQueryParam, IVFQueryParam, FtsQueryParam]], optional):
|
||||
Index-specific query parameters. Default is None.
|
||||
fts (Optional[Fts], optional): Full-text search parameters. Default is None.
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
>>> # Query by ID
|
||||
>>> q1 = zvec.Query(field_name="embedding", id="doc123")
|
||||
>>> # Query by vector
|
||||
>>> q2 = zvec.Query(
|
||||
... field_name="embedding",
|
||||
... vector=[0.1, 0.2, 0.3],
|
||||
... param=HnswQueryParam(ef=300)
|
||||
... )
|
||||
>>> # FTS query
|
||||
>>> q3 = zvec.Query(
|
||||
... field_name="content",
|
||||
... fts=Fts(match_string="machine learning")
|
||||
... )
|
||||
>>> # FTS query with custom operator
|
||||
>>> q4 = zvec.Query(
|
||||
... field_name="content",
|
||||
... fts=Fts(match_string="machine learning"),
|
||||
... param=FtsQueryParam(default_operator="AND")
|
||||
... )
|
||||
"""
|
||||
|
||||
field_name: str
|
||||
id: Optional[str] = None
|
||||
vector: VectorType = None
|
||||
param: Optional[
|
||||
Union[HnswQueryParam, HnswRabitqQueryParam, IVFQueryParam, FtsQueryParam]
|
||||
] = None
|
||||
fts: Optional[Fts] = None
|
||||
|
||||
def has_id(self) -> bool:
|
||||
"""Check if the query is based on a document ID.
|
||||
|
||||
Returns:
|
||||
bool: True if `id` is set, False otherwise.
|
||||
"""
|
||||
return self.id is not None
|
||||
|
||||
def has_vector(self) -> bool:
|
||||
"""Check if the query contains an explicit vector.
|
||||
|
||||
Returns:
|
||||
bool: True if `vector` is non-empty, False otherwise.
|
||||
"""
|
||||
return self.vector is not None and len(self.vector) > 0
|
||||
|
||||
def has_fts(self) -> bool:
|
||||
"""Check if the query contains an FTS (full-text search) condition.
|
||||
|
||||
Returns:
|
||||
bool: True if `fts` is set with a query_string or match_string.
|
||||
"""
|
||||
if self.fts is not None:
|
||||
return bool(self.fts.query_string) or bool(self.fts.match_string)
|
||||
return False
|
||||
|
||||
def _validate(self) -> None:
|
||||
if self.field_name is None:
|
||||
raise ValueError("Field name cannot be empty")
|
||||
if self.has_id() and self.has_vector():
|
||||
raise ValueError("Cannot provide both id and vector")
|
||||
if self.has_fts() and (self.has_vector() or self.has_id()):
|
||||
raise ValueError(
|
||||
"Cannot combine fts with vector search fields (id/vector) in a single Query"
|
||||
)
|
||||
if self.fts is not None and self.fts.query_string and self.fts.match_string:
|
||||
raise ValueError(
|
||||
"Cannot provide both query_string and match_string in Fts; "
|
||||
"they are mutually exclusive"
|
||||
)
|
||||
|
||||
|
||||
class VectorQuery(Query):
|
||||
"""Deprecated alias for Query. Use Query instead."""
|
||||
|
||||
def __new__(cls, *args, **kwargs): # noqa : ARG004
|
||||
warnings.warn(
|
||||
"VectorQuery is deprecated and will be removed in a future version. "
|
||||
"Use Query instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return super().__new__(cls)
|
||||
@@ -0,0 +1,21 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from zvec._zvec.schema import CollectionStats
|
||||
|
||||
from .collection_schema import CollectionSchema
|
||||
from .field_schema import FieldSchema, VectorSchema
|
||||
|
||||
__all__ = ["CollectionSchema", "CollectionStats", "FieldSchema", "VectorSchema"]
|
||||
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
This module contains the schema of Zvec
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections.abc
|
||||
import typing
|
||||
|
||||
import zvec._zvec.param
|
||||
import zvec._zvec.typing
|
||||
|
||||
from .collection_schema import CollectionSchema
|
||||
from .field_schema import FieldSchema, VectorSchema
|
||||
|
||||
__all__: list[str] = [
|
||||
"CollectionSchema",
|
||||
"CollectionStats",
|
||||
"FieldSchema",
|
||||
"VectorSchema",
|
||||
]
|
||||
|
||||
class CollectionStats:
|
||||
def __init__(self) -> None: ...
|
||||
def __repr__(self) -> str: ...
|
||||
@property
|
||||
def doc_count(self) -> int: ...
|
||||
@property
|
||||
def index_completeness(self) -> dict[str, float]: ...
|
||||
|
||||
class _CollectionSchema:
|
||||
__hash__: typing.ClassVar[None] = None
|
||||
|
||||
def __eq__(self, arg0: _CollectionSchema) -> bool: ...
|
||||
def __init__(
|
||||
self, name: str, fields: collections.abc.Sequence[_FieldSchema]
|
||||
) -> None:
|
||||
"""
|
||||
Construct with name and list of fields
|
||||
"""
|
||||
|
||||
def __ne__(self, arg0: _CollectionSchema) -> bool: ...
|
||||
def fields(self) -> list[_FieldSchema]:
|
||||
"""
|
||||
Return list of all field schemas.
|
||||
"""
|
||||
|
||||
def forward_fields(self) -> list[_FieldSchema]:
|
||||
"""
|
||||
Return list of forward-indexed fields.
|
||||
"""
|
||||
|
||||
def get_field(self, field_name: str) -> _FieldSchema:
|
||||
"""
|
||||
Get field by name (const pointer), returns None if not found.
|
||||
"""
|
||||
|
||||
def get_forward_field(self, field_name: str) -> _FieldSchema:
|
||||
"""
|
||||
Get forward field (used for filtering).
|
||||
"""
|
||||
|
||||
def get_vector_field(self, field_name: str) -> _FieldSchema:
|
||||
"""
|
||||
Get vector field by name.
|
||||
"""
|
||||
|
||||
def has_field(self, field_name: str) -> bool:
|
||||
"""
|
||||
Check if a field exists.
|
||||
"""
|
||||
|
||||
def vector_fields(self) -> list[_FieldSchema]:
|
||||
"""
|
||||
Return list of vector fields.
|
||||
"""
|
||||
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
|
||||
class _FieldSchema:
|
||||
__hash__: typing.ClassVar[None] = None
|
||||
|
||||
def __eq__(self, arg0: _FieldSchema) -> bool: ...
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
data_type: zvec._zvec.typing.DataType,
|
||||
nullable: bool = False,
|
||||
dimension: typing.SupportsInt = 0,
|
||||
index_param: zvec._zvec.param.IndexParam = None,
|
||||
) -> None: ...
|
||||
def __ne__(self, arg0: _FieldSchema) -> bool: ...
|
||||
@property
|
||||
def data_type(self) -> zvec._zvec.typing.DataType: ...
|
||||
@property
|
||||
def dimension(self) -> int: ...
|
||||
@property
|
||||
def index_param(self) -> typing.Any: ...
|
||||
@property
|
||||
def index_type(self) -> zvec._zvec.typing.IndexType: ...
|
||||
@property
|
||||
def is_dense_vector(self) -> bool: ...
|
||||
@property
|
||||
def is_sparse_vector(self) -> bool: ...
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
@property
|
||||
def nullable(self) -> bool: ...
|
||||
@@ -0,0 +1,215 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Optional, Union
|
||||
|
||||
from zvec._zvec.schema import _CollectionSchema, _FieldSchema
|
||||
|
||||
from .field_schema import FieldSchema, VectorSchema
|
||||
|
||||
__all__ = [
|
||||
"CollectionSchema",
|
||||
]
|
||||
|
||||
|
||||
class CollectionSchema:
|
||||
"""Defines the structure of a collection in Zvec.
|
||||
|
||||
A collection schema specifies the name of the collection and its fields,
|
||||
including both scalar fields (e.g., int, string) and vector fields.
|
||||
Field names must be unique across both scalar and vector fields.
|
||||
|
||||
Args:
|
||||
name (str): Name of the collection.
|
||||
fields (Optional[Union[FieldSchema, list[FieldSchema]]], optional):
|
||||
One or more scalar field definitions. Defaults to None.
|
||||
vectors (Optional[Union[VectorSchema, list[VectorSchema]]], optional):
|
||||
One or more vector field definitions. Defaults to None.
|
||||
|
||||
Raises:
|
||||
TypeError: If `fields` or `vectors` are of unsupported types.
|
||||
ValueError: If any field or vector name is duplicated.
|
||||
|
||||
Examples:
|
||||
>>> from zvec import FieldSchema, VectorSchema, DataType, IndexType
|
||||
>>> id_field = FieldSchema("id", DataType.INT64, is_primary=True)
|
||||
>>> emb_field = VectorSchema("embedding", dim=128, data_type=DataType.VECTOR_FP32)
|
||||
>>> schema = CollectionSchema(
|
||||
... name="my_collection",
|
||||
... fields=id_field,
|
||||
... vectors=emb_field
|
||||
... )
|
||||
>>> print(schema.name)
|
||||
my_collection
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
fields: Optional[Union[FieldSchema, list[FieldSchema]]] = None,
|
||||
vectors: Optional[Union[VectorSchema, list[VectorSchema]]] = None,
|
||||
):
|
||||
if name is None or not isinstance(name, str):
|
||||
raise ValueError(
|
||||
f"schema validate failed: collection name must be str, got {type(name).__name__}"
|
||||
)
|
||||
|
||||
# handle fields
|
||||
_fields_name: list[str] = []
|
||||
_fields_list: list[_FieldSchema] = []
|
||||
|
||||
self._check_fields(fields, _fields_name, _fields_list)
|
||||
self._check_vectors(vectors, _fields_name, _fields_list)
|
||||
|
||||
# init
|
||||
self._cpp_obj = _CollectionSchema(
|
||||
name=name,
|
||||
fields=_fields_list,
|
||||
)
|
||||
|
||||
def _check_fields(
|
||||
self,
|
||||
fields: Optional[Union[FieldSchema, list[FieldSchema]]],
|
||||
_fields_name: list[str],
|
||||
_fields_list: list[_FieldSchema],
|
||||
) -> None:
|
||||
field_items = []
|
||||
|
||||
if isinstance(fields, FieldSchema):
|
||||
field_items = [fields]
|
||||
elif isinstance(fields, list):
|
||||
field_items = fields
|
||||
elif fields is None:
|
||||
field_items = []
|
||||
else:
|
||||
raise TypeError(
|
||||
f"schema validate failed: invalid 'fields' type, expected FieldSchema or list[FieldSchema], "
|
||||
f"got {type(fields).__name__}"
|
||||
)
|
||||
|
||||
for idx, field in enumerate(field_items):
|
||||
if not isinstance(field, FieldSchema):
|
||||
raise TypeError(
|
||||
f"schema validate failed: invalid field type in 'fields' list, expected FieldSchema, "
|
||||
f"got {type(field).__name__} at index {idx}"
|
||||
)
|
||||
|
||||
if field.name in _fields_name:
|
||||
raise ValueError(
|
||||
f"schema validate failed: duplicate field name '{field.name}': field names must be unique"
|
||||
)
|
||||
_fields_name.append(field.name)
|
||||
_fields_list.append(field._get_object())
|
||||
|
||||
def _check_vectors(
|
||||
self,
|
||||
vectors: Optional[Union[VectorSchema, list[VectorSchema]]],
|
||||
_fields_name: list[str],
|
||||
_fields_list: list[_FieldSchema],
|
||||
) -> None:
|
||||
# handle vector
|
||||
if isinstance(vectors, VectorSchema):
|
||||
vectors_items = [vectors]
|
||||
elif isinstance(vectors, list):
|
||||
vectors_items = vectors
|
||||
elif vectors is None:
|
||||
vectors_items = []
|
||||
else:
|
||||
raise TypeError(
|
||||
f"schema validate failed: invalid 'vectors' type, expected VectorSchema or list[VectorSchema], "
|
||||
f"got {type(vectors).__name__}"
|
||||
)
|
||||
|
||||
for idx, vector in enumerate(vectors_items):
|
||||
if not isinstance(vector, VectorSchema):
|
||||
raise TypeError(
|
||||
f"schema validate failed: invalid vector type in 'vectors' list, expected VectorSchema, "
|
||||
f"got {type(vector).__name__} at index {idx}"
|
||||
)
|
||||
|
||||
if vector.name in _fields_name:
|
||||
raise ValueError(
|
||||
f"schema validate failed: duplicate vector name '{vector.name}', vector names must be unique "
|
||||
f"(conflicts with existing field or vector)"
|
||||
)
|
||||
_fields_name.append(vector.name)
|
||||
_fields_list.append(vector._get_object())
|
||||
|
||||
@classmethod
|
||||
def _from_core(cls, core_collection_schema: _CollectionSchema):
|
||||
inst = cls.__new__(cls)
|
||||
if not core_collection_schema:
|
||||
raise ValueError("schema validate failed: schema is null")
|
||||
inst._cpp_obj = core_collection_schema
|
||||
return inst
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""str: The name of the collection."""
|
||||
return self._cpp_obj.name
|
||||
|
||||
def field(self, name: str) -> Optional[FieldSchema]:
|
||||
"""Retrieve a scalar field by name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the field.
|
||||
|
||||
Returns:
|
||||
Optional[FieldSchema]: The field if found, otherwise None.
|
||||
"""
|
||||
_field = self._cpp_obj.get_forward_field(name)
|
||||
return FieldSchema._from_core(_field) if _field else None
|
||||
|
||||
def vector(self, name: str) -> Optional[VectorSchema]:
|
||||
"""Retrieve a vector field by name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the vector field.
|
||||
|
||||
Returns:
|
||||
Optional[VectorSchema]: The vector field if found, otherwise None.
|
||||
"""
|
||||
_field = self._cpp_obj.get_vector_field(name)
|
||||
return VectorSchema._from_core(_field) if _field else None
|
||||
|
||||
@property
|
||||
def fields(self) -> list[FieldSchema]:
|
||||
"""list[FieldSchema]: All scalar (non-vector) fields in the schema."""
|
||||
_fields = self._cpp_obj.forward_fields()
|
||||
return [FieldSchema._from_core(_field) for _field in _fields]
|
||||
|
||||
@property
|
||||
def vectors(self) -> list[VectorSchema]:
|
||||
"""list[VectorSchema]: All vector fields in the schema."""
|
||||
_vectors = self._cpp_obj.vector_fields()
|
||||
return [VectorSchema._from_core(_vector) for _vector in _vectors]
|
||||
|
||||
def _get_object(self) -> _CollectionSchema:
|
||||
return self._cpp_obj
|
||||
|
||||
def __repr__(self) -> str:
|
||||
try:
|
||||
schema = {
|
||||
"name": self.name,
|
||||
"fields": {field.name: field.__dict__() for field in self.fields},
|
||||
"vectors": {vector.name: vector.__dict__() for vector in self.vectors},
|
||||
}
|
||||
return json.dumps(schema, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"<CollectionSchema error during repr: {e}>"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__repr__()
|
||||
@@ -0,0 +1,310 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from zvec._zvec.schema import _FieldSchema
|
||||
from zvec.model.param import (
|
||||
FlatIndexParam,
|
||||
FtsIndexParam,
|
||||
HnswIndexParam,
|
||||
HnswRabitqIndexParam,
|
||||
InvertIndexParam,
|
||||
IVFIndexParam,
|
||||
)
|
||||
from zvec.typing import DataType
|
||||
|
||||
__all__ = [
|
||||
"FieldSchema",
|
||||
"VectorSchema",
|
||||
]
|
||||
|
||||
SUPPORT_VECTOR_DATA_TYPE = [
|
||||
DataType.VECTOR_FP16,
|
||||
DataType.VECTOR_FP32,
|
||||
DataType.VECTOR_FP64,
|
||||
DataType.VECTOR_INT8,
|
||||
DataType.SPARSE_VECTOR_FP16,
|
||||
DataType.SPARSE_VECTOR_FP32,
|
||||
]
|
||||
|
||||
SUPPORT_SCALAR_DATA_TYPE = [
|
||||
DataType.INT32,
|
||||
DataType.INT64,
|
||||
DataType.UINT32,
|
||||
DataType.UINT64,
|
||||
DataType.FLOAT,
|
||||
DataType.DOUBLE,
|
||||
DataType.STRING,
|
||||
DataType.BOOL,
|
||||
DataType.ARRAY_INT32,
|
||||
DataType.ARRAY_INT64,
|
||||
DataType.ARRAY_UINT32,
|
||||
DataType.ARRAY_UINT64,
|
||||
DataType.ARRAY_FLOAT,
|
||||
DataType.ARRAY_DOUBLE,
|
||||
DataType.ARRAY_STRING,
|
||||
DataType.ARRAY_BOOL,
|
||||
]
|
||||
|
||||
|
||||
class FieldSchema:
|
||||
"""Represents a scalar (non-vector) field in a collection schema.
|
||||
|
||||
A `FieldSchema` defines the name, data type, nullability, and optional
|
||||
inverted index configuration for a regular field (e.g., ID, timestamp, category).
|
||||
|
||||
Args:
|
||||
name (str): Name of the field. Must be unique within the collection.
|
||||
data_type (DataType): Data type of the field (e.g., INT64, STRING).
|
||||
nullable (bool, optional): Whether the field can contain null values.
|
||||
Defaults to False.
|
||||
index_param (Optional[Union[InvertIndexParam, FtsIndexParam]], optional):
|
||||
Index parameters for this field. Use ``InvertIndexParam`` for scalar
|
||||
inverted indexing, or ``FtsIndexParam`` for full-text search indexing
|
||||
on STRING fields. Defaults to None.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import DataType
|
||||
>>> from zvec.model.param import InvertIndexParam, FtsIndexParam
|
||||
>>> id_field = FieldSchema(
|
||||
... name="id",
|
||||
... data_type=DataType.INT64,
|
||||
... nullable=False,
|
||||
... index_param=InvertIndexParam(enable_range_optimization=True)
|
||||
... )
|
||||
>>> content_field = FieldSchema(
|
||||
... name="content",
|
||||
... data_type=DataType.STRING,
|
||||
... nullable=False,
|
||||
... index_param=FtsIndexParam(tokenizer_name="standard")
|
||||
... )
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
data_type: DataType,
|
||||
nullable: bool = False,
|
||||
index_param: Optional[Union[InvertIndexParam, FtsIndexParam]] = None,
|
||||
):
|
||||
if name is None or not isinstance(name, str):
|
||||
raise ValueError(
|
||||
f"schema validate failed: field name must be str, got {type(name).__name__}"
|
||||
)
|
||||
|
||||
if data_type not in SUPPORT_SCALAR_DATA_TYPE:
|
||||
raise ValueError(
|
||||
f"schema validate failed: scalar_field's data_type must be one of "
|
||||
f"{', '.join(str(dt) for dt in SUPPORT_SCALAR_DATA_TYPE)}, "
|
||||
f"but field[{name}]'s data_type is {data_type}"
|
||||
)
|
||||
|
||||
self._cpp_obj = _FieldSchema(
|
||||
name=name,
|
||||
data_type=data_type,
|
||||
dimension=0,
|
||||
nullable=nullable,
|
||||
index_param=index_param,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _from_core(cls, core_field_schema: _FieldSchema):
|
||||
if core_field_schema is None:
|
||||
raise ValueError("schema validate failed: field schema is None")
|
||||
inst = cls.__new__(cls)
|
||||
inst._cpp_obj = core_field_schema
|
||||
return inst
|
||||
|
||||
def _get_object(self) -> _FieldSchema:
|
||||
return self._cpp_obj
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""str: The name of the field."""
|
||||
return self._cpp_obj.name
|
||||
|
||||
@property
|
||||
def data_type(self) -> DataType:
|
||||
"""DataType: The data type of the field (e.g., INT64, STRING)."""
|
||||
return self._cpp_obj.data_type
|
||||
|
||||
@property
|
||||
def nullable(self) -> bool:
|
||||
"""bool: Whether the field allows null values."""
|
||||
return self._cpp_obj.nullable
|
||||
|
||||
@property
|
||||
def index_param(self) -> Optional[Union[InvertIndexParam, FtsIndexParam]]:
|
||||
"""Optional[Union[InvertIndexParam, FtsIndexParam]]: Index configuration, if any."""
|
||||
return self._cpp_obj.index_param
|
||||
|
||||
def __dict__(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"data_type": (
|
||||
self.data_type.name
|
||||
if hasattr(self.data_type, "name")
|
||||
else str(self.data_type)
|
||||
),
|
||||
"nullable": self.nullable,
|
||||
"index_param": (
|
||||
self.index_param.to_dict() if self.index_param is not None else None
|
||||
),
|
||||
}
|
||||
|
||||
def __repr__(self) -> str:
|
||||
try:
|
||||
schema = self.__dict__()
|
||||
return json.dumps(schema, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"<FieldSchema error during repr: {e}>"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__repr__()
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, FieldSchema):
|
||||
return False
|
||||
return self._cpp_obj == other._cpp_obj
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash((self.name, self.data_type, self.nullable))
|
||||
|
||||
|
||||
class VectorSchema:
|
||||
"""Represents a vector field in a collection schema.
|
||||
|
||||
A `VectorSchema` defines the name, data type, dimensionality, and index
|
||||
configuration for a vector field used in similarity search.
|
||||
|
||||
Args:
|
||||
name (str): Name of the vector field. Must be unique within the collection.
|
||||
data_type (DataType): Vector data type (e.g., VECTOR_FP32, VECTOR_INT8).
|
||||
dimension (int, optional): Dimensionality of the vector. Must be > 0 for dense vectors;
|
||||
may be `None` for sparse vectors.
|
||||
index_param (Union[HnswIndexParam, IVFIndexParam, FlatIndexParam], optional):
|
||||
Index configuration for this vector field. Defaults to
|
||||
``HnswIndexParam()``.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import DataType
|
||||
>>> from zvec.model.param import HnswIndexParam
|
||||
>>> emb_field = VectorSchema(
|
||||
... name="embedding",
|
||||
... data_type=DataType.VECTOR_FP32,
|
||||
... dimension=128,
|
||||
... index_param=HnswIndexParam(ef_construction=200, m=16)
|
||||
... )
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
data_type: DataType,
|
||||
dimension: Optional[int] = 0,
|
||||
index_param: Optional[
|
||||
Union[HnswIndexParam, HnswRabitqIndexParam, FlatIndexParam, IVFIndexParam]
|
||||
] = None,
|
||||
):
|
||||
if name is None or not isinstance(name, str):
|
||||
raise ValueError(
|
||||
f"schema validate failed: field name must be str, got {type(name).__name__}"
|
||||
)
|
||||
|
||||
if not isinstance(dimension, int) or dimension < 0:
|
||||
raise ValueError("schema validate failed: vector's dimension must be >= 0")
|
||||
|
||||
if data_type not in SUPPORT_VECTOR_DATA_TYPE:
|
||||
raise ValueError(
|
||||
f"schema validate failed: vector's data_type must be one of "
|
||||
f"{', '.join(str(dt) for dt in SUPPORT_VECTOR_DATA_TYPE)}, "
|
||||
f"but field[{name}]'s data_type is {data_type}"
|
||||
)
|
||||
|
||||
if index_param is None:
|
||||
index_param = FlatIndexParam()
|
||||
|
||||
self._cpp_obj = _FieldSchema(
|
||||
name=name,
|
||||
data_type=data_type,
|
||||
dimension=dimension,
|
||||
nullable=False,
|
||||
index_param=index_param,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _from_core(cls, core_field_schema: _FieldSchema):
|
||||
inst = cls.__new__(cls)
|
||||
inst._cpp_obj = core_field_schema
|
||||
return inst
|
||||
|
||||
def _get_object(self) -> _FieldSchema:
|
||||
return self._cpp_obj
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""str: The name of the vector field."""
|
||||
return self._cpp_obj.name
|
||||
|
||||
@property
|
||||
def data_type(self) -> DataType:
|
||||
"""DataType: The vector data type (e.g., VECTOR_FP32)."""
|
||||
return self._cpp_obj.data_type
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
"""int: The dimensionality of the vector."""
|
||||
return self._cpp_obj.dimension
|
||||
|
||||
@property
|
||||
def index_param(
|
||||
self,
|
||||
) -> Union[HnswIndexParam, HnswRabitqIndexParam, IVFIndexParam, FlatIndexParam]:
|
||||
"""Union[HnswIndexParam, HnswRabitqIndexParam, IVFIndexParam, FlatIndexParam]: Index configuration for the vector."""
|
||||
return self._cpp_obj.index_param
|
||||
|
||||
def __dict__(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"data_type": (
|
||||
self.data_type.name
|
||||
if hasattr(self.data_type, "name")
|
||||
else str(self.data_type)
|
||||
),
|
||||
"dimension": self.dimension,
|
||||
"index_param": (
|
||||
self.index_param.to_dict() if self.index_param is not None else None
|
||||
),
|
||||
}
|
||||
|
||||
def __repr__(self) -> str:
|
||||
try:
|
||||
schema = self.__dict__()
|
||||
return json.dumps(schema, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"<FieldSchema error during repr: {e}>"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__repr__()
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, VectorSchema):
|
||||
return False
|
||||
return self._cpp_obj == other._cpp_obj
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash((self.name, self.data_type, self.dimension))
|
||||
Reference in New Issue
Block a user