chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:47:42 +08:00
commit be3ef883e1
1214 changed files with 431743 additions and 0 deletions
+21
View File
@@ -0,0 +1,21 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from zvec._zvec.schema import CollectionStats
from .collection_schema import CollectionSchema
from .field_schema import FieldSchema, VectorSchema
__all__ = ["CollectionSchema", "CollectionStats", "FieldSchema", "VectorSchema"]
+109
View File
@@ -0,0 +1,109 @@
"""
This module contains the schema of Zvec
"""
from __future__ import annotations
import collections.abc
import typing
import zvec._zvec.param
import zvec._zvec.typing
from .collection_schema import CollectionSchema
from .field_schema import FieldSchema, VectorSchema
__all__: list[str] = [
"CollectionSchema",
"CollectionStats",
"FieldSchema",
"VectorSchema",
]
class CollectionStats:
def __init__(self) -> None: ...
def __repr__(self) -> str: ...
@property
def doc_count(self) -> int: ...
@property
def index_completeness(self) -> dict[str, float]: ...
class _CollectionSchema:
__hash__: typing.ClassVar[None] = None
def __eq__(self, arg0: _CollectionSchema) -> bool: ...
def __init__(
self, name: str, fields: collections.abc.Sequence[_FieldSchema]
) -> None:
"""
Construct with name and list of fields
"""
def __ne__(self, arg0: _CollectionSchema) -> bool: ...
def fields(self) -> list[_FieldSchema]:
"""
Return list of all field schemas.
"""
def forward_fields(self) -> list[_FieldSchema]:
"""
Return list of forward-indexed fields.
"""
def get_field(self, field_name: str) -> _FieldSchema:
"""
Get field by name (const pointer), returns None if not found.
"""
def get_forward_field(self, field_name: str) -> _FieldSchema:
"""
Get forward field (used for filtering).
"""
def get_vector_field(self, field_name: str) -> _FieldSchema:
"""
Get vector field by name.
"""
def has_field(self, field_name: str) -> bool:
"""
Check if a field exists.
"""
def vector_fields(self) -> list[_FieldSchema]:
"""
Return list of vector fields.
"""
@property
def name(self) -> str: ...
class _FieldSchema:
__hash__: typing.ClassVar[None] = None
def __eq__(self, arg0: _FieldSchema) -> bool: ...
def __init__(
self,
name: str,
data_type: zvec._zvec.typing.DataType,
nullable: bool = False,
dimension: typing.SupportsInt = 0,
index_param: zvec._zvec.param.IndexParam = None,
) -> None: ...
def __ne__(self, arg0: _FieldSchema) -> bool: ...
@property
def data_type(self) -> zvec._zvec.typing.DataType: ...
@property
def dimension(self) -> int: ...
@property
def index_param(self) -> typing.Any: ...
@property
def index_type(self) -> zvec._zvec.typing.IndexType: ...
@property
def is_dense_vector(self) -> bool: ...
@property
def is_sparse_vector(self) -> bool: ...
@property
def name(self) -> str: ...
@property
def nullable(self) -> bool: ...
@@ -0,0 +1,215 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import json
from typing import Optional, Union
from zvec._zvec.schema import _CollectionSchema, _FieldSchema
from .field_schema import FieldSchema, VectorSchema
__all__ = [
"CollectionSchema",
]
class CollectionSchema:
"""Defines the structure of a collection in Zvec.
A collection schema specifies the name of the collection and its fields,
including both scalar fields (e.g., int, string) and vector fields.
Field names must be unique across both scalar and vector fields.
Args:
name (str): Name of the collection.
fields (Optional[Union[FieldSchema, list[FieldSchema]]], optional):
One or more scalar field definitions. Defaults to None.
vectors (Optional[Union[VectorSchema, list[VectorSchema]]], optional):
One or more vector field definitions. Defaults to None.
Raises:
TypeError: If `fields` or `vectors` are of unsupported types.
ValueError: If any field or vector name is duplicated.
Examples:
>>> from zvec import FieldSchema, VectorSchema, DataType, IndexType
>>> id_field = FieldSchema("id", DataType.INT64, is_primary=True)
>>> emb_field = VectorSchema("embedding", dim=128, data_type=DataType.VECTOR_FP32)
>>> schema = CollectionSchema(
... name="my_collection",
... fields=id_field,
... vectors=emb_field
... )
>>> print(schema.name)
my_collection
"""
def __init__(
self,
name: str,
fields: Optional[Union[FieldSchema, list[FieldSchema]]] = None,
vectors: Optional[Union[VectorSchema, list[VectorSchema]]] = None,
):
if name is None or not isinstance(name, str):
raise ValueError(
f"schema validate failed: collection name must be str, got {type(name).__name__}"
)
# handle fields
_fields_name: list[str] = []
_fields_list: list[_FieldSchema] = []
self._check_fields(fields, _fields_name, _fields_list)
self._check_vectors(vectors, _fields_name, _fields_list)
# init
self._cpp_obj = _CollectionSchema(
name=name,
fields=_fields_list,
)
def _check_fields(
self,
fields: Optional[Union[FieldSchema, list[FieldSchema]]],
_fields_name: list[str],
_fields_list: list[_FieldSchema],
) -> None:
field_items = []
if isinstance(fields, FieldSchema):
field_items = [fields]
elif isinstance(fields, list):
field_items = fields
elif fields is None:
field_items = []
else:
raise TypeError(
f"schema validate failed: invalid 'fields' type, expected FieldSchema or list[FieldSchema], "
f"got {type(fields).__name__}"
)
for idx, field in enumerate(field_items):
if not isinstance(field, FieldSchema):
raise TypeError(
f"schema validate failed: invalid field type in 'fields' list, expected FieldSchema, "
f"got {type(field).__name__} at index {idx}"
)
if field.name in _fields_name:
raise ValueError(
f"schema validate failed: duplicate field name '{field.name}': field names must be unique"
)
_fields_name.append(field.name)
_fields_list.append(field._get_object())
def _check_vectors(
self,
vectors: Optional[Union[VectorSchema, list[VectorSchema]]],
_fields_name: list[str],
_fields_list: list[_FieldSchema],
) -> None:
# handle vector
if isinstance(vectors, VectorSchema):
vectors_items = [vectors]
elif isinstance(vectors, list):
vectors_items = vectors
elif vectors is None:
vectors_items = []
else:
raise TypeError(
f"schema validate failed: invalid 'vectors' type, expected VectorSchema or list[VectorSchema], "
f"got {type(vectors).__name__}"
)
for idx, vector in enumerate(vectors_items):
if not isinstance(vector, VectorSchema):
raise TypeError(
f"schema validate failed: invalid vector type in 'vectors' list, expected VectorSchema, "
f"got {type(vector).__name__} at index {idx}"
)
if vector.name in _fields_name:
raise ValueError(
f"schema validate failed: duplicate vector name '{vector.name}', vector names must be unique "
f"(conflicts with existing field or vector)"
)
_fields_name.append(vector.name)
_fields_list.append(vector._get_object())
@classmethod
def _from_core(cls, core_collection_schema: _CollectionSchema):
inst = cls.__new__(cls)
if not core_collection_schema:
raise ValueError("schema validate failed: schema is null")
inst._cpp_obj = core_collection_schema
return inst
@property
def name(self) -> str:
"""str: The name of the collection."""
return self._cpp_obj.name
def field(self, name: str) -> Optional[FieldSchema]:
"""Retrieve a scalar field by name.
Args:
name (str): Name of the field.
Returns:
Optional[FieldSchema]: The field if found, otherwise None.
"""
_field = self._cpp_obj.get_forward_field(name)
return FieldSchema._from_core(_field) if _field else None
def vector(self, name: str) -> Optional[VectorSchema]:
"""Retrieve a vector field by name.
Args:
name (str): Name of the vector field.
Returns:
Optional[VectorSchema]: The vector field if found, otherwise None.
"""
_field = self._cpp_obj.get_vector_field(name)
return VectorSchema._from_core(_field) if _field else None
@property
def fields(self) -> list[FieldSchema]:
"""list[FieldSchema]: All scalar (non-vector) fields in the schema."""
_fields = self._cpp_obj.forward_fields()
return [FieldSchema._from_core(_field) for _field in _fields]
@property
def vectors(self) -> list[VectorSchema]:
"""list[VectorSchema]: All vector fields in the schema."""
_vectors = self._cpp_obj.vector_fields()
return [VectorSchema._from_core(_vector) for _vector in _vectors]
def _get_object(self) -> _CollectionSchema:
return self._cpp_obj
def __repr__(self) -> str:
try:
schema = {
"name": self.name,
"fields": {field.name: field.__dict__() for field in self.fields},
"vectors": {vector.name: vector.__dict__() for vector in self.vectors},
}
return json.dumps(schema, indent=2, ensure_ascii=False)
except Exception as e:
return f"<CollectionSchema error during repr: {e}>"
def __str__(self) -> str:
return self.__repr__()
+310
View File
@@ -0,0 +1,310 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import json
from typing import Any, Optional, Union
from zvec._zvec.schema import _FieldSchema
from zvec.model.param import (
FlatIndexParam,
FtsIndexParam,
HnswIndexParam,
HnswRabitqIndexParam,
InvertIndexParam,
IVFIndexParam,
)
from zvec.typing import DataType
__all__ = [
"FieldSchema",
"VectorSchema",
]
SUPPORT_VECTOR_DATA_TYPE = [
DataType.VECTOR_FP16,
DataType.VECTOR_FP32,
DataType.VECTOR_FP64,
DataType.VECTOR_INT8,
DataType.SPARSE_VECTOR_FP16,
DataType.SPARSE_VECTOR_FP32,
]
SUPPORT_SCALAR_DATA_TYPE = [
DataType.INT32,
DataType.INT64,
DataType.UINT32,
DataType.UINT64,
DataType.FLOAT,
DataType.DOUBLE,
DataType.STRING,
DataType.BOOL,
DataType.ARRAY_INT32,
DataType.ARRAY_INT64,
DataType.ARRAY_UINT32,
DataType.ARRAY_UINT64,
DataType.ARRAY_FLOAT,
DataType.ARRAY_DOUBLE,
DataType.ARRAY_STRING,
DataType.ARRAY_BOOL,
]
class FieldSchema:
"""Represents a scalar (non-vector) field in a collection schema.
A `FieldSchema` defines the name, data type, nullability, and optional
inverted index configuration for a regular field (e.g., ID, timestamp, category).
Args:
name (str): Name of the field. Must be unique within the collection.
data_type (DataType): Data type of the field (e.g., INT64, STRING).
nullable (bool, optional): Whether the field can contain null values.
Defaults to False.
index_param (Optional[Union[InvertIndexParam, FtsIndexParam]], optional):
Index parameters for this field. Use ``InvertIndexParam`` for scalar
inverted indexing, or ``FtsIndexParam`` for full-text search indexing
on STRING fields. Defaults to None.
Examples:
>>> from zvec.typing import DataType
>>> from zvec.model.param import InvertIndexParam, FtsIndexParam
>>> id_field = FieldSchema(
... name="id",
... data_type=DataType.INT64,
... nullable=False,
... index_param=InvertIndexParam(enable_range_optimization=True)
... )
>>> content_field = FieldSchema(
... name="content",
... data_type=DataType.STRING,
... nullable=False,
... index_param=FtsIndexParam(tokenizer_name="standard")
... )
"""
def __init__(
self,
name: str,
data_type: DataType,
nullable: bool = False,
index_param: Optional[Union[InvertIndexParam, FtsIndexParam]] = None,
):
if name is None or not isinstance(name, str):
raise ValueError(
f"schema validate failed: field name must be str, got {type(name).__name__}"
)
if data_type not in SUPPORT_SCALAR_DATA_TYPE:
raise ValueError(
f"schema validate failed: scalar_field's data_type must be one of "
f"{', '.join(str(dt) for dt in SUPPORT_SCALAR_DATA_TYPE)}, "
f"but field[{name}]'s data_type is {data_type}"
)
self._cpp_obj = _FieldSchema(
name=name,
data_type=data_type,
dimension=0,
nullable=nullable,
index_param=index_param,
)
@classmethod
def _from_core(cls, core_field_schema: _FieldSchema):
if core_field_schema is None:
raise ValueError("schema validate failed: field schema is None")
inst = cls.__new__(cls)
inst._cpp_obj = core_field_schema
return inst
def _get_object(self) -> _FieldSchema:
return self._cpp_obj
@property
def name(self) -> str:
"""str: The name of the field."""
return self._cpp_obj.name
@property
def data_type(self) -> DataType:
"""DataType: The data type of the field (e.g., INT64, STRING)."""
return self._cpp_obj.data_type
@property
def nullable(self) -> bool:
"""bool: Whether the field allows null values."""
return self._cpp_obj.nullable
@property
def index_param(self) -> Optional[Union[InvertIndexParam, FtsIndexParam]]:
"""Optional[Union[InvertIndexParam, FtsIndexParam]]: Index configuration, if any."""
return self._cpp_obj.index_param
def __dict__(self) -> dict[str, Any]:
return {
"name": self.name,
"data_type": (
self.data_type.name
if hasattr(self.data_type, "name")
else str(self.data_type)
),
"nullable": self.nullable,
"index_param": (
self.index_param.to_dict() if self.index_param is not None else None
),
}
def __repr__(self) -> str:
try:
schema = self.__dict__()
return json.dumps(schema, indent=2, ensure_ascii=False)
except Exception as e:
return f"<FieldSchema error during repr: {e}>"
def __str__(self) -> str:
return self.__repr__()
def __eq__(self, other: object) -> bool:
if not isinstance(other, FieldSchema):
return False
return self._cpp_obj == other._cpp_obj
def __hash__(self) -> int:
return hash((self.name, self.data_type, self.nullable))
class VectorSchema:
"""Represents a vector field in a collection schema.
A `VectorSchema` defines the name, data type, dimensionality, and index
configuration for a vector field used in similarity search.
Args:
name (str): Name of the vector field. Must be unique within the collection.
data_type (DataType): Vector data type (e.g., VECTOR_FP32, VECTOR_INT8).
dimension (int, optional): Dimensionality of the vector. Must be > 0 for dense vectors;
may be `None` for sparse vectors.
index_param (Union[HnswIndexParam, IVFIndexParam, FlatIndexParam], optional):
Index configuration for this vector field. Defaults to
``HnswIndexParam()``.
Examples:
>>> from zvec.typing import DataType
>>> from zvec.model.param import HnswIndexParam
>>> emb_field = VectorSchema(
... name="embedding",
... data_type=DataType.VECTOR_FP32,
... dimension=128,
... index_param=HnswIndexParam(ef_construction=200, m=16)
... )
"""
def __init__(
self,
name: str,
data_type: DataType,
dimension: Optional[int] = 0,
index_param: Optional[
Union[HnswIndexParam, HnswRabitqIndexParam, FlatIndexParam, IVFIndexParam]
] = None,
):
if name is None or not isinstance(name, str):
raise ValueError(
f"schema validate failed: field name must be str, got {type(name).__name__}"
)
if not isinstance(dimension, int) or dimension < 0:
raise ValueError("schema validate failed: vector's dimension must be >= 0")
if data_type not in SUPPORT_VECTOR_DATA_TYPE:
raise ValueError(
f"schema validate failed: vector's data_type must be one of "
f"{', '.join(str(dt) for dt in SUPPORT_VECTOR_DATA_TYPE)}, "
f"but field[{name}]'s data_type is {data_type}"
)
if index_param is None:
index_param = FlatIndexParam()
self._cpp_obj = _FieldSchema(
name=name,
data_type=data_type,
dimension=dimension,
nullable=False,
index_param=index_param,
)
@classmethod
def _from_core(cls, core_field_schema: _FieldSchema):
inst = cls.__new__(cls)
inst._cpp_obj = core_field_schema
return inst
def _get_object(self) -> _FieldSchema:
return self._cpp_obj
@property
def name(self) -> str:
"""str: The name of the vector field."""
return self._cpp_obj.name
@property
def data_type(self) -> DataType:
"""DataType: The vector data type (e.g., VECTOR_FP32)."""
return self._cpp_obj.data_type
@property
def dimension(self) -> int:
"""int: The dimensionality of the vector."""
return self._cpp_obj.dimension
@property
def index_param(
self,
) -> Union[HnswIndexParam, HnswRabitqIndexParam, IVFIndexParam, FlatIndexParam]:
"""Union[HnswIndexParam, HnswRabitqIndexParam, IVFIndexParam, FlatIndexParam]: Index configuration for the vector."""
return self._cpp_obj.index_param
def __dict__(self) -> dict[str, Any]:
return {
"name": self.name,
"data_type": (
self.data_type.name
if hasattr(self.data_type, "name")
else str(self.data_type)
),
"dimension": self.dimension,
"index_param": (
self.index_param.to_dict() if self.index_param is not None else None
),
}
def __repr__(self) -> str:
try:
schema = self.__dict__()
return json.dumps(schema, indent=2, ensure_ascii=False)
except Exception as e:
return f"<FieldSchema error during repr: {e}>"
def __str__(self) -> str:
return self.__repr__()
def __eq__(self, other: object) -> bool:
if not isinstance(other, VectorSchema):
return False
return self._cpp_obj == other._cpp_obj
def __hash__(self) -> int:
return hash((self.name, self.data_type, self.dimension))