chore: import upstream snapshot with attribution
Build and test / Build and test AMD64 Ubuntu 22.04 (push) Failing after 0s
Publish Builder / amazonlinux2023 (push) Failing after 1s
Build and test / UT for Go (push) Has been skipped
Publish KRTE Images / KRTE (push) Failing after 1s
Build and test / Integration Test (push) Has been skipped
Build and test / Upload Code Coverage (push) Has been skipped
Publish Builder / rockylinux9 (push) Failing after 1s
Publish Builder / ubuntu22.04 (push) Failing after 0s
Publish Builder / ubuntu24.04 (push) Failing after 0s
Publish Gpu Builder / publish-gpu-builder (push) Failing after 1s
Publish Test Images / PyTest (push) Failing after 0s
Build and test / UT for Cpp (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:31:17 +08:00
commit 498b235461
5446 changed files with 2748612 additions and 0 deletions
@@ -0,0 +1,267 @@
import asyncio
import sys
from typing import Optional, List, Union, Dict
from pymilvus import (
AsyncMilvusClient,
AnnSearchRequest,
RRFRanker,
)
from pymilvus.orm.types import CONSISTENCY_STRONG
from pymilvus.orm.collection import CollectionSchema
from check.func_check import ResponseChecker
from utils.api_request import api_request, logger_interceptor
class AsyncMilvusClientWrapper:
async_milvus_client = None
def __init__(self, active_trace=False):
self.active_trace = active_trace
def init_async_client(self, uri: str = "http://localhost:19530",
user: str = "",
password: str = "",
db_name: str = "",
token: str = "",
timeout: Optional[float] = None,
active_trace=False,
check_task=None, check_items=None,
**kwargs):
self.active_trace = active_trace
""" In order to distinguish the same name of collection """
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([AsyncMilvusClient, uri, user, password, db_name, token,
timeout], **kwargs)
self.async_milvus_client = res if is_succ else None
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, **kwargs).run()
return res, check_result
@logger_interceptor()
async def list_collections(self, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.list_collections(timeout, **kwargs)
@logger_interceptor()
async def has_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.has_collection(collection_name, timeout, **kwargs)
@logger_interceptor()
async def has_partition(self, collection_name: str, partition_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.has_partition(collection_name, partition_name, timeout, **kwargs)
@logger_interceptor()
async def describe_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.describe_collection(collection_name, timeout, **kwargs)
@logger_interceptor()
async def list_partitions(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.list_partitions(collection_name, timeout, **kwargs)
@logger_interceptor()
async def get_collection_stats(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.get_collection_stats(collection_name, timeout, **kwargs)
@logger_interceptor()
async def flush(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.flush(collection_name, timeout, **kwargs)
@logger_interceptor()
async def get_load_state(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.get_load_state(collection_name, timeout, **kwargs)
@logger_interceptor()
async def describe_index(self, collection_name: str, index_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.describe_index(collection_name, index_name, timeout, **kwargs)
@logger_interceptor()
async def create_database(self, db_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.create_database(db_name, timeout, **kwargs)
@logger_interceptor()
async def drop_database(self, db_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.drop_database(db_name, timeout, **kwargs)
@logger_interceptor()
async def list_databases(self, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.list_databases(timeout, **kwargs)
@logger_interceptor()
async def list_indexes(self, collection_name: str, field_name: str = "", **kwargs):
return await self.async_milvus_client.list_indexes(collection_name, field_name, **kwargs)
@logger_interceptor()
async def create_collection(self,
collection_name: str,
dimension: Optional[int] = None,
primary_field_name: str = "id", # default is "id"
id_type: str = "int", # or "string",
vector_field_name: str = "vector", # default is "vector"
metric_type: str = "COSINE",
auto_id: bool = False,
timeout: Optional[float] = None,
schema: Optional[CollectionSchema] = None,
index_params=None,
**kwargs):
kwargs["consistency_level"] = kwargs.get("consistency_level", CONSISTENCY_STRONG)
return await self.async_milvus_client.create_collection(collection_name, dimension,
primary_field_name,
id_type, vector_field_name, metric_type,
auto_id,
timeout, schema, index_params, **kwargs)
@logger_interceptor()
async def drop_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.drop_collection(collection_name, timeout, **kwargs)
@logger_interceptor()
async def load_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.load_collection(collection_name, timeout, **kwargs)
@logger_interceptor()
async def release_collection(self, collection_name, timeout=None, **kwargs):
return await self.async_milvus_client.release_collection(collection_name, timeout, **kwargs)
@logger_interceptor()
async def truncate_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.truncate_collection(collection_name, timeout, **kwargs)
@logger_interceptor()
async def list_persistent_segments(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
return await self.async_milvus_client.list_persistent_segments(collection_name, timeout, **kwargs)
@logger_interceptor()
async def create_index(self, collection_name: str, index_params, timeout: Optional[float] = None,
**kwargs):
return await self.async_milvus_client.create_index(collection_name, index_params, timeout, **kwargs)
@logger_interceptor()
async def drop_index(self, collection_name, index_name, timeout=None, **kwargs):
return await self.async_milvus_client.drop_index(collection_name, index_name, timeout, **kwargs)
# @logger_interceptor()
# async def list_indexes(self, collection_name, field_name="", timeout=None, **kwargs):
# return await self.async_milvus_client.list_indexes(collection_name, field_name, timeout, **kwargs)
@logger_interceptor()
async def create_partition(self, collection_name, partition_name, timeout=None, **kwargs):
return await self.async_milvus_client.create_partition(collection_name, partition_name, timeout, **kwargs)
@logger_interceptor()
async def drop_partition(self, collection_name, partition_name, timeout=None, **kwargs):
return await self.async_milvus_client.drop_partition(collection_name, partition_name, timeout, **kwargs)
@logger_interceptor()
async def load_partitions(self, collection_name, partition_names, timeout=None, **kwargs):
return await self.async_milvus_client.load_partitions(collection_name, partition_names, timeout, **kwargs)
@logger_interceptor()
async def release_partitions(self, collection_name, partition_names, timeout=None, **kwargs):
return await self.async_milvus_client.release_partitions(collection_name, partition_names, timeout, **kwargs)
@logger_interceptor()
async def insert(self,
collection_name: str,
data: Union[Dict, List[Dict]],
timeout: Optional[float] = None,
partition_name: Optional[str] = "",
**kwargs):
return await self.async_milvus_client.insert(collection_name, data, timeout, partition_name, **kwargs)
@logger_interceptor()
async def upsert(self,
collection_name: str,
data: Union[Dict, List[Dict]],
timeout: Optional[float] = None,
partition_name: Optional[str] = "",
**kwargs):
return await self.async_milvus_client.upsert(collection_name, data, timeout, partition_name, **kwargs)
@logger_interceptor()
async def search(self,
collection_name: str,
data: Union[List[list], list],
filter: str = "",
limit: int = 10,
output_fields: Optional[List[str]] = None,
search_params: Optional[dict] = None,
timeout: Optional[float] = None,
partition_names: Optional[List[str]] = None,
anns_field: Optional[str] = None,
**kwargs):
return await self.async_milvus_client.search(collection_name, data,
filter,
limit, output_fields, search_params,
timeout,
partition_names, anns_field, **kwargs)
@logger_interceptor()
async def hybrid_search(self,
collection_name: str,
reqs: List[AnnSearchRequest],
ranker: RRFRanker,
limit: int = 10,
output_fields: Optional[List[str]] = None,
timeout: Optional[float] = None,
partition_names: Optional[List[str]] = None,
**kwargs):
return await self.async_milvus_client.hybrid_search(collection_name, reqs,
ranker,
limit, output_fields,
timeout, partition_names, **kwargs)
@logger_interceptor()
async def query(self,
collection_name: str,
filter: str = "",
output_fields: Optional[List[str]] = None,
timeout: Optional[float] = None,
ids: Optional[Union[List, str, int]] = None,
partition_names: Optional[List[str]] = None,
**kwargs):
return await self.async_milvus_client.query(collection_name, filter,
output_fields, timeout,
ids, partition_names,
**kwargs)
@logger_interceptor()
async def get(self,
collection_name: str,
ids: Union[list, str, int],
output_fields: Optional[List[str]] = None,
timeout: Optional[float] = None,
partition_names: Optional[List[str]] = None,
**kwargs):
return await self.async_milvus_client.get(collection_name, ids,
output_fields, timeout,
partition_names,
**kwargs)
@logger_interceptor()
async def delete(self,
collection_name: str,
ids: Optional[Union[list, str, int]] = None,
timeout: Optional[float] = None,
filter: Optional[str] = None,
partition_name: Optional[str] = None,
**kwargs):
return await self.async_milvus_client.delete(collection_name, ids,
timeout, filter,
partition_name,
**kwargs)
@classmethod
def create_schema(cls, **kwargs):
kwargs["check_fields"] = False # do not check fields for now
return CollectionSchema([], **kwargs)
@classmethod
def prepare_index_params(cls, field_name: str = "", **kwargs):
res, check = api_request([AsyncMilvusClient.prepare_index_params, field_name], **kwargs)
return res, check
@logger_interceptor()
async def close(self, **kwargs):
return await self.async_milvus_client.close(**kwargs)
+649
View File
@@ -0,0 +1,649 @@
import sys
from base.database_wrapper import ApiDatabaseWrapper
from pymilvus import DefaultConfig
sys.path.append("..")
import pymilvus
from base.async_milvus_client_wrapper import AsyncMilvusClientWrapper
from base.collection_wrapper import ApiCollectionWrapper
from base.connections_wrapper import ApiConnectionsWrapper
from base.index_wrapper import ApiIndexWrapper
from base.partition_wrapper import ApiPartitionWrapper
from base.schema_wrapper import ApiCollectionSchemaWrapper, ApiFieldSchemaWrapper
from base.utility_wrapper import ApiUtilityWrapper
from common import common_func as cf
from common import common_type as ct
from common.common_params import IndexPrams
from pymilvus import DataType, MilvusClient, ResourceGroupInfo, utility
from utils.util_log import test_log as log
class Base:
"""Initialize class object"""
connection_wrap = None
collection_wrap = None
partition_wrap = None
index_wrap = None
utility_wrap = None
collection_schema_wrap = None
field_schema_wrap = None
database_wrap = None
tear_down_collection_names = []
tear_down_role_names = []
tear_down_user_names = []
resource_group_list = []
async_milvus_client_wrap = None
skip_connection = False
skip_global_role_cleanup = False
def setup_class(self):
log.info("[setup_class] Start setup class...")
def teardown_class(self):
log.info("[teardown_class] Start teardown class...")
def setup_method(self, method):
log.info(("*" * 35) + " setup " + ("*" * 35))
log.info(f"pymilvus version: {pymilvus.__version__}")
log.info(f"[setup_method] Start setup test case {method.__name__}.")
self._setup_objects()
def _setup_objects(self):
self.connection_wrap = ApiConnectionsWrapper()
self.utility_wrap = ApiUtilityWrapper()
self.collection_wrap = ApiCollectionWrapper()
self.partition_wrap = ApiPartitionWrapper()
self.index_wrap = ApiIndexWrapper()
self.collection_schema_wrap = ApiCollectionSchemaWrapper()
self.field_schema_wrap = ApiFieldSchemaWrapper()
self.database_wrap = ApiDatabaseWrapper()
self.async_milvus_client_wrap = AsyncMilvusClientWrapper()
def teardown_method(self, method):
log.info(("*" * 35) + " teardown " + ("*" * 35))
log.info(f"[teardown_method] Start teardown test case {method.__name__}...")
self._teardown_objects()
def _teardown_objects(self):
# Prioritize uri and token for connection
if cf.param_info.param_uri:
uri = cf.param_info.param_uri
else:
uri = "http://" + cf.param_info.param_host + ":" + str(cf.param_info.param_port)
if cf.param_info.param_token:
token = cf.param_info.param_token
else:
token = (
f"{cf.param_info.param_user}:{cf.param_info.param_password}"
if cf.param_info.param_user and cf.param_info.param_password
else None
)
try:
""" Drop collection before disconnect """
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
if token:
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri, token=token)
else:
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri)
if self.collection_wrap.collection is not None:
if self.collection_wrap.collection.name.startswith("alias"):
log.info(f"collection {self.collection_wrap.collection.name} is alias, skip drop operation")
else:
self.collection_wrap.drop(check_task=ct.CheckTasks.check_nothing)
collection_list = self.utility_wrap.list_collections()[0]
for collection_name in self.tear_down_collection_names:
if collection_name is not None and collection_name in collection_list:
alias_list = self.utility_wrap.list_aliases(collection_name)[0]
if alias_list:
for alias in alias_list:
self.utility_wrap.drop_alias(alias)
self.utility_wrap.drop_collection(collection_name)
""" Clean up the rgs before disconnect """
rgs_list = self.utility_wrap.list_resource_groups()[0]
for rg_name in self.resource_group_list:
if rg_name is not None and rg_name in rgs_list:
rg = self.utility_wrap.describe_resource_group(
name=rg_name, check_task=ct.CheckTasks.check_nothing
)[0]
if isinstance(rg, ResourceGroupInfo):
if rg.num_available_node > 0:
self.utility_wrap.transfer_node(
source=rg_name, target=ct.default_resource_group_name, num_node=rg.num_available_node
)
self.utility_wrap.drop_resource_group(rg_name, check_task=ct.CheckTasks.check_nothing)
except Exception as e:
log.debug(str(e))
if not self.skip_global_role_cleanup:
try:
""" Drop roles before disconnect """
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
if token:
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri, token=token)
else:
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri)
role_list = self.utility_wrap.list_roles(False)[0]
for role in role_list.groups:
role_name = role.role_name
if role_name not in ["admin", "public"]:
each_role = self.utility_wrap.init_role(name=role_name)[0]
each_role.drop()
except Exception as e:
log.debug(str(e))
try:
""" Delete connection and reset configuration"""
res = self.connection_wrap.list_connections()
for i in res[0]:
self.connection_wrap.remove_connection(i[0])
# because the connection is in singleton mode, it needs to be restored to the original state after teardown
self.connection_wrap.add_connection(
default={"host": DefaultConfig.DEFAULT_HOST, "port": DefaultConfig.DEFAULT_PORT}
)
except Exception as e:
log.debug(str(e))
class TestcaseBase(Base):
"""
Additional methods;
Public methods that can be used for test cases.
"""
client = None
def _connect(self, enable_milvus_client_api=False):
"""Add a connection and create the connect"""
if self.skip_connection:
return None
# Prioritize uri and token for connection
if cf.param_info.param_uri:
uri = cf.param_info.param_uri
else:
uri = "http://" + cf.param_info.param_host + ":" + str(cf.param_info.param_port)
if cf.param_info.param_token:
token = cf.param_info.param_token
else:
token = (
f"{cf.param_info.param_user}:{cf.param_info.param_password}"
if cf.param_info.param_user and cf.param_info.param_password
else None
)
if enable_milvus_client_api:
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri, token=token)
res, is_succ = self.connection_wrap.MilvusClient(uri=uri, token=token)
self.client = MilvusClient(uri=uri, token=token)
else:
if token:
res, is_succ = self.connection_wrap.connect(
alias=DefaultConfig.DEFAULT_USING, uri=uri, token=token, secure=cf.param_info.param_secure
)
else:
res, is_succ = self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri)
self.client = MilvusClient(uri=uri, token=token)
server_version = utility.get_server_version()
log.info(f"server version: {server_version}")
return res
def get_tokens_by_analyzer(self, text, analyzer_params):
if cf.param_info.param_uri:
uri = cf.param_info.param_uri
else:
uri = "http://" + cf.param_info.param_host + ":" + str(cf.param_info.param_port)
client = MilvusClient(uri=uri, token=cf.param_info.param_token)
res = client.run_analyzer(text, analyzer_params, with_detail=True, with_hash=True)
tokens = [r["token"] for r in res.tokens]
return tokens
# def init_async_milvus_client(self):
# uri = cf.param_info.param_uri or f"http://{cf.param_info.param_host}:{cf.param_info.param_port}"
# kwargs = {
# "uri": uri,
# "user": cf.param_info.param_user,
# "password": cf.param_info.param_password,
# "token": cf.param_info.param_token,
# }
# self.async_milvus_client_wrap.init_async_client(**kwargs)
def init_collection_wrap(
self,
name=None,
schema=None,
check_task=None,
check_items=None,
enable_dynamic_field=False,
with_json=True,
**kwargs,
):
name = cf.gen_collection_name_by_testcase_name(2) if name is None else name
schema = (
cf.gen_default_collection_schema(enable_dynamic_field=enable_dynamic_field, with_json=with_json)
if schema is None
else schema
)
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
self._connect()
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=name, schema=schema, check_task=check_task, check_items=check_items, **kwargs)
self.tear_down_collection_names.append(name)
return collection_w
def init_multi_fields_collection_wrap(self, name=cf.gen_unique_str()):
vec_fields = [cf.gen_float_vec_field(ct.another_float_vec_field_name)]
schema = cf.gen_schema_multi_vector_fields(vec_fields)
collection_w = self.init_collection_wrap(name=name, schema=schema)
df = cf.gen_dataframe_multi_vec_fields(vec_fields=vec_fields)
collection_w.insert(df)
assert collection_w.num_entities == ct.default_nb
return collection_w, df
def init_partition_wrap(
self, collection_wrap=None, name=None, description=None, check_task=None, check_items=None, **kwargs
):
name = cf.gen_unique_str("partition_") if name is None else name
description = cf.gen_unique_str("partition_des_") if description is None else description
collection_wrap = self.init_collection_wrap() if collection_wrap is None else collection_wrap
partition_wrap = ApiPartitionWrapper()
partition_wrap.init_partition(
collection_wrap.collection, name, description, check_task=check_task, check_items=check_items, **kwargs
)
return partition_wrap
def insert_data_general(
self,
prefix="test",
insert_data=False,
nb=ct.default_nb,
partition_num=0,
is_binary=False,
is_all_data_type=False,
auto_id=False,
dim=ct.default_dim,
primary_field=ct.default_int64_field_name,
is_flush=True,
name=None,
enable_dynamic_field=False,
with_json=True,
**kwargs,
):
""" """
self._connect()
collection_name = cf.gen_unique_str(prefix)
if name is not None:
collection_name = name
vectors = []
binary_raw_vectors = []
insert_ids = []
time_stamp = 0
# 1 create collection
default_schema = cf.gen_default_collection_schema(
auto_id=auto_id,
dim=dim,
primary_field=primary_field,
enable_dynamic_field=enable_dynamic_field,
with_json=with_json,
)
if is_binary:
default_schema = cf.gen_default_binary_collection_schema(
auto_id=auto_id, dim=dim, primary_field=primary_field
)
if is_all_data_type:
default_schema = cf.gen_collection_schema_all_datatype(
auto_id=auto_id,
dim=dim,
primary_field=primary_field,
enable_dynamic_field=enable_dynamic_field,
with_json=with_json,
)
log.info("insert_data_general: collection creation")
collection_w = self.init_collection_wrap(name=collection_name, schema=default_schema, **kwargs)
pre_entities = collection_w.num_entities
if insert_data:
collection_w, vectors, binary_raw_vectors, insert_ids, time_stamp = cf.insert_data(
collection_w,
nb,
is_binary,
is_all_data_type,
auto_id=auto_id,
dim=dim,
enable_dynamic_field=enable_dynamic_field,
with_json=with_json,
)
if is_flush:
collection_w.flush()
assert collection_w.num_entities == nb + pre_entities
return collection_w, vectors, binary_raw_vectors, insert_ids, time_stamp
def init_collection_general(
self,
prefix="test",
insert_data=False,
nb=ct.default_nb,
partition_num=0,
is_binary=False,
is_all_data_type=False,
auto_id=False,
dim=ct.default_dim,
is_index=True,
primary_field=ct.default_int64_field_name,
is_flush=True,
name=None,
enable_dynamic_field=False,
with_json=True,
random_primary_key=False,
multiple_dim_array=[],
is_partition_key=None,
vector_data_type=DataType.FLOAT_VECTOR,
nullable_fields={},
default_value_fields={},
language=None,
**kwargs,
):
"""
target: create specified collections
method: 1. create collections (binary/non-binary, default/all data type, auto_id or not)
2. create partitions if specified
3. insert specified (binary/non-binary, default/all data type) data
into each partition if any
4. not load if specifying is_index as True
5. enable insert null data: nullable_fields = {"nullable_fields_name": null data percent}
6. enable insert default value: default_value_fields = {"default_fields_name": default value}
expected: return collection and raw data, insert ids
"""
log.info("Test case of search interface: initialize before test case")
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
self._connect()
collection_name = cf.gen_collection_name_by_testcase_name(2)
if name is not None:
collection_name = name
if not isinstance(nullable_fields, dict):
log.error("nullable_fields should a dict like {'nullable_fields_name': null data percent}")
assert False
if not isinstance(default_value_fields, dict):
log.error("default_value_fields should a dict like {'default_fields_name': default value}")
assert False
vectors = []
binary_raw_vectors = []
insert_ids = []
time_stamp = 0
# 1 create collection
default_schema = cf.gen_default_collection_schema(
auto_id=auto_id,
dim=dim,
primary_field=primary_field,
enable_dynamic_field=enable_dynamic_field,
with_json=with_json,
multiple_dim_array=multiple_dim_array,
is_partition_key=is_partition_key,
vector_data_type=vector_data_type,
nullable_fields=nullable_fields,
default_value_fields=default_value_fields,
)
if is_binary:
default_schema = cf.gen_default_binary_collection_schema(
auto_id=auto_id,
dim=dim,
primary_field=primary_field,
nullable_fields=nullable_fields,
default_value_fields=default_value_fields,
)
if vector_data_type == DataType.SPARSE_FLOAT_VECTOR:
default_schema = cf.gen_default_sparse_schema(
auto_id=auto_id,
primary_field=primary_field,
enable_dynamic_field=enable_dynamic_field,
with_json=with_json,
multiple_dim_array=multiple_dim_array,
nullable_fields=nullable_fields,
default_value_fields=default_value_fields,
)
if is_all_data_type:
default_schema = cf.gen_collection_schema_all_datatype(
auto_id=auto_id,
dim=dim,
primary_field=primary_field,
enable_dynamic_field=enable_dynamic_field,
with_json=with_json,
multiple_dim_array=multiple_dim_array,
nullable_fields=nullable_fields,
default_value_fields=default_value_fields,
)
log.info("init_collection_general: collection creation")
collection_w = self.init_collection_wrap(name=collection_name, schema=default_schema, **kwargs)
vector_name_list = cf.extract_vector_field_name_list(collection_w)
# 2 add extra partitions if specified (default is 1 partition named "_default")
if partition_num > 0:
cf.gen_partitions(collection_w, partition_num)
# 3 insert data if specified
if insert_data:
collection_w, vectors, binary_raw_vectors, insert_ids, time_stamp = cf.insert_data(
collection_w,
nb,
is_binary,
is_all_data_type,
auto_id=auto_id,
dim=dim,
enable_dynamic_field=enable_dynamic_field,
with_json=with_json,
random_primary_key=random_primary_key,
multiple_dim_array=multiple_dim_array,
primary_field=primary_field,
vector_data_type=vector_data_type,
nullable_fields=nullable_fields,
language=language,
)
if is_flush:
assert collection_w.is_empty is False
assert collection_w.num_entities == nb
# 4 create default index if specified
if is_index:
# This condition will be removed after auto index feature
if is_binary:
collection_w.create_index(ct.default_binary_vec_field_name, ct.default_bin_flat_index)
elif vector_data_type == DataType.SPARSE_FLOAT_VECTOR:
for vector_name in vector_name_list:
collection_w.create_index(vector_name, ct.default_sparse_inverted_index)
else:
if len(multiple_dim_array) == 0 or is_all_data_type is False:
vector_name_list.append(ct.default_float_vec_field_name)
for vector_name in vector_name_list:
# Unlike dense vectors, sparse vectors cannot create flat index.
if DataType.SPARSE_FLOAT_VECTOR.name in vector_name:
collection_w.create_index(vector_name, ct.default_sparse_inverted_index)
elif vector_data_type == DataType.INT8_VECTOR:
collection_w.create_index(vector_name, ct.int8_vector_index)
else:
collection_w.create_index(vector_name, ct.default_flat_index)
collection_w.load()
return collection_w, vectors, binary_raw_vectors, insert_ids, time_stamp
def insert_entities_into_two_partitions_in_half(self, half, prefix="query"):
"""
insert default entities into two partitions(partition_w and _default) in half(int64 and float fields values)
:param half: half of nb
:return: collection wrap and partition wrap
"""
self._connect()
collection_name = cf.gen_collection_name_by_testcase_name(2)
collection_w = self.init_collection_wrap(name=collection_name)
partition_w = self.init_partition_wrap(collection_wrap=collection_w)
# insert [0, half) into partition_w
df_partition = cf.gen_default_dataframe_data(nb=half, start=0)
partition_w.insert(df_partition)
# insert [half, nb) into _default
df_default = cf.gen_default_dataframe_data(nb=half, start=half)
collection_w.insert(df_default)
# flush
collection_w.num_entities
collection_w.create_index(ct.default_float_vec_field_name, index_params=ct.default_flat_index)
collection_w.load(partition_names=[partition_w.name, "_default"])
return collection_w, partition_w, df_partition, df_default
def collection_insert_multi_segments_one_shard(
self, collection_prefix, num_of_segment=2, nb_of_segment=1, is_dup=True
):
"""
init collection with one shard, insert data into two segments on one shard (they can be merged)
:param collection_prefix: collection name prefix
:param num_of_segment: number of segments
:param nb_of_segment: number of entities per segment
:param is_dup: whether the primary keys of each segment is duplicated
:return: collection wrap and partition wrap
"""
collection_name = cf.gen_collection_name_by_testcase_name(2)
collection_w = self.init_collection_wrap(name=collection_name, shards_num=1)
for i in range(num_of_segment):
start = 0 if is_dup else i * nb_of_segment
df = cf.gen_default_dataframe_data(nb_of_segment, start=start)
collection_w.insert(df)
assert collection_w.num_entities == nb_of_segment * (i + 1)
return collection_w
def init_resource_group(self, name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
self._connect()
utility_w = ApiUtilityWrapper()
res, check_result = utility_w.create_resource_group(
name=name, using=using, timeout=timeout, check_task=check_task, check_items=check_items, **kwargs
)
if res is None and check_result:
self.resource_group_list.append(name)
return res, check_result
def init_user_with_privilege(self, privilege_object, object_name, privilege, db_name="default"):
"""
init a user and role, grant privilege to the role with the db, then bind the role to the user
:param privilege_object: privilege object: Global, Collection, User
:type privilege_object: str
:param object_name: privilege object name
:type object_name: str
:param privilege: privilege
:type privilege: str
:param db_name: database name
:type db_name: str
:return: user name, user pwd, role name
:rtype: str, str, str
"""
tmp_user = cf.gen_unique_str("user")
tmp_pwd = cf.gen_unique_str("pwd")
tmp_role = cf.gen_unique_str("role")
# create user
self.utility_wrap.create_user(tmp_user, tmp_pwd)
# create role
self.utility_wrap.init_role(tmp_role)
self.utility_wrap.create_role()
# grant privilege to the role
self.utility_wrap.role_grant(
object=privilege_object, object_name=object_name, privilege=privilege, db_name=db_name
)
# bind the role to the user
self.utility_wrap.role_add_user(tmp_user)
return tmp_user, tmp_pwd, tmp_role
def build_multi_index(self, index_params: dict[str, IndexPrams], collection_obj: ApiCollectionWrapper = None):
collection_obj = collection_obj or self.collection_wrap
for k, v in index_params.items():
collection_obj.create_index(field_name=k, index_params=v.to_dict, index_name=k)
log.info(f"[TestcaseBase] Build all indexes done: {list(index_params.keys())}")
return collection_obj
def drop_multi_index(
self, index_names: list[str], collection_obj: ApiCollectionWrapper = None, check_task=None, check_items=None
):
collection_obj = collection_obj or self.collection_wrap
for n in index_names:
collection_obj.drop_index(index_name=n, check_task=check_task, check_items=check_items)
log.info(f"[TestcaseBase] Drop all indexes done: {index_names}")
return collection_obj
def show_indexes(self, collection_obj: ApiCollectionWrapper = None):
collection_obj = collection_obj or self.collection_wrap
indexes = {n.field_name: n.params for n in self.collection_wrap.indexes}
log.info(f"[TestcaseBase] Collection: `{collection_obj.name}` index: {indexes}")
return indexes
""" Property """
@property
def all_scalar_fields(self):
dtypes = [
DataType.INT8,
DataType.INT16,
DataType.INT32,
DataType.INT64,
DataType.VARCHAR,
DataType.BOOL,
DataType.FLOAT,
DataType.DOUBLE,
]
dtype_names = [f"{n.name}" for n in dtypes] + [f"ARRAY_{n.name}" for n in dtypes] + [DataType.JSON.name]
return dtype_names
@property
def all_index_scalar_fields(self):
return list(set(self.all_scalar_fields) - {DataType.JSON.name})
@property
def inverted_support_dtype_names(self):
return self.all_index_scalar_fields
@property
def inverted_not_support_dtype_names(self):
return [DataType.JSON.name]
@property
def bitmap_support_dtype_names(self):
dtypes = [DataType.INT8, DataType.INT16, DataType.INT32, DataType.INT64, DataType.BOOL, DataType.VARCHAR]
dtype_names = [f"{n.name}" for n in dtypes] + [f"ARRAY_{n.name}" for n in dtypes]
return dtype_names
@property
def bitmap_not_support_dtype_names(self):
return list(set(self.all_scalar_fields) - set(self.bitmap_support_dtype_names))
class TestCaseClassBase(TestcaseBase):
"""
Setup objects on class
"""
def setup_class(self):
log.info("[setup_class] " + " Start setup class ".center(100, "~"))
self._setup_objects(self)
def teardown_class(self):
log.info("[teardown_class]" + " Start teardown class ".center(100, "~"))
self._teardown_objects(self)
def setup_method(self, method):
log.info(" setup ".center(80, "*"))
log.info(f"[setup_method] Start setup test case {method.__name__}.")
def teardown_method(self, method):
log.info(" teardown ".center(80, "*"))
log.info(f"[teardown_method] Start teardown test case {method.__name__}...")
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,387 @@
import sys
import time
from numpy import NaN
from pymilvus import Collection
sys.path.append("..")
from check.func_check import ResponseChecker
from utils.api_request import api_request
from utils.wrapper import trace
from utils.util_log import test_log as log
from pymilvus.orm.types import CONSISTENCY_STRONG
from common.common_func import param_info
TIMEOUT = 180
INDEX_NAME = ""
# keep small timeout for stability tests
# TIMEOUT = 5
class ApiCollectionWrapper:
collection = None
def __init__(self, active_trace=False):
self.active_trace = active_trace
def init_collection(self, name, schema=None, using="default", check_task=None, check_items=None,
active_trace=False, **kwargs):
self.active_trace = active_trace
consistency_level = kwargs.get("consistency_level", CONSISTENCY_STRONG)
kwargs.update({"consistency_level": consistency_level})
""" In order to distinguish the same name of collection """
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([Collection, name, schema, using], **kwargs)
self.collection = res if is_succ else None
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
name=name, schema=schema, using=using, **kwargs).run()
return res, check_result
@property
def schema(self):
return self.collection.schema
@property
def description(self):
return self.collection.description
@property
def name(self):
return self.collection.name
@property
def is_empty(self):
self.flush()
return self.collection.is_empty
@property
def is_empty_without_flush(self):
return self.collection.is_empty
@property
def num_entities(self):
self.flush()
return self.collection.num_entities
@property
def num_shards(self):
return self.collection.num_shards
@property
def num_entities_without_flush(self):
return self.collection.num_entities
@property
def primary_field(self):
return self.collection.primary_field
@property
def aliases(self):
return self.collection.aliases
@trace()
def construct_from_dataframe(self, name, dataframe, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([Collection.construct_from_dataframe, name, dataframe], **kwargs)
self.collection = res[0] if is_succ else None
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
name=name, dataframe=dataframe, **kwargs).run()
return res, check_result
@trace()
def drop(self, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.drop], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def load(self, partition_names=None, replica_number=NaN, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
replica_number = param_info.param_replica_num if replica_number is NaN else replica_number
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.load, partition_names, replica_number, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
partition_names=partition_names, **kwargs).run()
return res, check_result
@trace()
def release(self, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.release], **kwargs)
check_result = ResponseChecker(res, func_name, check_task,
check_items, check, **kwargs).run()
return res, check_result
@trace()
def insert(self, data, partition_name=None, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.insert, data, partition_name], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
data=data, partition_name=partition_name,
**kwargs).run()
return res, check_result
@trace()
def flush(self, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.flush], **kwargs)
check_result = ResponseChecker(res, func_name, check_task,
check_items, check, **kwargs).run()
return res, check_result
@trace()
def search(self, data=None, anns_field=None, param=None, limit=None, expr=None,
partition_names=None, output_fields=None, timeout=None, round_decimal=-1,
check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.search, data, anns_field, param, limit,
expr, partition_names, output_fields, timeout, round_decimal], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
data=data, anns_field=anns_field, param=param, limit=limit,
expr=expr, partition_names=partition_names,
output_fields=output_fields,
timeout=timeout, **kwargs).run()
return res, check_result
@trace()
def hybrid_search(self, reqs, rerank, limit, partition_names=None,
output_fields=None, timeout=None, round_decimal=-1,
check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.hybrid_search, reqs, rerank, limit, partition_names,
output_fields, timeout, round_decimal], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
reqs=reqs, rerank=rerank, limit=limit,
partition_names=partition_names,
output_fields=output_fields,
timeout=timeout, **kwargs).run()
return res, check_result
@trace()
def search_iterator(self, data=None, anns_field=None, param=None, batch_size=None, limit=-1, expr=None,
partition_names=None, output_fields=None, timeout=None, round_decimal=-1,
check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.search_iterator, data, anns_field, param, batch_size, limit,
expr, partition_names, output_fields, timeout, round_decimal], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
data=data, anns_field=anns_field, param=param, limit=limit,
expr=expr, partition_names=partition_names,
output_fields=output_fields,
timeout=timeout, **kwargs).run()
return res, check_result
@trace()
def query(self, expr, output_fields=None, partition_names=None, timeout=None, check_task=None, check_items=None,
**kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.query, expr, output_fields, partition_names, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
expression=expr, partition_names=partition_names,
output_fields=output_fields,
timeout=timeout, **kwargs).run()
return res, check_result
@trace()
def query_iterator(self, batch_size=1000, limit=-1, expr=None, output_fields=None, partition_names=None, timeout=None,
check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.query_iterator, batch_size, limit, expr, output_fields, partition_names,
timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
batch_size=batch_size, limit=limit, expression=expr,
output_fields=output_fields, partition_names=partition_names,
timeout=timeout, **kwargs).run()
return res, check_result
@property
def partitions(self):
return self.collection.partitions
@trace()
def partition(self, partition_name, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.collection.partition, partition_name])
check_result = ResponseChecker(res, func_name, check_task, check_items,
succ, partition_name=partition_name).run()
return res, check_result
@trace()
def has_partition(self, partition_name, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.collection.has_partition, partition_name])
check_result = ResponseChecker(res, func_name, check_task, check_items,
succ, partition_name=partition_name).run()
return res, check_result
@trace()
def drop_partition(self, partition_name, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.drop_partition, partition_name], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, partition_name=partition_name,
**kwargs).run()
return res, check_result
@trace()
def create_partition(self, partition_name, check_task=None, check_items=None, description=""):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.create_partition, partition_name, description])
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
partition_name=partition_name).run()
return res, check_result
@property
def indexes(self):
return self.collection.indexes
@trace()
def index(self, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
if "index_name" in kwargs:
index_name = kwargs.get('index_name')
kwargs.update({"index_name": index_name})
res, check = api_request([self.collection.index], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def create_index(self, field_name, index_params=None, index_name=None, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = 1200 if timeout is None else timeout
index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name)
kwargs.update({"index_name": index_name})
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.create_index, field_name, index_params, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def has_index(self, index_name=None, check_task=None, check_items=None, **kwargs):
index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name)
kwargs.update({"index_name": index_name})
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.has_index], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def drop_index(self, index_name=None, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name)
kwargs.update({"timeout": timeout, "index_name": index_name})
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.drop_index], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def delete(self, expr, partition_name=None, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.delete, expr, partition_name, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def upsert(self, data, partition_name=None, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.upsert, data, partition_name, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def compact(self, is_clustering=False, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.compact, is_clustering, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def get_compaction_state(self, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.get_compaction_state, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def get_compaction_plans(self, timeout=None, check_task=None, check_items={}, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.get_compaction_plans, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def wait_for_compaction_completed(self, timeout=None, **kwargs):
timeout = TIMEOUT * 3 if timeout is None else timeout
res = self.collection.wait_for_compaction_completed(timeout, **kwargs)
# log.debug(res)
return res
@trace()
def get_replicas(self, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.get_replicas, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def describe(self, timeout=None, check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.describe, timeout])
check_result = ResponseChecker(res, func_name, check_task, check_items, check).run()
return res, check_result
@trace()
def alter_index(self, index_name, extra_params={}, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.alter_index, index_name, extra_params, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@trace()
def set_properties(self, extra_params={}, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.set_properties, extra_params, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@@ -0,0 +1,70 @@
from pymilvus import Connections
from pymilvus import DefaultConfig
from pymilvus import MilvusClient
import sys
sys.path.append("..")
from check.func_check import ResponseChecker
from utils.api_request import api_request
class ApiConnectionsWrapper:
def __init__(self):
self.connection = Connections()
def add_connection(self, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.connection.add_connection], **kwargs)
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, **kwargs).run()
return response, check_result
def disconnect(self, alias, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.connection.disconnect, alias])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, alias=alias).run()
return response, check_result
def remove_connection(self, alias, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.connection.remove_connection, alias])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, alias=alias).run()
return response, check_result
def connect(self, alias=DefaultConfig.DEFAULT_USING, user="", password="", db_name="default", token: str = "",
check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
response, succ = api_request([self.connection.connect, alias, user, password, db_name, token], **kwargs)
check_result = ResponseChecker(response, func_name, check_task, check_items, succ, alias=alias, user=user,
password=password, db_name=db_name, token=token, **kwargs).run()
return response, check_result
def has_connection(self, alias=DefaultConfig.DEFAULT_USING, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, succ = api_request([self.connection.has_connection, alias])
check_result = ResponseChecker(response, func_name, check_task, check_items, succ, alias=alias).run()
return response, check_result
# def get_connection(self, alias=DefaultConfig.DEFAULT_USING, check_task=None, check_items=None):
# func_name = sys._getframe().f_code.co_name
# response, is_succ = api_request([self.connection.get_connection, alias])
# check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, alias=alias).run()
# return response, check_result
def list_connections(self, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.connection.list_connections])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
return response, check_result
def get_connection_addr(self, alias, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.connection.get_connection_addr, alias])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, alias=alias).run()
return response, check_result
# high level api
def MilvusClient(self, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
response, succ = api_request([MilvusClient], **kwargs)
check_result = ResponseChecker(response, func_name, check_task, check_items, succ, **kwargs).run()
return response, check_result
@@ -0,0 +1,47 @@
import sys
from pymilvus import db
from utils.api_request import api_request
from check.func_check import ResponseChecker
class ApiDatabaseWrapper:
""" wrapper of database """
database = db
def create_database(self, db_name, using="default", timeout=None, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.database.create_database, db_name, using, timeout])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
return response, check_result
def using_database(self, db_name, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.database.using_database, db_name, using])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
return response, check_result
def drop_database(self, db_name, using="default", timeout=None, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.database.drop_database, db_name, using, timeout])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
return response, check_result
def list_database(self, using="default", timeout=None, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.database.list_database, using, timeout])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
return response, check_result
def set_properties(self, db_name: str, properties: dict, using="default", timeout=None, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.database.set_properties, db_name, properties, using, timeout])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
return response, check_result
def describe_database(self, db_name: str, using="default", timeout=None, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.database.describe_database, db_name, using, timeout])
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
return response, check_result
+53
View File
@@ -0,0 +1,53 @@
import sys
from pymilvus import Index
sys.path.append("..")
from check.func_check import ResponseChecker
from utils.api_request import api_request
TIMEOUT = 20
INDEX_NAME = ""
class ApiIndexWrapper:
index = None
def init_index(self, collection, field_name, index_params, index_name=None, check_task=None, check_items=None,
**kwargs):
disktimeout = 600
timeout = kwargs.get("timeout", disktimeout * 2)
index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name)
kwargs.update({"timeout": timeout, "index_name": index_name})
""" In order to distinguish the same name of index """
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([Index, collection, field_name, index_params], **kwargs)
self.index = res if is_succ is True else None
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection=collection, field_name=field_name,
index_params=index_params, **kwargs).run()
return res, check_result
def drop(self, index_name=None, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name)
kwargs.update({"timeout": timeout, "index_name": index_name})
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.index.drop], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, **kwargs).run()
return res, check_result
@property
def params(self):
return self.index.params
@property
def collection_name(self):
return self.index.collection_name
@property
def field_name(self):
return self.index.field_name
@@ -0,0 +1,159 @@
import sys
from numpy import NaN
from pymilvus import Partition
sys.path.append("..")
from check.func_check import ResponseChecker
from utils.api_request import api_request
from common.common_func import param_info
TIMEOUT = 180
class ApiPartitionWrapper:
partition = None
def init_partition(self, collection, name, description="",
check_task=None, check_items=None, **kwargs):
""" In order to distinguish the same name of partition """
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([Partition, collection, name, description], **kwargs)
self.partition = response if is_succ is True else None
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ,
**kwargs).run()
return response, check_result
@property
def description(self):
return self.partition.description if self.partition else None
@property
def name(self):
return self.partition.name if self.partition else None
@property
def is_empty(self):
self.flush()
return self.partition.is_empty if self.partition else None
@property
def is_empty_without_flush(self):
return self.partition.is_empty if self.partition else None
@property
def num_entities(self):
self.flush()
return self.partition.num_entities if self.partition else None
@property
def num_entities_without_flush(self):
return self.partition.num_entities if self.partition else None
def drop(self, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.partition.drop], **kwargs)
check_result = ResponseChecker(res, func_name,
check_task, check_items, succ, **kwargs).run()
return res, check_result
def load(self, replica_number=NaN, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
replica_number = param_info.param_replica_num if replica_number is NaN else replica_number
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.partition.load, replica_number, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task,
check_items, is_succ=succ,
**kwargs).run()
return res, check_result
def release(self, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.partition.release], **kwargs)
check_result = ResponseChecker(res, func_name, check_task,
check_items, is_succ=succ,
**kwargs).run()
return res, check_result
def flush(self, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.partition.flush], **kwargs)
check_result = ResponseChecker(res, func_name, check_task,
check_items, is_succ=succ,
**kwargs).run()
return res, check_result
def insert(self, data, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.partition.insert, data], **kwargs)
check_result = ResponseChecker(res, func_name, check_task,
check_items, is_succ=succ, data=data,
**kwargs).run()
return res, check_result
def search(self, data, anns_field, params, limit, expr=None, output_fields=None,
check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.partition.search, data, anns_field, params,
limit, expr, output_fields], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items,
is_succ=succ, data=data, anns_field=anns_field,
params=params, limit=limit, expr=expr,
output_fields=output_fields, **kwargs).run()
return res, check_result
def query(self, expr, output_fields=None, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.partition.query, expr, output_fields, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
expression=expr, output_fields=output_fields,
timeout=timeout, **kwargs).run()
return res, check_result
def delete(self, expr, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.partition.delete, expr], **kwargs)
check_result = ResponseChecker(res, func_name, check_task,
check_items, is_succ=succ, expr=expr,
**kwargs).run()
return res, check_result
def upsert(self, data, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT)
kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name
res, succ = api_request([self.partition.upsert, data], **kwargs)
check_result = ResponseChecker(res, func_name, check_task,
check_items, is_succ=succ, data=data,
**kwargs).run()
return res, check_result
def get_replicas(self, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.partition.get_replicas, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@@ -0,0 +1,83 @@
import sys
sys.path.append("..")
from check.func_check import ResponseChecker
from utils.api_request import api_request
from pymilvus import CollectionSchema, FieldSchema
class ApiCollectionSchemaWrapper:
collection_schema = None
def init_collection_schema(self, fields, description="", check_task=None, check_items=None, **kwargs):
"""In order to distinguish the same name of CollectionSchema"""
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([CollectionSchema, fields, description], **kwargs)
self.collection_schema = response if is_succ else None
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ=is_succ, fields=fields,
description=description, **kwargs).run()
return response, check_result
@property
def primary_field(self):
return self.collection_schema.primary_field if self.collection_schema else None
@property
def partition_key_field(self):
return self.collection_schema.partition_key_field if self.collection_schema else None
@property
def fields(self):
return self.collection_schema.fields if self.collection_schema else None
@property
def description(self):
return self.collection_schema.description if self.collection_schema else None
@property
def auto_id(self):
return self.collection_schema.auto_id if self.collection_schema else None
@property
def enable_dynamic_field(self):
return self.collection_schema.enable_dynamic_field if self.collection_schema else None
@property
def to_dict(self):
return self.collection_schema.to_dict if self.collection_schema else None
@property
def verify(self):
return self.collection_schema.verify if self.collection_schema else None
def add_field(self, field_name, datatype, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([self.collection_schema.add_field, field_name, datatype], **kwargs)
check_result = ResponseChecker(response, func_name, check_task, check_items,
field_name=field_name, datatype=datatype, **kwargs).run()
return response, check_result
class ApiFieldSchemaWrapper:
field_schema = None
def init_field_schema(self, name, dtype, description="", check_task=None, check_items=None, **kwargs):
"""In order to distinguish the same name of FieldSchema"""
func_name = sys._getframe().f_code.co_name
response, is_succ = api_request([FieldSchema, name, dtype, description], **kwargs)
self.field_schema = response if is_succ else None
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, name=name, dtype=dtype,
description=description, **kwargs).run()
return response, check_result
@property
def description(self):
return self.field_schema.description if self.field_schema else None
@property
def params(self):
return self.field_schema.params if self.field_schema else None
@property
def dtype(self):
return self.field_schema.dtype if self.field_schema else None
+602
View File
@@ -0,0 +1,602 @@
from datetime import datetime
import time
from pymilvus import utility
import sys
sys.path.append("..")
from check.func_check import ResponseChecker
from utils.api_request import api_request
from pymilvus import BulkInsertState
from pymilvus.orm.role import Role
from utils.util_log import test_log as log
TIMEOUT = 20
class ApiUtilityWrapper:
""" Method of encapsulating utility files """
ut = utility
role = None
def do_bulk_insert(self, collection_name, files="", partition_name=None, timeout=None,
using="default", check_task=None, check_items=None, **kwargs):
working_tasks = self.get_bulk_insert_working_list()
log.info(f"before bulk load, there are {len(working_tasks)} working tasks")
log.info(f"files to load: {files}")
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.do_bulk_insert, collection_name,
files, partition_name, timeout, using], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection_name=collection_name, using=using).run()
time.sleep(1)
working_tasks = self.get_bulk_insert_working_list()
log.info(f"after bulk load, there are {len(working_tasks)} working tasks")
return res, check_result
def get_bulk_insert_state(self, task_id, timeout=None, using="default", check_task=None, check_items=None,
**kwargs):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.get_bulk_insert_state, task_id, timeout, using], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
task_id=task_id, using=using).run()
return res, check_result
def list_bulk_insert_tasks(self, limit=0, collection_name=None, timeout=None, using="default", check_task=None,
check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.list_bulk_insert_tasks, limit, collection_name, timeout, using], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
limit=limit, collection_name=collection_name, using=using).run()
return res, check_result
def get_bulk_insert_pending_list(self):
tasks = {}
for task in self.ut.list_bulk_insert_tasks():
if task.state == BulkInsertState.ImportPending:
tasks[task.task_id] = task
return tasks
def get_bulk_insert_working_list(self):
tasks = {}
for task in self.ut.list_bulk_insert_tasks():
if task.state in [BulkInsertState.ImportStarted]:
tasks[task.task_id] = task
return tasks
def list_all_bulk_insert_tasks(self, limit=0):
tasks, _ = self.list_bulk_insert_tasks(limit=limit)
pending = 0
started = 0
persisted = 0
completed = 0
failed = 0
failed_and_cleaned = 0
unknown = 0
for task in tasks:
print(task)
if task.state == BulkInsertState.ImportPending:
pending = pending + 1
elif task.state == BulkInsertState.ImportStarted:
started = started + 1
elif task.state == BulkInsertState.ImportPersisted:
persisted = persisted + 1
elif task.state == BulkInsertState.ImportCompleted:
completed = completed + 1
elif task.state == BulkInsertState.ImportFailed:
failed = failed + 1
elif task.state == BulkInsertState.ImportFailedAndCleaned:
failed_and_cleaned = failed_and_cleaned + 1
else:
unknown = unknown + 1
log.info("There are", len(tasks), "bulkload tasks.", pending, "pending,", started, "started,", persisted,
"persisted,", completed, "completed,", failed, "failed", failed_and_cleaned, "failed_and_cleaned",
unknown, "unknown")
def wait_for_bulk_insert_tasks_completed(self, task_ids, target_state=BulkInsertState.ImportCompleted,
timeout=None, using="default", **kwargs):
tasks_state_distribution = {
"success": set(),
"failed": set(),
"in_progress": set()
}
tasks_state = {}
if timeout is not None:
task_timeout = timeout
else:
task_timeout = TIMEOUT
start = time.time()
end = time.time()
log.info(f"wait bulk load timeout is {task_timeout}")
pending_tasks = self.get_bulk_insert_pending_list()
log.info(f"before waiting, there are {len(pending_tasks)} pending tasks")
while len(tasks_state_distribution["success"]) + len(tasks_state_distribution["failed"]) < len(
task_ids) and end - start <= task_timeout:
time.sleep(2)
for task_id in task_ids:
if task_id in tasks_state_distribution["success"] or task_id in tasks_state_distribution["failed"]:
continue
else:
state, _ = self.get_bulk_insert_state(task_id, task_timeout, using, **kwargs)
tasks_state[task_id] = state
if target_state == BulkInsertState.ImportPersisted:
if state.state in [BulkInsertState.ImportPersisted, BulkInsertState.ImportCompleted]:
if task_id in tasks_state_distribution["in_progress"]:
tasks_state_distribution["in_progress"].remove(task_id)
tasks_state_distribution["success"].add(task_id)
elif state.state in [BulkInsertState.ImportPending, BulkInsertState.ImportStarted]:
tasks_state_distribution["in_progress"].add(task_id)
else:
tasks_state_distribution["failed"].add(task_id)
if target_state == BulkInsertState.ImportCompleted:
if state.state in [BulkInsertState.ImportCompleted]:
if task_id in tasks_state_distribution["in_progress"]:
tasks_state_distribution["in_progress"].remove(task_id)
tasks_state_distribution["success"].add(task_id)
elif state.state in [BulkInsertState.ImportPending, BulkInsertState.ImportStarted,
BulkInsertState.ImportPersisted]:
tasks_state_distribution["in_progress"].add(task_id)
else:
tasks_state_distribution["failed"].add(task_id)
end = time.time()
pending_tasks = self.get_bulk_insert_pending_list()
log.info(f"after waiting, there are {len(pending_tasks)} pending tasks")
log.info(f"task state distribution: {tasks_state_distribution}")
log.info(tasks_state)
if len(tasks_state_distribution["success"]) == len(task_ids):
log.info(f"wait for bulk load tasks completed successfully, cost time: {end - start}")
return True, tasks_state
else:
log.info(f"wait for bulk load tasks completed failed, cost time: {end - start}")
return False, tasks_state
def wait_all_pending_tasks_finished(self):
task_states_map = {}
all_tasks, _ = self.list_bulk_insert_tasks()
# log.info(f"all tasks: {all_tasks}")
for task in all_tasks:
if task.state in [BulkInsertState.ImportStarted, BulkInsertState.ImportPersisted]:
task_states_map[task.task_id] = task.state
log.info(f"current tasks states: {task_states_map}")
pending_tasks = self.get_bulk_insert_pending_list()
working_tasks = self.get_bulk_insert_working_list()
log.info(
f"in the start, there are {len(working_tasks)} working tasks, {working_tasks} {len(pending_tasks)} pending tasks, {pending_tasks}")
time_cnt = 0
pending_task_ids = set()
while len(pending_tasks) > 0:
time.sleep(5)
time_cnt += 5
pending_tasks = self.get_bulk_insert_pending_list()
working_tasks = self.get_bulk_insert_working_list()
cur_pending_task_ids = []
for task_id in pending_tasks.keys():
cur_pending_task_ids.append(task_id)
pending_task_ids.add(task_id)
log.info(
f"after {time_cnt}, there are {len(working_tasks)} working tasks, {len(pending_tasks)} pending tasks")
log.debug(f"total pending tasks: {pending_task_ids} current pending tasks: {cur_pending_task_ids}")
log.info(f"after {time_cnt}, all pending tasks are finished")
all_tasks, _ = self.list_bulk_insert_tasks()
for task in all_tasks:
if task.task_id in pending_task_ids:
log.info(f"task {task.task_id} state transfer from pending to {task.state_name}")
def wait_index_build_completed(self, collection_name, timeout=None):
start = time.time()
if timeout is not None:
task_timeout = timeout
else:
task_timeout = TIMEOUT
end = time.time()
while end - start <= task_timeout:
time.sleep(0.5)
index_states, _ = self.index_building_progress(collection_name)
log.debug(f"index states: {index_states}")
if index_states["total_rows"] == index_states["indexed_rows"]:
log.info(f"index build completed")
return True
end = time.time()
log.info(f"index build timeout")
return False
def get_query_segment_info(self, collection_name, timeout=None, using="default", check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.get_query_segment_info, collection_name, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection_name=collection_name, timeout=timeout, using=using).run()
return res, check_result
def loading_progress(self, collection_name, partition_names=None,
using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.loading_progress, collection_name, partition_names, using])
check_result = ResponseChecker(res, func_name, check_task,
check_items, is_succ, collection_name=collection_name,
partition_names=partition_names, using=using).run()
return res, check_result
def load_state(self, collection_name, partition_names=None, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.load_state, collection_name, partition_names, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection_name=collection_name, partition_names=partition_names,
using=using).run()
return res, check_result
def wait_for_loading_complete(self, collection_name, partition_names=None, timeout=None, using="default",
check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.wait_for_loading_complete, collection_name,
partition_names, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection_name=collection_name, partition_names=partition_names,
timeout=timeout, using=using).run()
return res, check_result
def index_building_progress(self, collection_name, index_name="", using="default",
check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.index_building_progress, collection_name, index_name, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection_name=collection_name, index_name=index_name,
using=using).run()
return res, check_result
def wait_for_index_building_complete(self, collection_name, index_name="", timeout=None, using="default",
check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.wait_for_index_building_complete, collection_name,
index_name, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection_name=collection_name, index_name=index_name,
timeout=timeout, using=using).run()
return res, check_result
def has_collection(self, collection_name, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.has_collection, collection_name, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection_name=collection_name, using=using).run()
return res, check_result
def has_partition(self, collection_name, partition_name, using="default",
check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.has_partition, collection_name, partition_name, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection_name=collection_name,
partition_name=partition_name, using=using).run()
return res, check_result
def drop_collection(self, collection_name, timeout=None, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.drop_collection, collection_name, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
collection_name=collection_name,
timeout=timeout, using=using).run()
return res, check_result
def list_collections(self, timeout=None, using="default", check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.list_collections, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
timeout=timeout, using=using).run()
return res, check_result
def calc_distance(self, vectors_left, vectors_right, params=None, timeout=None,
using="default", check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.calc_distance, vectors_left, vectors_right,
params, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
timeout=timeout, using=using).run()
return res, check_result
def load_balance(self, collection_name, src_node_id, dst_node_ids, sealed_segment_ids, timeout=None,
using="default", check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.load_balance, collection_name, src_node_id, dst_node_ids,
sealed_segment_ids, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
timeout=timeout, using=using).run()
return res, check_result
def create_alias(self, collection_name, alias, timeout=None, using="default", check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.create_alias, collection_name, alias, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
timeout=timeout, using=using).run()
return res, check_result
def drop_alias(self, alias, timeout=None, using="default", check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.drop_alias, alias, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
timeout=timeout, using=using).run()
return res, check_result
def alter_alias(self, collection_name, alias, timeout=None, using="default", check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.alter_alias, collection_name, alias, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
timeout=timeout, using=using).run()
return res, check_result
def list_aliases(self, collection_name, timeout=None, using="default", check_task=None, check_items=None):
timeout = TIMEOUT if timeout is None else timeout
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.list_aliases, collection_name, timeout, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
timeout=timeout, using=using).run()
return res, check_result
def mkts_from_datetime(self, d_time=None, milliseconds=0., delta=None):
d_time = datetime.now() if d_time is None else d_time
res, _ = api_request([self.ut.mkts_from_datetime, d_time, milliseconds, delta])
return res
def mkts_from_hybridts(self, hybridts, milliseconds=0., delta=None):
res, _ = api_request([self.ut.mkts_from_hybridts, hybridts, milliseconds, delta])
return res
def create_user(self, user, password, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.create_user, user, password, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, using=using).run()
return res, check_result
def list_usernames(self, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.list_usernames, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
using=using).run()
return res, check_result
def reset_password(self, user, old_password, new_password, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.reset_password, user, old_password, new_password])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ).run()
return res, check_result
def update_password(self, user, old_password, new_password, check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.update_password, user, old_password, new_password])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ).run()
return res, check_result
def delete_user(self, user, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.delete_user, user, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
using=using).run()
return res, check_result
def list_roles(self, include_user_info: bool, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.list_roles, include_user_info, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, using=using).run()
return res, check_result
def list_user(self, username: str, include_role_info: bool, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.list_user, username, include_role_info, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, using=using).run()
return res, check_result
def list_users(self, include_role_info: bool, using="default", check_task=None, check_items=None):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.ut.list_users, include_role_info, using])
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, using=using).run()
return res, check_result
def init_role(self, name, using="default", check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([Role, name, using], **kwargs)
self.role = res if is_succ else None
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
name=name, **kwargs).run()
return res, check_result
def create_role(self, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([self.role.create], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
**kwargs).run()
return res, check_result
def role_drop(self, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.drop], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def role_is_exist(self, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.is_exist], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def role_add_user(self, username: str, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.add_user, username], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def role_remove_user(self, username: str, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.remove_user, username], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def role_get_users(self, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.get_users], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
@property
def role_name(self):
return self.role.name
def role_grant(self, object: str, object_name: str, privilege: str, db_name: str = "", check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.grant, object, object_name, privilege, db_name], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def role_revoke(self, object: str, object_name: str, privilege: str, db_name: str = "", check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.revoke, object, object_name, privilege, db_name], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def role_grant_v2(self, privilege: str, collection_name: str, db_name: str = None, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.grant_v2, privilege, collection_name, db_name], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def role_revoke_v2(self, privilege: str, collection_name: str, db_name: str = None, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.revoke_v2, privilege, collection_name, db_name], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def role_list_grant(self, object: str, object_name: str, db_name: str = "", check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.list_grant, object, object_name, db_name], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def role_list_grants(self, db_name: str = "", check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.list_grants, db_name], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def create_resource_group(self, name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.create_resource_group, name, using, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def drop_resource_group(self, name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.drop_resource_group, name, using, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def list_resource_groups(self, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.list_resource_groups, using, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def describe_resource_group(self, name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.describe_resource_group, name, using, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def update_resource_group(self, config, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.update_resource_groups, config, using, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def transfer_node(self, source, target, num_node, using="default", timeout=None, check_task=None, check_items=None,
**kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.transfer_node, source, target, num_node, using, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def transfer_replica(self, source, target, collection_name, num_replica, using="default", timeout=None,
check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request(
[self.ut.transfer_replica, source, target, collection_name, num_replica, using, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def rename_collection(self, old_collection_name, new_collection_name, new_db_name="", timeout=None,
check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.rename_collection, old_collection_name, new_collection_name, new_db_name,
timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
old_collection_name=old_collection_name, new_collection_name=new_collection_name,
new_db_name=new_db_name, timeout=timeout, **kwargs).run()
return res, check_result
def flush_all(self, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.flush_all, using, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
using=using, timeout=timeout, **kwargs).run()
return res, check_result
def get_server_type(self, using="default", check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.get_server_type, using], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
using=using, **kwargs).run()
return res, check_result
def list_indexes(self, collection_name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.ut.list_indexes, collection_name, using, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
collection_name=collection_name, using=using, timeout=timeout, **kwargs).run()
return res, check_result
def create_privilege_group(self, privilege_group: str, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.create_privilege_group, privilege_group], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def drop_privilege_group(self, privilege_group: str, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.drop_privilege_group, privilege_group], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def list_privilege_groups(self, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.list_privilege_groups], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def add_privileges_to_group(self, privilege_group: str, privileges: list, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.add_privileges_to_group, privilege_group, privileges], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result
def remove_privileges_from_group(self, privilege_group: str, privileges: list, check_task=None, check_items=None, **kwargs):
func_name = sys._getframe().f_code.co_name
res, check = api_request([self.role.remove_privileges_from_group, privilege_group, privileges], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
return res, check_result