chore: import upstream snapshot with attribution
Build and test / Build and test AMD64 Ubuntu 22.04 (push) Failing after 0s
Publish Builder / amazonlinux2023 (push) Failing after 1s
Build and test / UT for Go (push) Has been skipped
Publish KRTE Images / KRTE (push) Failing after 1s
Build and test / Integration Test (push) Has been skipped
Build and test / Upload Code Coverage (push) Has been skipped
Publish Builder / rockylinux9 (push) Failing after 1s
Publish Builder / ubuntu22.04 (push) Failing after 0s
Publish Builder / ubuntu24.04 (push) Failing after 0s
Publish Gpu Builder / publish-gpu-builder (push) Failing after 1s
Publish Test Images / PyTest (push) Failing after 0s
Build and test / UT for Cpp (push) Has been cancelled
Build and test / Build and test AMD64 Ubuntu 22.04 (push) Failing after 0s
Publish Builder / amazonlinux2023 (push) Failing after 1s
Build and test / UT for Go (push) Has been skipped
Publish KRTE Images / KRTE (push) Failing after 1s
Build and test / Integration Test (push) Has been skipped
Build and test / Upload Code Coverage (push) Has been skipped
Publish Builder / rockylinux9 (push) Failing after 1s
Publish Builder / ubuntu22.04 (push) Failing after 0s
Publish Builder / ubuntu24.04 (push) Failing after 0s
Publish Gpu Builder / publish-gpu-builder (push) Failing after 1s
Publish Test Images / PyTest (push) Failing after 0s
Build and test / UT for Cpp (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,267 @@
|
||||
import asyncio
|
||||
import sys
|
||||
from typing import Optional, List, Union, Dict
|
||||
|
||||
from pymilvus import (
|
||||
AsyncMilvusClient,
|
||||
AnnSearchRequest,
|
||||
RRFRanker,
|
||||
)
|
||||
from pymilvus.orm.types import CONSISTENCY_STRONG
|
||||
from pymilvus.orm.collection import CollectionSchema
|
||||
|
||||
from check.func_check import ResponseChecker
|
||||
from utils.api_request import api_request, logger_interceptor
|
||||
|
||||
|
||||
class AsyncMilvusClientWrapper:
|
||||
async_milvus_client = None
|
||||
|
||||
def __init__(self, active_trace=False):
|
||||
self.active_trace = active_trace
|
||||
|
||||
def init_async_client(self, uri: str = "http://localhost:19530",
|
||||
user: str = "",
|
||||
password: str = "",
|
||||
db_name: str = "",
|
||||
token: str = "",
|
||||
timeout: Optional[float] = None,
|
||||
active_trace=False,
|
||||
check_task=None, check_items=None,
|
||||
**kwargs):
|
||||
self.active_trace = active_trace
|
||||
|
||||
""" In order to distinguish the same name of collection """
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([AsyncMilvusClient, uri, user, password, db_name, token,
|
||||
timeout], **kwargs)
|
||||
self.async_milvus_client = res if is_succ else None
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@logger_interceptor()
|
||||
async def list_collections(self, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.list_collections(timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def has_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.has_collection(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def has_partition(self, collection_name: str, partition_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.has_partition(collection_name, partition_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def describe_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.describe_collection(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def list_partitions(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.list_partitions(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def get_collection_stats(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.get_collection_stats(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def flush(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.flush(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def get_load_state(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.get_load_state(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def describe_index(self, collection_name: str, index_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.describe_index(collection_name, index_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def create_database(self, db_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.create_database(db_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def drop_database(self, db_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.drop_database(db_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def list_databases(self, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.list_databases(timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def list_indexes(self, collection_name: str, field_name: str = "", **kwargs):
|
||||
return await self.async_milvus_client.list_indexes(collection_name, field_name, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def create_collection(self,
|
||||
collection_name: str,
|
||||
dimension: Optional[int] = None,
|
||||
primary_field_name: str = "id", # default is "id"
|
||||
id_type: str = "int", # or "string",
|
||||
vector_field_name: str = "vector", # default is "vector"
|
||||
metric_type: str = "COSINE",
|
||||
auto_id: bool = False,
|
||||
timeout: Optional[float] = None,
|
||||
schema: Optional[CollectionSchema] = None,
|
||||
index_params=None,
|
||||
**kwargs):
|
||||
kwargs["consistency_level"] = kwargs.get("consistency_level", CONSISTENCY_STRONG)
|
||||
|
||||
return await self.async_milvus_client.create_collection(collection_name, dimension,
|
||||
primary_field_name,
|
||||
id_type, vector_field_name, metric_type,
|
||||
auto_id,
|
||||
timeout, schema, index_params, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def drop_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.drop_collection(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def load_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.load_collection(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def release_collection(self, collection_name, timeout=None, **kwargs):
|
||||
return await self.async_milvus_client.release_collection(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def truncate_collection(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.truncate_collection(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def list_persistent_segments(self, collection_name: str, timeout: Optional[float] = None, **kwargs):
|
||||
return await self.async_milvus_client.list_persistent_segments(collection_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def create_index(self, collection_name: str, index_params, timeout: Optional[float] = None,
|
||||
**kwargs):
|
||||
return await self.async_milvus_client.create_index(collection_name, index_params, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def drop_index(self, collection_name, index_name, timeout=None, **kwargs):
|
||||
return await self.async_milvus_client.drop_index(collection_name, index_name, timeout, **kwargs)
|
||||
|
||||
# @logger_interceptor()
|
||||
# async def list_indexes(self, collection_name, field_name="", timeout=None, **kwargs):
|
||||
# return await self.async_milvus_client.list_indexes(collection_name, field_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def create_partition(self, collection_name, partition_name, timeout=None, **kwargs):
|
||||
return await self.async_milvus_client.create_partition(collection_name, partition_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def drop_partition(self, collection_name, partition_name, timeout=None, **kwargs):
|
||||
return await self.async_milvus_client.drop_partition(collection_name, partition_name, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def load_partitions(self, collection_name, partition_names, timeout=None, **kwargs):
|
||||
return await self.async_milvus_client.load_partitions(collection_name, partition_names, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def release_partitions(self, collection_name, partition_names, timeout=None, **kwargs):
|
||||
return await self.async_milvus_client.release_partitions(collection_name, partition_names, timeout, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def insert(self,
|
||||
collection_name: str,
|
||||
data: Union[Dict, List[Dict]],
|
||||
timeout: Optional[float] = None,
|
||||
partition_name: Optional[str] = "",
|
||||
**kwargs):
|
||||
return await self.async_milvus_client.insert(collection_name, data, timeout, partition_name, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def upsert(self,
|
||||
collection_name: str,
|
||||
data: Union[Dict, List[Dict]],
|
||||
timeout: Optional[float] = None,
|
||||
partition_name: Optional[str] = "",
|
||||
**kwargs):
|
||||
return await self.async_milvus_client.upsert(collection_name, data, timeout, partition_name, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def search(self,
|
||||
collection_name: str,
|
||||
data: Union[List[list], list],
|
||||
filter: str = "",
|
||||
limit: int = 10,
|
||||
output_fields: Optional[List[str]] = None,
|
||||
search_params: Optional[dict] = None,
|
||||
timeout: Optional[float] = None,
|
||||
partition_names: Optional[List[str]] = None,
|
||||
anns_field: Optional[str] = None,
|
||||
**kwargs):
|
||||
return await self.async_milvus_client.search(collection_name, data,
|
||||
filter,
|
||||
limit, output_fields, search_params,
|
||||
timeout,
|
||||
partition_names, anns_field, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def hybrid_search(self,
|
||||
collection_name: str,
|
||||
reqs: List[AnnSearchRequest],
|
||||
ranker: RRFRanker,
|
||||
limit: int = 10,
|
||||
output_fields: Optional[List[str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
partition_names: Optional[List[str]] = None,
|
||||
**kwargs):
|
||||
return await self.async_milvus_client.hybrid_search(collection_name, reqs,
|
||||
ranker,
|
||||
limit, output_fields,
|
||||
timeout, partition_names, **kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def query(self,
|
||||
collection_name: str,
|
||||
filter: str = "",
|
||||
output_fields: Optional[List[str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
ids: Optional[Union[List, str, int]] = None,
|
||||
partition_names: Optional[List[str]] = None,
|
||||
**kwargs):
|
||||
return await self.async_milvus_client.query(collection_name, filter,
|
||||
output_fields, timeout,
|
||||
ids, partition_names,
|
||||
**kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def get(self,
|
||||
collection_name: str,
|
||||
ids: Union[list, str, int],
|
||||
output_fields: Optional[List[str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
partition_names: Optional[List[str]] = None,
|
||||
**kwargs):
|
||||
return await self.async_milvus_client.get(collection_name, ids,
|
||||
output_fields, timeout,
|
||||
partition_names,
|
||||
**kwargs)
|
||||
|
||||
@logger_interceptor()
|
||||
async def delete(self,
|
||||
collection_name: str,
|
||||
ids: Optional[Union[list, str, int]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
filter: Optional[str] = None,
|
||||
partition_name: Optional[str] = None,
|
||||
**kwargs):
|
||||
return await self.async_milvus_client.delete(collection_name, ids,
|
||||
timeout, filter,
|
||||
partition_name,
|
||||
**kwargs)
|
||||
|
||||
@classmethod
|
||||
def create_schema(cls, **kwargs):
|
||||
kwargs["check_fields"] = False # do not check fields for now
|
||||
return CollectionSchema([], **kwargs)
|
||||
|
||||
@classmethod
|
||||
def prepare_index_params(cls, field_name: str = "", **kwargs):
|
||||
res, check = api_request([AsyncMilvusClient.prepare_index_params, field_name], **kwargs)
|
||||
return res, check
|
||||
|
||||
@logger_interceptor()
|
||||
async def close(self, **kwargs):
|
||||
return await self.async_milvus_client.close(**kwargs)
|
||||
@@ -0,0 +1,649 @@
|
||||
import sys
|
||||
|
||||
from base.database_wrapper import ApiDatabaseWrapper
|
||||
from pymilvus import DefaultConfig
|
||||
|
||||
sys.path.append("..")
|
||||
import pymilvus
|
||||
from base.async_milvus_client_wrapper import AsyncMilvusClientWrapper
|
||||
from base.collection_wrapper import ApiCollectionWrapper
|
||||
from base.connections_wrapper import ApiConnectionsWrapper
|
||||
from base.index_wrapper import ApiIndexWrapper
|
||||
from base.partition_wrapper import ApiPartitionWrapper
|
||||
from base.schema_wrapper import ApiCollectionSchemaWrapper, ApiFieldSchemaWrapper
|
||||
from base.utility_wrapper import ApiUtilityWrapper
|
||||
from common import common_func as cf
|
||||
from common import common_type as ct
|
||||
from common.common_params import IndexPrams
|
||||
from pymilvus import DataType, MilvusClient, ResourceGroupInfo, utility
|
||||
from utils.util_log import test_log as log
|
||||
|
||||
|
||||
class Base:
|
||||
"""Initialize class object"""
|
||||
|
||||
connection_wrap = None
|
||||
collection_wrap = None
|
||||
partition_wrap = None
|
||||
index_wrap = None
|
||||
utility_wrap = None
|
||||
collection_schema_wrap = None
|
||||
field_schema_wrap = None
|
||||
database_wrap = None
|
||||
tear_down_collection_names = []
|
||||
tear_down_role_names = []
|
||||
tear_down_user_names = []
|
||||
resource_group_list = []
|
||||
async_milvus_client_wrap = None
|
||||
skip_connection = False
|
||||
skip_global_role_cleanup = False
|
||||
|
||||
def setup_class(self):
|
||||
log.info("[setup_class] Start setup class...")
|
||||
|
||||
def teardown_class(self):
|
||||
log.info("[teardown_class] Start teardown class...")
|
||||
|
||||
def setup_method(self, method):
|
||||
log.info(("*" * 35) + " setup " + ("*" * 35))
|
||||
log.info(f"pymilvus version: {pymilvus.__version__}")
|
||||
log.info(f"[setup_method] Start setup test case {method.__name__}.")
|
||||
self._setup_objects()
|
||||
|
||||
def _setup_objects(self):
|
||||
self.connection_wrap = ApiConnectionsWrapper()
|
||||
self.utility_wrap = ApiUtilityWrapper()
|
||||
self.collection_wrap = ApiCollectionWrapper()
|
||||
self.partition_wrap = ApiPartitionWrapper()
|
||||
self.index_wrap = ApiIndexWrapper()
|
||||
self.collection_schema_wrap = ApiCollectionSchemaWrapper()
|
||||
self.field_schema_wrap = ApiFieldSchemaWrapper()
|
||||
self.database_wrap = ApiDatabaseWrapper()
|
||||
self.async_milvus_client_wrap = AsyncMilvusClientWrapper()
|
||||
|
||||
def teardown_method(self, method):
|
||||
log.info(("*" * 35) + " teardown " + ("*" * 35))
|
||||
log.info(f"[teardown_method] Start teardown test case {method.__name__}...")
|
||||
self._teardown_objects()
|
||||
|
||||
def _teardown_objects(self):
|
||||
# Prioritize uri and token for connection
|
||||
if cf.param_info.param_uri:
|
||||
uri = cf.param_info.param_uri
|
||||
else:
|
||||
uri = "http://" + cf.param_info.param_host + ":" + str(cf.param_info.param_port)
|
||||
|
||||
if cf.param_info.param_token:
|
||||
token = cf.param_info.param_token
|
||||
else:
|
||||
token = (
|
||||
f"{cf.param_info.param_user}:{cf.param_info.param_password}"
|
||||
if cf.param_info.param_user and cf.param_info.param_password
|
||||
else None
|
||||
)
|
||||
|
||||
try:
|
||||
""" Drop collection before disconnect """
|
||||
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
|
||||
if token:
|
||||
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri, token=token)
|
||||
else:
|
||||
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri)
|
||||
|
||||
if self.collection_wrap.collection is not None:
|
||||
if self.collection_wrap.collection.name.startswith("alias"):
|
||||
log.info(f"collection {self.collection_wrap.collection.name} is alias, skip drop operation")
|
||||
else:
|
||||
self.collection_wrap.drop(check_task=ct.CheckTasks.check_nothing)
|
||||
|
||||
collection_list = self.utility_wrap.list_collections()[0]
|
||||
for collection_name in self.tear_down_collection_names:
|
||||
if collection_name is not None and collection_name in collection_list:
|
||||
alias_list = self.utility_wrap.list_aliases(collection_name)[0]
|
||||
if alias_list:
|
||||
for alias in alias_list:
|
||||
self.utility_wrap.drop_alias(alias)
|
||||
self.utility_wrap.drop_collection(collection_name)
|
||||
|
||||
""" Clean up the rgs before disconnect """
|
||||
rgs_list = self.utility_wrap.list_resource_groups()[0]
|
||||
for rg_name in self.resource_group_list:
|
||||
if rg_name is not None and rg_name in rgs_list:
|
||||
rg = self.utility_wrap.describe_resource_group(
|
||||
name=rg_name, check_task=ct.CheckTasks.check_nothing
|
||||
)[0]
|
||||
if isinstance(rg, ResourceGroupInfo):
|
||||
if rg.num_available_node > 0:
|
||||
self.utility_wrap.transfer_node(
|
||||
source=rg_name, target=ct.default_resource_group_name, num_node=rg.num_available_node
|
||||
)
|
||||
self.utility_wrap.drop_resource_group(rg_name, check_task=ct.CheckTasks.check_nothing)
|
||||
|
||||
except Exception as e:
|
||||
log.debug(str(e))
|
||||
|
||||
if not self.skip_global_role_cleanup:
|
||||
try:
|
||||
""" Drop roles before disconnect """
|
||||
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
|
||||
if token:
|
||||
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri, token=token)
|
||||
else:
|
||||
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri)
|
||||
|
||||
role_list = self.utility_wrap.list_roles(False)[0]
|
||||
for role in role_list.groups:
|
||||
role_name = role.role_name
|
||||
if role_name not in ["admin", "public"]:
|
||||
each_role = self.utility_wrap.init_role(name=role_name)[0]
|
||||
each_role.drop()
|
||||
|
||||
except Exception as e:
|
||||
log.debug(str(e))
|
||||
|
||||
try:
|
||||
""" Delete connection and reset configuration"""
|
||||
res = self.connection_wrap.list_connections()
|
||||
for i in res[0]:
|
||||
self.connection_wrap.remove_connection(i[0])
|
||||
|
||||
# because the connection is in singleton mode, it needs to be restored to the original state after teardown
|
||||
self.connection_wrap.add_connection(
|
||||
default={"host": DefaultConfig.DEFAULT_HOST, "port": DefaultConfig.DEFAULT_PORT}
|
||||
)
|
||||
except Exception as e:
|
||||
log.debug(str(e))
|
||||
|
||||
|
||||
class TestcaseBase(Base):
|
||||
"""
|
||||
Additional methods;
|
||||
Public methods that can be used for test cases.
|
||||
"""
|
||||
|
||||
client = None
|
||||
|
||||
def _connect(self, enable_milvus_client_api=False):
|
||||
"""Add a connection and create the connect"""
|
||||
if self.skip_connection:
|
||||
return None
|
||||
|
||||
# Prioritize uri and token for connection
|
||||
if cf.param_info.param_uri:
|
||||
uri = cf.param_info.param_uri
|
||||
else:
|
||||
uri = "http://" + cf.param_info.param_host + ":" + str(cf.param_info.param_port)
|
||||
|
||||
if cf.param_info.param_token:
|
||||
token = cf.param_info.param_token
|
||||
else:
|
||||
token = (
|
||||
f"{cf.param_info.param_user}:{cf.param_info.param_password}"
|
||||
if cf.param_info.param_user and cf.param_info.param_password
|
||||
else None
|
||||
)
|
||||
|
||||
if enable_milvus_client_api:
|
||||
self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri, token=token)
|
||||
res, is_succ = self.connection_wrap.MilvusClient(uri=uri, token=token)
|
||||
self.client = MilvusClient(uri=uri, token=token)
|
||||
else:
|
||||
if token:
|
||||
res, is_succ = self.connection_wrap.connect(
|
||||
alias=DefaultConfig.DEFAULT_USING, uri=uri, token=token, secure=cf.param_info.param_secure
|
||||
)
|
||||
else:
|
||||
res, is_succ = self.connection_wrap.connect(alias=DefaultConfig.DEFAULT_USING, uri=uri)
|
||||
|
||||
self.client = MilvusClient(uri=uri, token=token)
|
||||
server_version = utility.get_server_version()
|
||||
log.info(f"server version: {server_version}")
|
||||
return res
|
||||
|
||||
def get_tokens_by_analyzer(self, text, analyzer_params):
|
||||
if cf.param_info.param_uri:
|
||||
uri = cf.param_info.param_uri
|
||||
else:
|
||||
uri = "http://" + cf.param_info.param_host + ":" + str(cf.param_info.param_port)
|
||||
|
||||
client = MilvusClient(uri=uri, token=cf.param_info.param_token)
|
||||
res = client.run_analyzer(text, analyzer_params, with_detail=True, with_hash=True)
|
||||
tokens = [r["token"] for r in res.tokens]
|
||||
return tokens
|
||||
|
||||
# def init_async_milvus_client(self):
|
||||
# uri = cf.param_info.param_uri or f"http://{cf.param_info.param_host}:{cf.param_info.param_port}"
|
||||
# kwargs = {
|
||||
# "uri": uri,
|
||||
# "user": cf.param_info.param_user,
|
||||
# "password": cf.param_info.param_password,
|
||||
# "token": cf.param_info.param_token,
|
||||
# }
|
||||
# self.async_milvus_client_wrap.init_async_client(**kwargs)
|
||||
|
||||
def init_collection_wrap(
|
||||
self,
|
||||
name=None,
|
||||
schema=None,
|
||||
check_task=None,
|
||||
check_items=None,
|
||||
enable_dynamic_field=False,
|
||||
with_json=True,
|
||||
**kwargs,
|
||||
):
|
||||
name = cf.gen_collection_name_by_testcase_name(2) if name is None else name
|
||||
schema = (
|
||||
cf.gen_default_collection_schema(enable_dynamic_field=enable_dynamic_field, with_json=with_json)
|
||||
if schema is None
|
||||
else schema
|
||||
)
|
||||
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
|
||||
self._connect()
|
||||
collection_w = ApiCollectionWrapper()
|
||||
collection_w.init_collection(name=name, schema=schema, check_task=check_task, check_items=check_items, **kwargs)
|
||||
self.tear_down_collection_names.append(name)
|
||||
return collection_w
|
||||
|
||||
def init_multi_fields_collection_wrap(self, name=cf.gen_unique_str()):
|
||||
vec_fields = [cf.gen_float_vec_field(ct.another_float_vec_field_name)]
|
||||
schema = cf.gen_schema_multi_vector_fields(vec_fields)
|
||||
collection_w = self.init_collection_wrap(name=name, schema=schema)
|
||||
df = cf.gen_dataframe_multi_vec_fields(vec_fields=vec_fields)
|
||||
collection_w.insert(df)
|
||||
assert collection_w.num_entities == ct.default_nb
|
||||
return collection_w, df
|
||||
|
||||
def init_partition_wrap(
|
||||
self, collection_wrap=None, name=None, description=None, check_task=None, check_items=None, **kwargs
|
||||
):
|
||||
name = cf.gen_unique_str("partition_") if name is None else name
|
||||
description = cf.gen_unique_str("partition_des_") if description is None else description
|
||||
collection_wrap = self.init_collection_wrap() if collection_wrap is None else collection_wrap
|
||||
partition_wrap = ApiPartitionWrapper()
|
||||
partition_wrap.init_partition(
|
||||
collection_wrap.collection, name, description, check_task=check_task, check_items=check_items, **kwargs
|
||||
)
|
||||
return partition_wrap
|
||||
|
||||
def insert_data_general(
|
||||
self,
|
||||
prefix="test",
|
||||
insert_data=False,
|
||||
nb=ct.default_nb,
|
||||
partition_num=0,
|
||||
is_binary=False,
|
||||
is_all_data_type=False,
|
||||
auto_id=False,
|
||||
dim=ct.default_dim,
|
||||
primary_field=ct.default_int64_field_name,
|
||||
is_flush=True,
|
||||
name=None,
|
||||
enable_dynamic_field=False,
|
||||
with_json=True,
|
||||
**kwargs,
|
||||
):
|
||||
""" """
|
||||
self._connect()
|
||||
collection_name = cf.gen_unique_str(prefix)
|
||||
if name is not None:
|
||||
collection_name = name
|
||||
vectors = []
|
||||
binary_raw_vectors = []
|
||||
insert_ids = []
|
||||
time_stamp = 0
|
||||
# 1 create collection
|
||||
default_schema = cf.gen_default_collection_schema(
|
||||
auto_id=auto_id,
|
||||
dim=dim,
|
||||
primary_field=primary_field,
|
||||
enable_dynamic_field=enable_dynamic_field,
|
||||
with_json=with_json,
|
||||
)
|
||||
if is_binary:
|
||||
default_schema = cf.gen_default_binary_collection_schema(
|
||||
auto_id=auto_id, dim=dim, primary_field=primary_field
|
||||
)
|
||||
if is_all_data_type:
|
||||
default_schema = cf.gen_collection_schema_all_datatype(
|
||||
auto_id=auto_id,
|
||||
dim=dim,
|
||||
primary_field=primary_field,
|
||||
enable_dynamic_field=enable_dynamic_field,
|
||||
with_json=with_json,
|
||||
)
|
||||
log.info("insert_data_general: collection creation")
|
||||
collection_w = self.init_collection_wrap(name=collection_name, schema=default_schema, **kwargs)
|
||||
pre_entities = collection_w.num_entities
|
||||
if insert_data:
|
||||
collection_w, vectors, binary_raw_vectors, insert_ids, time_stamp = cf.insert_data(
|
||||
collection_w,
|
||||
nb,
|
||||
is_binary,
|
||||
is_all_data_type,
|
||||
auto_id=auto_id,
|
||||
dim=dim,
|
||||
enable_dynamic_field=enable_dynamic_field,
|
||||
with_json=with_json,
|
||||
)
|
||||
if is_flush:
|
||||
collection_w.flush()
|
||||
assert collection_w.num_entities == nb + pre_entities
|
||||
|
||||
return collection_w, vectors, binary_raw_vectors, insert_ids, time_stamp
|
||||
|
||||
def init_collection_general(
|
||||
self,
|
||||
prefix="test",
|
||||
insert_data=False,
|
||||
nb=ct.default_nb,
|
||||
partition_num=0,
|
||||
is_binary=False,
|
||||
is_all_data_type=False,
|
||||
auto_id=False,
|
||||
dim=ct.default_dim,
|
||||
is_index=True,
|
||||
primary_field=ct.default_int64_field_name,
|
||||
is_flush=True,
|
||||
name=None,
|
||||
enable_dynamic_field=False,
|
||||
with_json=True,
|
||||
random_primary_key=False,
|
||||
multiple_dim_array=[],
|
||||
is_partition_key=None,
|
||||
vector_data_type=DataType.FLOAT_VECTOR,
|
||||
nullable_fields={},
|
||||
default_value_fields={},
|
||||
language=None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
target: create specified collections
|
||||
method: 1. create collections (binary/non-binary, default/all data type, auto_id or not)
|
||||
2. create partitions if specified
|
||||
3. insert specified (binary/non-binary, default/all data type) data
|
||||
into each partition if any
|
||||
4. not load if specifying is_index as True
|
||||
5. enable insert null data: nullable_fields = {"nullable_fields_name": null data percent}
|
||||
6. enable insert default value: default_value_fields = {"default_fields_name": default value}
|
||||
expected: return collection and raw data, insert ids
|
||||
"""
|
||||
log.info("Test case of search interface: initialize before test case")
|
||||
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
|
||||
self._connect()
|
||||
collection_name = cf.gen_collection_name_by_testcase_name(2)
|
||||
if name is not None:
|
||||
collection_name = name
|
||||
if not isinstance(nullable_fields, dict):
|
||||
log.error("nullable_fields should a dict like {'nullable_fields_name': null data percent}")
|
||||
assert False
|
||||
if not isinstance(default_value_fields, dict):
|
||||
log.error("default_value_fields should a dict like {'default_fields_name': default value}")
|
||||
assert False
|
||||
vectors = []
|
||||
binary_raw_vectors = []
|
||||
insert_ids = []
|
||||
time_stamp = 0
|
||||
# 1 create collection
|
||||
default_schema = cf.gen_default_collection_schema(
|
||||
auto_id=auto_id,
|
||||
dim=dim,
|
||||
primary_field=primary_field,
|
||||
enable_dynamic_field=enable_dynamic_field,
|
||||
with_json=with_json,
|
||||
multiple_dim_array=multiple_dim_array,
|
||||
is_partition_key=is_partition_key,
|
||||
vector_data_type=vector_data_type,
|
||||
nullable_fields=nullable_fields,
|
||||
default_value_fields=default_value_fields,
|
||||
)
|
||||
if is_binary:
|
||||
default_schema = cf.gen_default_binary_collection_schema(
|
||||
auto_id=auto_id,
|
||||
dim=dim,
|
||||
primary_field=primary_field,
|
||||
nullable_fields=nullable_fields,
|
||||
default_value_fields=default_value_fields,
|
||||
)
|
||||
if vector_data_type == DataType.SPARSE_FLOAT_VECTOR:
|
||||
default_schema = cf.gen_default_sparse_schema(
|
||||
auto_id=auto_id,
|
||||
primary_field=primary_field,
|
||||
enable_dynamic_field=enable_dynamic_field,
|
||||
with_json=with_json,
|
||||
multiple_dim_array=multiple_dim_array,
|
||||
nullable_fields=nullable_fields,
|
||||
default_value_fields=default_value_fields,
|
||||
)
|
||||
if is_all_data_type:
|
||||
default_schema = cf.gen_collection_schema_all_datatype(
|
||||
auto_id=auto_id,
|
||||
dim=dim,
|
||||
primary_field=primary_field,
|
||||
enable_dynamic_field=enable_dynamic_field,
|
||||
with_json=with_json,
|
||||
multiple_dim_array=multiple_dim_array,
|
||||
nullable_fields=nullable_fields,
|
||||
default_value_fields=default_value_fields,
|
||||
)
|
||||
log.info("init_collection_general: collection creation")
|
||||
collection_w = self.init_collection_wrap(name=collection_name, schema=default_schema, **kwargs)
|
||||
vector_name_list = cf.extract_vector_field_name_list(collection_w)
|
||||
# 2 add extra partitions if specified (default is 1 partition named "_default")
|
||||
if partition_num > 0:
|
||||
cf.gen_partitions(collection_w, partition_num)
|
||||
# 3 insert data if specified
|
||||
if insert_data:
|
||||
collection_w, vectors, binary_raw_vectors, insert_ids, time_stamp = cf.insert_data(
|
||||
collection_w,
|
||||
nb,
|
||||
is_binary,
|
||||
is_all_data_type,
|
||||
auto_id=auto_id,
|
||||
dim=dim,
|
||||
enable_dynamic_field=enable_dynamic_field,
|
||||
with_json=with_json,
|
||||
random_primary_key=random_primary_key,
|
||||
multiple_dim_array=multiple_dim_array,
|
||||
primary_field=primary_field,
|
||||
vector_data_type=vector_data_type,
|
||||
nullable_fields=nullable_fields,
|
||||
language=language,
|
||||
)
|
||||
if is_flush:
|
||||
assert collection_w.is_empty is False
|
||||
assert collection_w.num_entities == nb
|
||||
# 4 create default index if specified
|
||||
if is_index:
|
||||
# This condition will be removed after auto index feature
|
||||
if is_binary:
|
||||
collection_w.create_index(ct.default_binary_vec_field_name, ct.default_bin_flat_index)
|
||||
elif vector_data_type == DataType.SPARSE_FLOAT_VECTOR:
|
||||
for vector_name in vector_name_list:
|
||||
collection_w.create_index(vector_name, ct.default_sparse_inverted_index)
|
||||
else:
|
||||
if len(multiple_dim_array) == 0 or is_all_data_type is False:
|
||||
vector_name_list.append(ct.default_float_vec_field_name)
|
||||
for vector_name in vector_name_list:
|
||||
# Unlike dense vectors, sparse vectors cannot create flat index.
|
||||
if DataType.SPARSE_FLOAT_VECTOR.name in vector_name:
|
||||
collection_w.create_index(vector_name, ct.default_sparse_inverted_index)
|
||||
elif vector_data_type == DataType.INT8_VECTOR:
|
||||
collection_w.create_index(vector_name, ct.int8_vector_index)
|
||||
else:
|
||||
collection_w.create_index(vector_name, ct.default_flat_index)
|
||||
|
||||
collection_w.load()
|
||||
|
||||
return collection_w, vectors, binary_raw_vectors, insert_ids, time_stamp
|
||||
|
||||
def insert_entities_into_two_partitions_in_half(self, half, prefix="query"):
|
||||
"""
|
||||
insert default entities into two partitions(partition_w and _default) in half(int64 and float fields values)
|
||||
:param half: half of nb
|
||||
:return: collection wrap and partition wrap
|
||||
"""
|
||||
self._connect()
|
||||
collection_name = cf.gen_collection_name_by_testcase_name(2)
|
||||
collection_w = self.init_collection_wrap(name=collection_name)
|
||||
partition_w = self.init_partition_wrap(collection_wrap=collection_w)
|
||||
# insert [0, half) into partition_w
|
||||
df_partition = cf.gen_default_dataframe_data(nb=half, start=0)
|
||||
partition_w.insert(df_partition)
|
||||
# insert [half, nb) into _default
|
||||
df_default = cf.gen_default_dataframe_data(nb=half, start=half)
|
||||
collection_w.insert(df_default)
|
||||
# flush
|
||||
collection_w.num_entities
|
||||
collection_w.create_index(ct.default_float_vec_field_name, index_params=ct.default_flat_index)
|
||||
collection_w.load(partition_names=[partition_w.name, "_default"])
|
||||
return collection_w, partition_w, df_partition, df_default
|
||||
|
||||
def collection_insert_multi_segments_one_shard(
|
||||
self, collection_prefix, num_of_segment=2, nb_of_segment=1, is_dup=True
|
||||
):
|
||||
"""
|
||||
init collection with one shard, insert data into two segments on one shard (they can be merged)
|
||||
:param collection_prefix: collection name prefix
|
||||
:param num_of_segment: number of segments
|
||||
:param nb_of_segment: number of entities per segment
|
||||
:param is_dup: whether the primary keys of each segment is duplicated
|
||||
:return: collection wrap and partition wrap
|
||||
"""
|
||||
collection_name = cf.gen_collection_name_by_testcase_name(2)
|
||||
collection_w = self.init_collection_wrap(name=collection_name, shards_num=1)
|
||||
|
||||
for i in range(num_of_segment):
|
||||
start = 0 if is_dup else i * nb_of_segment
|
||||
df = cf.gen_default_dataframe_data(nb_of_segment, start=start)
|
||||
collection_w.insert(df)
|
||||
assert collection_w.num_entities == nb_of_segment * (i + 1)
|
||||
return collection_w
|
||||
|
||||
def init_resource_group(self, name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
if not self.connection_wrap.has_connection(alias=DefaultConfig.DEFAULT_USING)[0]:
|
||||
self._connect()
|
||||
utility_w = ApiUtilityWrapper()
|
||||
res, check_result = utility_w.create_resource_group(
|
||||
name=name, using=using, timeout=timeout, check_task=check_task, check_items=check_items, **kwargs
|
||||
)
|
||||
if res is None and check_result:
|
||||
self.resource_group_list.append(name)
|
||||
return res, check_result
|
||||
|
||||
def init_user_with_privilege(self, privilege_object, object_name, privilege, db_name="default"):
|
||||
"""
|
||||
init a user and role, grant privilege to the role with the db, then bind the role to the user
|
||||
:param privilege_object: privilege object: Global, Collection, User
|
||||
:type privilege_object: str
|
||||
:param object_name: privilege object name
|
||||
:type object_name: str
|
||||
:param privilege: privilege
|
||||
:type privilege: str
|
||||
:param db_name: database name
|
||||
:type db_name: str
|
||||
:return: user name, user pwd, role name
|
||||
:rtype: str, str, str
|
||||
"""
|
||||
tmp_user = cf.gen_unique_str("user")
|
||||
tmp_pwd = cf.gen_unique_str("pwd")
|
||||
tmp_role = cf.gen_unique_str("role")
|
||||
|
||||
# create user
|
||||
self.utility_wrap.create_user(tmp_user, tmp_pwd)
|
||||
|
||||
# create role
|
||||
self.utility_wrap.init_role(tmp_role)
|
||||
self.utility_wrap.create_role()
|
||||
|
||||
# grant privilege to the role
|
||||
self.utility_wrap.role_grant(
|
||||
object=privilege_object, object_name=object_name, privilege=privilege, db_name=db_name
|
||||
)
|
||||
|
||||
# bind the role to the user
|
||||
self.utility_wrap.role_add_user(tmp_user)
|
||||
|
||||
return tmp_user, tmp_pwd, tmp_role
|
||||
|
||||
def build_multi_index(self, index_params: dict[str, IndexPrams], collection_obj: ApiCollectionWrapper = None):
|
||||
collection_obj = collection_obj or self.collection_wrap
|
||||
for k, v in index_params.items():
|
||||
collection_obj.create_index(field_name=k, index_params=v.to_dict, index_name=k)
|
||||
log.info(f"[TestcaseBase] Build all indexes done: {list(index_params.keys())}")
|
||||
return collection_obj
|
||||
|
||||
def drop_multi_index(
|
||||
self, index_names: list[str], collection_obj: ApiCollectionWrapper = None, check_task=None, check_items=None
|
||||
):
|
||||
collection_obj = collection_obj or self.collection_wrap
|
||||
for n in index_names:
|
||||
collection_obj.drop_index(index_name=n, check_task=check_task, check_items=check_items)
|
||||
log.info(f"[TestcaseBase] Drop all indexes done: {index_names}")
|
||||
return collection_obj
|
||||
|
||||
def show_indexes(self, collection_obj: ApiCollectionWrapper = None):
|
||||
collection_obj = collection_obj or self.collection_wrap
|
||||
indexes = {n.field_name: n.params for n in self.collection_wrap.indexes}
|
||||
log.info(f"[TestcaseBase] Collection: `{collection_obj.name}` index: {indexes}")
|
||||
return indexes
|
||||
|
||||
""" Property """
|
||||
|
||||
@property
|
||||
def all_scalar_fields(self):
|
||||
dtypes = [
|
||||
DataType.INT8,
|
||||
DataType.INT16,
|
||||
DataType.INT32,
|
||||
DataType.INT64,
|
||||
DataType.VARCHAR,
|
||||
DataType.BOOL,
|
||||
DataType.FLOAT,
|
||||
DataType.DOUBLE,
|
||||
]
|
||||
dtype_names = [f"{n.name}" for n in dtypes] + [f"ARRAY_{n.name}" for n in dtypes] + [DataType.JSON.name]
|
||||
return dtype_names
|
||||
|
||||
@property
|
||||
def all_index_scalar_fields(self):
|
||||
return list(set(self.all_scalar_fields) - {DataType.JSON.name})
|
||||
|
||||
@property
|
||||
def inverted_support_dtype_names(self):
|
||||
return self.all_index_scalar_fields
|
||||
|
||||
@property
|
||||
def inverted_not_support_dtype_names(self):
|
||||
return [DataType.JSON.name]
|
||||
|
||||
@property
|
||||
def bitmap_support_dtype_names(self):
|
||||
dtypes = [DataType.INT8, DataType.INT16, DataType.INT32, DataType.INT64, DataType.BOOL, DataType.VARCHAR]
|
||||
dtype_names = [f"{n.name}" for n in dtypes] + [f"ARRAY_{n.name}" for n in dtypes]
|
||||
return dtype_names
|
||||
|
||||
@property
|
||||
def bitmap_not_support_dtype_names(self):
|
||||
return list(set(self.all_scalar_fields) - set(self.bitmap_support_dtype_names))
|
||||
|
||||
|
||||
class TestCaseClassBase(TestcaseBase):
|
||||
"""
|
||||
Setup objects on class
|
||||
"""
|
||||
|
||||
def setup_class(self):
|
||||
log.info("[setup_class] " + " Start setup class ".center(100, "~"))
|
||||
self._setup_objects(self)
|
||||
|
||||
def teardown_class(self):
|
||||
log.info("[teardown_class]" + " Start teardown class ".center(100, "~"))
|
||||
self._teardown_objects(self)
|
||||
|
||||
def setup_method(self, method):
|
||||
log.info(" setup ".center(80, "*"))
|
||||
log.info(f"[setup_method] Start setup test case {method.__name__}.")
|
||||
|
||||
def teardown_method(self, method):
|
||||
log.info(" teardown ".center(80, "*"))
|
||||
log.info(f"[teardown_method] Start teardown test case {method.__name__}...")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,387 @@
|
||||
import sys
|
||||
import time
|
||||
from numpy import NaN
|
||||
|
||||
from pymilvus import Collection
|
||||
|
||||
sys.path.append("..")
|
||||
from check.func_check import ResponseChecker
|
||||
from utils.api_request import api_request
|
||||
from utils.wrapper import trace
|
||||
from utils.util_log import test_log as log
|
||||
from pymilvus.orm.types import CONSISTENCY_STRONG
|
||||
from common.common_func import param_info
|
||||
|
||||
TIMEOUT = 180
|
||||
INDEX_NAME = ""
|
||||
|
||||
|
||||
# keep small timeout for stability tests
|
||||
# TIMEOUT = 5
|
||||
|
||||
|
||||
class ApiCollectionWrapper:
|
||||
collection = None
|
||||
|
||||
def __init__(self, active_trace=False):
|
||||
self.active_trace = active_trace
|
||||
|
||||
def init_collection(self, name, schema=None, using="default", check_task=None, check_items=None,
|
||||
active_trace=False, **kwargs):
|
||||
self.active_trace = active_trace
|
||||
consistency_level = kwargs.get("consistency_level", CONSISTENCY_STRONG)
|
||||
kwargs.update({"consistency_level": consistency_level})
|
||||
|
||||
""" In order to distinguish the same name of collection """
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([Collection, name, schema, using], **kwargs)
|
||||
self.collection = res if is_succ else None
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
name=name, schema=schema, using=using, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@property
|
||||
def schema(self):
|
||||
return self.collection.schema
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
return self.collection.description
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.collection.name
|
||||
|
||||
@property
|
||||
def is_empty(self):
|
||||
self.flush()
|
||||
return self.collection.is_empty
|
||||
|
||||
@property
|
||||
def is_empty_without_flush(self):
|
||||
return self.collection.is_empty
|
||||
|
||||
@property
|
||||
def num_entities(self):
|
||||
self.flush()
|
||||
return self.collection.num_entities
|
||||
|
||||
@property
|
||||
def num_shards(self):
|
||||
return self.collection.num_shards
|
||||
|
||||
@property
|
||||
def num_entities_without_flush(self):
|
||||
return self.collection.num_entities
|
||||
|
||||
@property
|
||||
def primary_field(self):
|
||||
return self.collection.primary_field
|
||||
|
||||
@property
|
||||
def aliases(self):
|
||||
return self.collection.aliases
|
||||
|
||||
@trace()
|
||||
def construct_from_dataframe(self, name, dataframe, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([Collection.construct_from_dataframe, name, dataframe], **kwargs)
|
||||
self.collection = res[0] if is_succ else None
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
name=name, dataframe=dataframe, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def drop(self, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.drop], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def load(self, partition_names=None, replica_number=NaN, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
replica_number = param_info.param_replica_num if replica_number is NaN else replica_number
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.load, partition_names, replica_number, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
partition_names=partition_names, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def release(self, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.release], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task,
|
||||
check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def insert(self, data, partition_name=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.insert, data, partition_name], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
data=data, partition_name=partition_name,
|
||||
**kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def flush(self, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.flush], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task,
|
||||
check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def search(self, data=None, anns_field=None, param=None, limit=None, expr=None,
|
||||
partition_names=None, output_fields=None, timeout=None, round_decimal=-1,
|
||||
check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.search, data, anns_field, param, limit,
|
||||
expr, partition_names, output_fields, timeout, round_decimal], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
data=data, anns_field=anns_field, param=param, limit=limit,
|
||||
expr=expr, partition_names=partition_names,
|
||||
output_fields=output_fields,
|
||||
timeout=timeout, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def hybrid_search(self, reqs, rerank, limit, partition_names=None,
|
||||
output_fields=None, timeout=None, round_decimal=-1,
|
||||
check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.hybrid_search, reqs, rerank, limit, partition_names,
|
||||
output_fields, timeout, round_decimal], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
reqs=reqs, rerank=rerank, limit=limit,
|
||||
partition_names=partition_names,
|
||||
output_fields=output_fields,
|
||||
timeout=timeout, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def search_iterator(self, data=None, anns_field=None, param=None, batch_size=None, limit=-1, expr=None,
|
||||
partition_names=None, output_fields=None, timeout=None, round_decimal=-1,
|
||||
check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.search_iterator, data, anns_field, param, batch_size, limit,
|
||||
expr, partition_names, output_fields, timeout, round_decimal], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
data=data, anns_field=anns_field, param=param, limit=limit,
|
||||
expr=expr, partition_names=partition_names,
|
||||
output_fields=output_fields,
|
||||
timeout=timeout, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def query(self, expr, output_fields=None, partition_names=None, timeout=None, check_task=None, check_items=None,
|
||||
**kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.query, expr, output_fields, partition_names, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
expression=expr, partition_names=partition_names,
|
||||
output_fields=output_fields,
|
||||
timeout=timeout, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def query_iterator(self, batch_size=1000, limit=-1, expr=None, output_fields=None, partition_names=None, timeout=None,
|
||||
check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.query_iterator, batch_size, limit, expr, output_fields, partition_names,
|
||||
timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
batch_size=batch_size, limit=limit, expression=expr,
|
||||
output_fields=output_fields, partition_names=partition_names,
|
||||
timeout=timeout, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@property
|
||||
def partitions(self):
|
||||
return self.collection.partitions
|
||||
|
||||
@trace()
|
||||
def partition(self, partition_name, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.collection.partition, partition_name])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items,
|
||||
succ, partition_name=partition_name).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def has_partition(self, partition_name, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.collection.has_partition, partition_name])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items,
|
||||
succ, partition_name=partition_name).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def drop_partition(self, partition_name, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.drop_partition, partition_name], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, partition_name=partition_name,
|
||||
**kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def create_partition(self, partition_name, check_task=None, check_items=None, description=""):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.create_partition, partition_name, description])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
partition_name=partition_name).run()
|
||||
return res, check_result
|
||||
|
||||
@property
|
||||
def indexes(self):
|
||||
return self.collection.indexes
|
||||
|
||||
@trace()
|
||||
def index(self, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
if "index_name" in kwargs:
|
||||
index_name = kwargs.get('index_name')
|
||||
kwargs.update({"index_name": index_name})
|
||||
res, check = api_request([self.collection.index], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def create_index(self, field_name, index_params=None, index_name=None, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = 1200 if timeout is None else timeout
|
||||
index_name = INDEX_NAME if index_name is None else index_name
|
||||
index_name = kwargs.get("index_name", index_name)
|
||||
kwargs.update({"index_name": index_name})
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.create_index, field_name, index_params, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def has_index(self, index_name=None, check_task=None, check_items=None, **kwargs):
|
||||
index_name = INDEX_NAME if index_name is None else index_name
|
||||
index_name = kwargs.get("index_name", index_name)
|
||||
kwargs.update({"index_name": index_name})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.has_index], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def drop_index(self, index_name=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
index_name = INDEX_NAME if index_name is None else index_name
|
||||
index_name = kwargs.get("index_name", index_name)
|
||||
kwargs.update({"timeout": timeout, "index_name": index_name})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.drop_index], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def delete(self, expr, partition_name=None, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.delete, expr, partition_name, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def upsert(self, data, partition_name=None, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.upsert, data, partition_name, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def compact(self, is_clustering=False, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.compact, is_clustering, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def get_compaction_state(self, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.get_compaction_state, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def get_compaction_plans(self, timeout=None, check_task=None, check_items={}, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.get_compaction_plans, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def wait_for_compaction_completed(self, timeout=None, **kwargs):
|
||||
timeout = TIMEOUT * 3 if timeout is None else timeout
|
||||
res = self.collection.wait_for_compaction_completed(timeout, **kwargs)
|
||||
# log.debug(res)
|
||||
return res
|
||||
|
||||
@trace()
|
||||
def get_replicas(self, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.get_replicas, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def describe(self, timeout=None, check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.describe, timeout])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def alter_index(self, index_name, extra_params={}, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.alter_index, index_name, extra_params, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@trace()
|
||||
def set_properties(self, extra_params={}, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.collection.set_properties, extra_params, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
@@ -0,0 +1,70 @@
|
||||
from pymilvus import Connections
|
||||
from pymilvus import DefaultConfig
|
||||
from pymilvus import MilvusClient
|
||||
import sys
|
||||
|
||||
sys.path.append("..")
|
||||
from check.func_check import ResponseChecker
|
||||
from utils.api_request import api_request
|
||||
|
||||
|
||||
class ApiConnectionsWrapper:
|
||||
def __init__(self):
|
||||
self.connection = Connections()
|
||||
|
||||
def add_connection(self, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.connection.add_connection], **kwargs)
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, **kwargs).run()
|
||||
return response, check_result
|
||||
|
||||
def disconnect(self, alias, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.connection.disconnect, alias])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, alias=alias).run()
|
||||
return response, check_result
|
||||
|
||||
def remove_connection(self, alias, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.connection.remove_connection, alias])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, alias=alias).run()
|
||||
return response, check_result
|
||||
|
||||
def connect(self, alias=DefaultConfig.DEFAULT_USING, user="", password="", db_name="default", token: str = "",
|
||||
check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, succ = api_request([self.connection.connect, alias, user, password, db_name, token], **kwargs)
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, succ, alias=alias, user=user,
|
||||
password=password, db_name=db_name, token=token, **kwargs).run()
|
||||
return response, check_result
|
||||
|
||||
def has_connection(self, alias=DefaultConfig.DEFAULT_USING, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, succ = api_request([self.connection.has_connection, alias])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, succ, alias=alias).run()
|
||||
return response, check_result
|
||||
|
||||
# def get_connection(self, alias=DefaultConfig.DEFAULT_USING, check_task=None, check_items=None):
|
||||
# func_name = sys._getframe().f_code.co_name
|
||||
# response, is_succ = api_request([self.connection.get_connection, alias])
|
||||
# check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, alias=alias).run()
|
||||
# return response, check_result
|
||||
|
||||
def list_connections(self, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.connection.list_connections])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
|
||||
return response, check_result
|
||||
|
||||
def get_connection_addr(self, alias, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.connection.get_connection_addr, alias])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, alias=alias).run()
|
||||
return response, check_result
|
||||
|
||||
# high level api
|
||||
def MilvusClient(self, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, succ = api_request([MilvusClient], **kwargs)
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, succ, **kwargs).run()
|
||||
return response, check_result
|
||||
@@ -0,0 +1,47 @@
|
||||
import sys
|
||||
|
||||
from pymilvus import db
|
||||
|
||||
from utils.api_request import api_request
|
||||
from check.func_check import ResponseChecker
|
||||
|
||||
|
||||
class ApiDatabaseWrapper:
|
||||
""" wrapper of database """
|
||||
database = db
|
||||
|
||||
def create_database(self, db_name, using="default", timeout=None, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.database.create_database, db_name, using, timeout])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
|
||||
return response, check_result
|
||||
|
||||
def using_database(self, db_name, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.database.using_database, db_name, using])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
|
||||
return response, check_result
|
||||
|
||||
def drop_database(self, db_name, using="default", timeout=None, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.database.drop_database, db_name, using, timeout])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
|
||||
return response, check_result
|
||||
|
||||
def list_database(self, using="default", timeout=None, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.database.list_database, using, timeout])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
|
||||
return response, check_result
|
||||
|
||||
def set_properties(self, db_name: str, properties: dict, using="default", timeout=None, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.database.set_properties, db_name, properties, using, timeout])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
|
||||
return response, check_result
|
||||
|
||||
def describe_database(self, db_name: str, using="default", timeout=None, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.database.describe_database, db_name, using, timeout])
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ).run()
|
||||
return response, check_result
|
||||
@@ -0,0 +1,53 @@
|
||||
import sys
|
||||
from pymilvus import Index
|
||||
|
||||
sys.path.append("..")
|
||||
from check.func_check import ResponseChecker
|
||||
from utils.api_request import api_request
|
||||
|
||||
TIMEOUT = 20
|
||||
INDEX_NAME = ""
|
||||
|
||||
|
||||
class ApiIndexWrapper:
|
||||
index = None
|
||||
|
||||
def init_index(self, collection, field_name, index_params, index_name=None, check_task=None, check_items=None,
|
||||
**kwargs):
|
||||
disktimeout = 600
|
||||
timeout = kwargs.get("timeout", disktimeout * 2)
|
||||
index_name = INDEX_NAME if index_name is None else index_name
|
||||
index_name = kwargs.get("index_name", index_name)
|
||||
kwargs.update({"timeout": timeout, "index_name": index_name})
|
||||
|
||||
""" In order to distinguish the same name of index """
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([Index, collection, field_name, index_params], **kwargs)
|
||||
self.index = res if is_succ is True else None
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection=collection, field_name=field_name,
|
||||
index_params=index_params, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def drop(self, index_name=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
index_name = INDEX_NAME if index_name is None else index_name
|
||||
index_name = kwargs.get("index_name", index_name)
|
||||
kwargs.update({"timeout": timeout, "index_name": index_name})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.index.drop], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@property
|
||||
def params(self):
|
||||
return self.index.params
|
||||
|
||||
@property
|
||||
def collection_name(self):
|
||||
return self.index.collection_name
|
||||
|
||||
@property
|
||||
def field_name(self):
|
||||
return self.index.field_name
|
||||
@@ -0,0 +1,159 @@
|
||||
import sys
|
||||
from numpy import NaN
|
||||
|
||||
from pymilvus import Partition
|
||||
|
||||
sys.path.append("..")
|
||||
from check.func_check import ResponseChecker
|
||||
from utils.api_request import api_request
|
||||
from common.common_func import param_info
|
||||
|
||||
|
||||
TIMEOUT = 180
|
||||
|
||||
|
||||
class ApiPartitionWrapper:
|
||||
partition = None
|
||||
|
||||
def init_partition(self, collection, name, description="",
|
||||
check_task=None, check_items=None, **kwargs):
|
||||
""" In order to distinguish the same name of partition """
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([Partition, collection, name, description], **kwargs)
|
||||
self.partition = response if is_succ is True else None
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ,
|
||||
**kwargs).run()
|
||||
return response, check_result
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
return self.partition.description if self.partition else None
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.partition.name if self.partition else None
|
||||
|
||||
@property
|
||||
def is_empty(self):
|
||||
self.flush()
|
||||
return self.partition.is_empty if self.partition else None
|
||||
|
||||
@property
|
||||
def is_empty_without_flush(self):
|
||||
return self.partition.is_empty if self.partition else None
|
||||
|
||||
@property
|
||||
def num_entities(self):
|
||||
self.flush()
|
||||
return self.partition.num_entities if self.partition else None
|
||||
|
||||
@property
|
||||
def num_entities_without_flush(self):
|
||||
return self.partition.num_entities if self.partition else None
|
||||
|
||||
def drop(self, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.partition.drop], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name,
|
||||
check_task, check_items, succ, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def load(self, replica_number=NaN, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
replica_number = param_info.param_replica_num if replica_number is NaN else replica_number
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.partition.load, replica_number, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task,
|
||||
check_items, is_succ=succ,
|
||||
**kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def release(self, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.partition.release], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task,
|
||||
check_items, is_succ=succ,
|
||||
**kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def flush(self, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.partition.flush], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task,
|
||||
check_items, is_succ=succ,
|
||||
**kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def insert(self, data, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.partition.insert, data], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task,
|
||||
check_items, is_succ=succ, data=data,
|
||||
**kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def search(self, data, anns_field, params, limit, expr=None, output_fields=None,
|
||||
check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.partition.search, data, anns_field, params,
|
||||
limit, expr, output_fields], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items,
|
||||
is_succ=succ, data=data, anns_field=anns_field,
|
||||
params=params, limit=limit, expr=expr,
|
||||
output_fields=output_fields, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def query(self, expr, output_fields=None, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.partition.query, expr, output_fields, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
expression=expr, output_fields=output_fields,
|
||||
timeout=timeout, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def delete(self, expr, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.partition.delete, expr], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task,
|
||||
check_items, is_succ=succ, expr=expr,
|
||||
**kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def upsert(self, data, check_task=None, check_items=None, **kwargs):
|
||||
timeout = kwargs.get("timeout", TIMEOUT)
|
||||
kwargs.update({"timeout": timeout})
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, succ = api_request([self.partition.upsert, data], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task,
|
||||
check_items, is_succ=succ, data=data,
|
||||
**kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def get_replicas(self, timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.partition.get_replicas, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
@@ -0,0 +1,83 @@
|
||||
import sys
|
||||
|
||||
sys.path.append("..")
|
||||
from check.func_check import ResponseChecker
|
||||
from utils.api_request import api_request
|
||||
from pymilvus import CollectionSchema, FieldSchema
|
||||
|
||||
|
||||
class ApiCollectionSchemaWrapper:
|
||||
collection_schema = None
|
||||
|
||||
def init_collection_schema(self, fields, description="", check_task=None, check_items=None, **kwargs):
|
||||
"""In order to distinguish the same name of CollectionSchema"""
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([CollectionSchema, fields, description], **kwargs)
|
||||
self.collection_schema = response if is_succ else None
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ=is_succ, fields=fields,
|
||||
description=description, **kwargs).run()
|
||||
return response, check_result
|
||||
|
||||
@property
|
||||
def primary_field(self):
|
||||
return self.collection_schema.primary_field if self.collection_schema else None
|
||||
|
||||
@property
|
||||
def partition_key_field(self):
|
||||
return self.collection_schema.partition_key_field if self.collection_schema else None
|
||||
|
||||
@property
|
||||
def fields(self):
|
||||
return self.collection_schema.fields if self.collection_schema else None
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
return self.collection_schema.description if self.collection_schema else None
|
||||
|
||||
@property
|
||||
def auto_id(self):
|
||||
return self.collection_schema.auto_id if self.collection_schema else None
|
||||
|
||||
@property
|
||||
def enable_dynamic_field(self):
|
||||
return self.collection_schema.enable_dynamic_field if self.collection_schema else None
|
||||
|
||||
@property
|
||||
def to_dict(self):
|
||||
return self.collection_schema.to_dict if self.collection_schema else None
|
||||
|
||||
@property
|
||||
def verify(self):
|
||||
return self.collection_schema.verify if self.collection_schema else None
|
||||
|
||||
def add_field(self, field_name, datatype, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([self.collection_schema.add_field, field_name, datatype], **kwargs)
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items,
|
||||
field_name=field_name, datatype=datatype, **kwargs).run()
|
||||
return response, check_result
|
||||
|
||||
|
||||
class ApiFieldSchemaWrapper:
|
||||
field_schema = None
|
||||
|
||||
def init_field_schema(self, name, dtype, description="", check_task=None, check_items=None, **kwargs):
|
||||
"""In order to distinguish the same name of FieldSchema"""
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
response, is_succ = api_request([FieldSchema, name, dtype, description], **kwargs)
|
||||
self.field_schema = response if is_succ else None
|
||||
check_result = ResponseChecker(response, func_name, check_task, check_items, is_succ, name=name, dtype=dtype,
|
||||
description=description, **kwargs).run()
|
||||
return response, check_result
|
||||
|
||||
@property
|
||||
def description(self):
|
||||
return self.field_schema.description if self.field_schema else None
|
||||
|
||||
@property
|
||||
def params(self):
|
||||
return self.field_schema.params if self.field_schema else None
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
return self.field_schema.dtype if self.field_schema else None
|
||||
@@ -0,0 +1,602 @@
|
||||
from datetime import datetime
|
||||
import time
|
||||
from pymilvus import utility
|
||||
import sys
|
||||
|
||||
sys.path.append("..")
|
||||
from check.func_check import ResponseChecker
|
||||
from utils.api_request import api_request
|
||||
from pymilvus import BulkInsertState
|
||||
from pymilvus.orm.role import Role
|
||||
from utils.util_log import test_log as log
|
||||
|
||||
TIMEOUT = 20
|
||||
|
||||
|
||||
class ApiUtilityWrapper:
|
||||
""" Method of encapsulating utility files """
|
||||
|
||||
ut = utility
|
||||
role = None
|
||||
|
||||
def do_bulk_insert(self, collection_name, files="", partition_name=None, timeout=None,
|
||||
using="default", check_task=None, check_items=None, **kwargs):
|
||||
working_tasks = self.get_bulk_insert_working_list()
|
||||
log.info(f"before bulk load, there are {len(working_tasks)} working tasks")
|
||||
log.info(f"files to load: {files}")
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.do_bulk_insert, collection_name,
|
||||
files, partition_name, timeout, using], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection_name=collection_name, using=using).run()
|
||||
time.sleep(1)
|
||||
working_tasks = self.get_bulk_insert_working_list()
|
||||
log.info(f"after bulk load, there are {len(working_tasks)} working tasks")
|
||||
return res, check_result
|
||||
|
||||
def get_bulk_insert_state(self, task_id, timeout=None, using="default", check_task=None, check_items=None,
|
||||
**kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.get_bulk_insert_state, task_id, timeout, using], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
task_id=task_id, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def list_bulk_insert_tasks(self, limit=0, collection_name=None, timeout=None, using="default", check_task=None,
|
||||
check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.list_bulk_insert_tasks, limit, collection_name, timeout, using], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
limit=limit, collection_name=collection_name, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def get_bulk_insert_pending_list(self):
|
||||
tasks = {}
|
||||
for task in self.ut.list_bulk_insert_tasks():
|
||||
if task.state == BulkInsertState.ImportPending:
|
||||
tasks[task.task_id] = task
|
||||
return tasks
|
||||
|
||||
def get_bulk_insert_working_list(self):
|
||||
tasks = {}
|
||||
for task in self.ut.list_bulk_insert_tasks():
|
||||
if task.state in [BulkInsertState.ImportStarted]:
|
||||
tasks[task.task_id] = task
|
||||
return tasks
|
||||
|
||||
def list_all_bulk_insert_tasks(self, limit=0):
|
||||
tasks, _ = self.list_bulk_insert_tasks(limit=limit)
|
||||
pending = 0
|
||||
started = 0
|
||||
persisted = 0
|
||||
completed = 0
|
||||
failed = 0
|
||||
failed_and_cleaned = 0
|
||||
unknown = 0
|
||||
for task in tasks:
|
||||
print(task)
|
||||
if task.state == BulkInsertState.ImportPending:
|
||||
pending = pending + 1
|
||||
elif task.state == BulkInsertState.ImportStarted:
|
||||
started = started + 1
|
||||
elif task.state == BulkInsertState.ImportPersisted:
|
||||
persisted = persisted + 1
|
||||
elif task.state == BulkInsertState.ImportCompleted:
|
||||
completed = completed + 1
|
||||
elif task.state == BulkInsertState.ImportFailed:
|
||||
failed = failed + 1
|
||||
elif task.state == BulkInsertState.ImportFailedAndCleaned:
|
||||
failed_and_cleaned = failed_and_cleaned + 1
|
||||
else:
|
||||
unknown = unknown + 1
|
||||
|
||||
log.info("There are", len(tasks), "bulkload tasks.", pending, "pending,", started, "started,", persisted,
|
||||
"persisted,", completed, "completed,", failed, "failed", failed_and_cleaned, "failed_and_cleaned",
|
||||
unknown, "unknown")
|
||||
|
||||
def wait_for_bulk_insert_tasks_completed(self, task_ids, target_state=BulkInsertState.ImportCompleted,
|
||||
timeout=None, using="default", **kwargs):
|
||||
tasks_state_distribution = {
|
||||
"success": set(),
|
||||
"failed": set(),
|
||||
"in_progress": set()
|
||||
}
|
||||
tasks_state = {}
|
||||
if timeout is not None:
|
||||
task_timeout = timeout
|
||||
else:
|
||||
task_timeout = TIMEOUT
|
||||
start = time.time()
|
||||
end = time.time()
|
||||
log.info(f"wait bulk load timeout is {task_timeout}")
|
||||
pending_tasks = self.get_bulk_insert_pending_list()
|
||||
log.info(f"before waiting, there are {len(pending_tasks)} pending tasks")
|
||||
while len(tasks_state_distribution["success"]) + len(tasks_state_distribution["failed"]) < len(
|
||||
task_ids) and end - start <= task_timeout:
|
||||
time.sleep(2)
|
||||
|
||||
for task_id in task_ids:
|
||||
if task_id in tasks_state_distribution["success"] or task_id in tasks_state_distribution["failed"]:
|
||||
continue
|
||||
else:
|
||||
state, _ = self.get_bulk_insert_state(task_id, task_timeout, using, **kwargs)
|
||||
tasks_state[task_id] = state
|
||||
|
||||
if target_state == BulkInsertState.ImportPersisted:
|
||||
if state.state in [BulkInsertState.ImportPersisted, BulkInsertState.ImportCompleted]:
|
||||
if task_id in tasks_state_distribution["in_progress"]:
|
||||
tasks_state_distribution["in_progress"].remove(task_id)
|
||||
tasks_state_distribution["success"].add(task_id)
|
||||
elif state.state in [BulkInsertState.ImportPending, BulkInsertState.ImportStarted]:
|
||||
tasks_state_distribution["in_progress"].add(task_id)
|
||||
else:
|
||||
tasks_state_distribution["failed"].add(task_id)
|
||||
|
||||
if target_state == BulkInsertState.ImportCompleted:
|
||||
if state.state in [BulkInsertState.ImportCompleted]:
|
||||
if task_id in tasks_state_distribution["in_progress"]:
|
||||
tasks_state_distribution["in_progress"].remove(task_id)
|
||||
tasks_state_distribution["success"].add(task_id)
|
||||
elif state.state in [BulkInsertState.ImportPending, BulkInsertState.ImportStarted,
|
||||
BulkInsertState.ImportPersisted]:
|
||||
tasks_state_distribution["in_progress"].add(task_id)
|
||||
else:
|
||||
tasks_state_distribution["failed"].add(task_id)
|
||||
|
||||
end = time.time()
|
||||
pending_tasks = self.get_bulk_insert_pending_list()
|
||||
log.info(f"after waiting, there are {len(pending_tasks)} pending tasks")
|
||||
log.info(f"task state distribution: {tasks_state_distribution}")
|
||||
log.info(tasks_state)
|
||||
if len(tasks_state_distribution["success"]) == len(task_ids):
|
||||
log.info(f"wait for bulk load tasks completed successfully, cost time: {end - start}")
|
||||
return True, tasks_state
|
||||
else:
|
||||
log.info(f"wait for bulk load tasks completed failed, cost time: {end - start}")
|
||||
return False, tasks_state
|
||||
|
||||
def wait_all_pending_tasks_finished(self):
|
||||
task_states_map = {}
|
||||
all_tasks, _ = self.list_bulk_insert_tasks()
|
||||
# log.info(f"all tasks: {all_tasks}")
|
||||
for task in all_tasks:
|
||||
if task.state in [BulkInsertState.ImportStarted, BulkInsertState.ImportPersisted]:
|
||||
task_states_map[task.task_id] = task.state
|
||||
|
||||
log.info(f"current tasks states: {task_states_map}")
|
||||
pending_tasks = self.get_bulk_insert_pending_list()
|
||||
working_tasks = self.get_bulk_insert_working_list()
|
||||
log.info(
|
||||
f"in the start, there are {len(working_tasks)} working tasks, {working_tasks} {len(pending_tasks)} pending tasks, {pending_tasks}")
|
||||
time_cnt = 0
|
||||
pending_task_ids = set()
|
||||
while len(pending_tasks) > 0:
|
||||
time.sleep(5)
|
||||
time_cnt += 5
|
||||
pending_tasks = self.get_bulk_insert_pending_list()
|
||||
working_tasks = self.get_bulk_insert_working_list()
|
||||
cur_pending_task_ids = []
|
||||
for task_id in pending_tasks.keys():
|
||||
cur_pending_task_ids.append(task_id)
|
||||
pending_task_ids.add(task_id)
|
||||
log.info(
|
||||
f"after {time_cnt}, there are {len(working_tasks)} working tasks, {len(pending_tasks)} pending tasks")
|
||||
log.debug(f"total pending tasks: {pending_task_ids} current pending tasks: {cur_pending_task_ids}")
|
||||
log.info(f"after {time_cnt}, all pending tasks are finished")
|
||||
all_tasks, _ = self.list_bulk_insert_tasks()
|
||||
for task in all_tasks:
|
||||
if task.task_id in pending_task_ids:
|
||||
log.info(f"task {task.task_id} state transfer from pending to {task.state_name}")
|
||||
|
||||
def wait_index_build_completed(self, collection_name, timeout=None):
|
||||
start = time.time()
|
||||
if timeout is not None:
|
||||
task_timeout = timeout
|
||||
else:
|
||||
task_timeout = TIMEOUT
|
||||
end = time.time()
|
||||
while end - start <= task_timeout:
|
||||
time.sleep(0.5)
|
||||
index_states, _ = self.index_building_progress(collection_name)
|
||||
log.debug(f"index states: {index_states}")
|
||||
if index_states["total_rows"] == index_states["indexed_rows"]:
|
||||
log.info(f"index build completed")
|
||||
return True
|
||||
end = time.time()
|
||||
log.info(f"index build timeout")
|
||||
return False
|
||||
|
||||
def get_query_segment_info(self, collection_name, timeout=None, using="default", check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.get_query_segment_info, collection_name, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection_name=collection_name, timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def loading_progress(self, collection_name, partition_names=None,
|
||||
using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.loading_progress, collection_name, partition_names, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task,
|
||||
check_items, is_succ, collection_name=collection_name,
|
||||
partition_names=partition_names, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def load_state(self, collection_name, partition_names=None, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.load_state, collection_name, partition_names, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection_name=collection_name, partition_names=partition_names,
|
||||
using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def wait_for_loading_complete(self, collection_name, partition_names=None, timeout=None, using="default",
|
||||
check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.wait_for_loading_complete, collection_name,
|
||||
partition_names, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection_name=collection_name, partition_names=partition_names,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def index_building_progress(self, collection_name, index_name="", using="default",
|
||||
check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.index_building_progress, collection_name, index_name, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection_name=collection_name, index_name=index_name,
|
||||
using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def wait_for_index_building_complete(self, collection_name, index_name="", timeout=None, using="default",
|
||||
check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.wait_for_index_building_complete, collection_name,
|
||||
index_name, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection_name=collection_name, index_name=index_name,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def has_collection(self, collection_name, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.has_collection, collection_name, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection_name=collection_name, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def has_partition(self, collection_name, partition_name, using="default",
|
||||
check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.has_partition, collection_name, partition_name, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection_name=collection_name,
|
||||
partition_name=partition_name, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def drop_collection(self, collection_name, timeout=None, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.drop_collection, collection_name, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
collection_name=collection_name,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def list_collections(self, timeout=None, using="default", check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.list_collections, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def calc_distance(self, vectors_left, vectors_right, params=None, timeout=None,
|
||||
using="default", check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.calc_distance, vectors_left, vectors_right,
|
||||
params, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def load_balance(self, collection_name, src_node_id, dst_node_ids, sealed_segment_ids, timeout=None,
|
||||
using="default", check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.load_balance, collection_name, src_node_id, dst_node_ids,
|
||||
sealed_segment_ids, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def create_alias(self, collection_name, alias, timeout=None, using="default", check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.create_alias, collection_name, alias, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def drop_alias(self, alias, timeout=None, using="default", check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.drop_alias, alias, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def alter_alias(self, collection_name, alias, timeout=None, using="default", check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.alter_alias, collection_name, alias, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def list_aliases(self, collection_name, timeout=None, using="default", check_task=None, check_items=None):
|
||||
timeout = TIMEOUT if timeout is None else timeout
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.list_aliases, collection_name, timeout, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
timeout=timeout, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def mkts_from_datetime(self, d_time=None, milliseconds=0., delta=None):
|
||||
d_time = datetime.now() if d_time is None else d_time
|
||||
res, _ = api_request([self.ut.mkts_from_datetime, d_time, milliseconds, delta])
|
||||
return res
|
||||
|
||||
def mkts_from_hybridts(self, hybridts, milliseconds=0., delta=None):
|
||||
res, _ = api_request([self.ut.mkts_from_hybridts, hybridts, milliseconds, delta])
|
||||
return res
|
||||
|
||||
def create_user(self, user, password, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.create_user, user, password, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def list_usernames(self, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.list_usernames, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def reset_password(self, user, old_password, new_password, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.reset_password, user, old_password, new_password])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ).run()
|
||||
return res, check_result
|
||||
|
||||
def update_password(self, user, old_password, new_password, check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.update_password, user, old_password, new_password])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ).run()
|
||||
return res, check_result
|
||||
|
||||
def delete_user(self, user, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.delete_user, user, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def list_roles(self, include_user_info: bool, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.list_roles, include_user_info, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def list_user(self, username: str, include_role_info: bool, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.list_user, username, include_role_info, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def list_users(self, include_role_info: bool, using="default", check_task=None, check_items=None):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.ut.list_users, include_role_info, using])
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ, using=using).run()
|
||||
return res, check_result
|
||||
|
||||
def init_role(self, name, using="default", check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([Role, name, using], **kwargs)
|
||||
self.role = res if is_succ else None
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
name=name, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def create_role(self, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, is_succ = api_request([self.role.create], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, is_succ,
|
||||
**kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_drop(self, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.drop], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_is_exist(self, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.is_exist], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_add_user(self, username: str, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.add_user, username], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_remove_user(self, username: str, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.remove_user, username], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_get_users(self, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.get_users], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
@property
|
||||
def role_name(self):
|
||||
return self.role.name
|
||||
|
||||
def role_grant(self, object: str, object_name: str, privilege: str, db_name: str = "", check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.grant, object, object_name, privilege, db_name], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_revoke(self, object: str, object_name: str, privilege: str, db_name: str = "", check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.revoke, object, object_name, privilege, db_name], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_grant_v2(self, privilege: str, collection_name: str, db_name: str = None, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.grant_v2, privilege, collection_name, db_name], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_revoke_v2(self, privilege: str, collection_name: str, db_name: str = None, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.revoke_v2, privilege, collection_name, db_name], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_list_grant(self, object: str, object_name: str, db_name: str = "", check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.list_grant, object, object_name, db_name], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def role_list_grants(self, db_name: str = "", check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.list_grants, db_name], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def create_resource_group(self, name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.create_resource_group, name, using, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def drop_resource_group(self, name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.drop_resource_group, name, using, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def list_resource_groups(self, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.list_resource_groups, using, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def describe_resource_group(self, name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.describe_resource_group, name, using, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def update_resource_group(self, config, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.update_resource_groups, config, using, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def transfer_node(self, source, target, num_node, using="default", timeout=None, check_task=None, check_items=None,
|
||||
**kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.transfer_node, source, target, num_node, using, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def transfer_replica(self, source, target, collection_name, num_replica, using="default", timeout=None,
|
||||
check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request(
|
||||
[self.ut.transfer_replica, source, target, collection_name, num_replica, using, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def rename_collection(self, old_collection_name, new_collection_name, new_db_name="", timeout=None,
|
||||
check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.rename_collection, old_collection_name, new_collection_name, new_db_name,
|
||||
timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
old_collection_name=old_collection_name, new_collection_name=new_collection_name,
|
||||
new_db_name=new_db_name, timeout=timeout, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def flush_all(self, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.flush_all, using, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
using=using, timeout=timeout, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def get_server_type(self, using="default", check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.get_server_type, using], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
using=using, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def list_indexes(self, collection_name, using="default", timeout=None, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.ut.list_indexes, collection_name, using, timeout], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check,
|
||||
collection_name=collection_name, using=using, timeout=timeout, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def create_privilege_group(self, privilege_group: str, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.create_privilege_group, privilege_group], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def drop_privilege_group(self, privilege_group: str, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.drop_privilege_group, privilege_group], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def list_privilege_groups(self, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.list_privilege_groups], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def add_privileges_to_group(self, privilege_group: str, privileges: list, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.add_privileges_to_group, privilege_group, privileges], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
|
||||
def remove_privileges_from_group(self, privilege_group: str, privileges: list, check_task=None, check_items=None, **kwargs):
|
||||
func_name = sys._getframe().f_code.co_name
|
||||
res, check = api_request([self.role.remove_privileges_from_group, privilege_group, privileges], **kwargs)
|
||||
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()
|
||||
return res, check_result
|
||||
Reference in New Issue
Block a user