chore: import upstream snapshot with attribution

tests/python_client/deploy/scripts/action_after_reinstall.py

@@ -0,0 +1,47 @@
+from pymilvus import connections
+from utils import *
+
+
+def task_1(data_size, host):
+    """
+    task_1:
+        before reinstall: create collection, insert data, create index and insert data, load and search
+        after reinstall: get collection, load, search, release, insert data, create index, load, and search
+    """
+    prefix = "task_1_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    load_and_search(prefix)
+    release_collection(prefix)
+    create_collections_and_insert_data(prefix,count=data_size)
+    load_and_search(prefix)
+
+
+def task_2(data_zise, host):
+    """
+    task_2:
+        before reinstall: create collection, insert data and create index, load and search
+        after reinstall: get collection, load, search, insert data, release, create index, load, and search
+    """
+    prefix = "task_2_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    release_collection(prefix)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+if __name__ == '__main__':
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for deploy test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    parser.add_argument('--data_size', type=int, default=3000, help='data size')
+    args = parser.parse_args()
+    host = args.host 
+    data_size = args.data_size
+    logger.info(f"data size: {data_size}")
+    task_1(data_size, host)
+    task_2(data_size, host)

tests/python_client/deploy/scripts/action_after_upgrade.py

@@ -0,0 +1,114 @@
+from pymilvus import connections
+import sys
+sys.path.append("..")
+sys.path.append("../..")
+from common.milvus_sys import MilvusSys
+from utils import *
+
+
+def task_1(data_size, host):
+    """
+    task_1:
+        before upgrade: create collection and insert data with flush, create index, load and search
+        after upgrade: get collection, load, search, insert data with flush, release, create index, load, and search
+    """
+    prefix = "task_1_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    create_index(prefix)
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    release_collection(prefix)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+def task_2(data_size, host):
+    """
+    task_2:
+        before upgrade: create collection, insert data and create index, load and search
+        after upgrade: get collection, load, search, insert data, release, create index, load, and search
+    """
+    prefix = "task_2_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    release_collection(prefix)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+def task_3(data_size, host):
+    """
+    task_3:
+        before upgrade: create collection, insert data, flush, create index, load with one replicas and search
+        after upgrade: get collection, load, search, insert data, release, create index, load with multi replicas, and search
+    """
+    prefix = "task_3_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    release_collection(prefix)
+    create_index(prefix)
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+
+
+def task_4(data_size, host):
+    """
+    task_4:
+        before upgrade: create collection, insert data, flush, and create index
+        after upgrade: get collection, load with multi replicas, search, insert data, load with multi replicas and search
+    """
+    prefix = "task_4_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+    create_collections_and_insert_data(prefix, flush=False, count=data_size)
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+
+
+def task_5(data_size, host):
+    """
+    task_5_:
+        before upgrade: create collection and insert data without flush
+        after upgrade: get collection, create index, load with multi replicas, search, insert data with flush, load with multi replicas and search
+    """
+    prefix = "task_5_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    create_index(prefix)
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+    create_collections_and_insert_data(prefix, flush=True, count=data_size)
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+
+
+if __name__ == '__main__':
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for deploy test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    parser.add_argument('--data_size', type=int, default=3000, help='data size')
+    args = parser.parse_args()
+    data_size = args.data_size
+    host = args.host
+    logger.info(f"data size: {data_size}")
+    connections.connect(host=host, port=19530, timeout=60)
+    ms = MilvusSys()
+    # create index for flat
+    logger.info("create index for flat start")
+    create_index_flat()
+    logger.info("create index for flat done")
+    task_1(data_size, host)
+    task_2(data_size, host)
+    if len(ms.query_nodes) >= NUM_REPLICAS:
+        task_3(data_size, host)
+        task_4(data_size, host)
+        task_5(data_size, host)
+

tests/python_client/deploy/scripts/action_before_reinstall.py

@@ -0,0 +1,48 @@
+from pymilvus import connections
+from utils import *
+
+
+def task_1(data_size, host):
+    """
+    task_1:
+        before reinstall: create collection, insert data, create index and insert data, load and search
+        after reinstall: get collection, load, search, release, insert data, create index, load, and search
+    """
+    prefix = "task_1_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix,count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix,count=data_size)
+    load_and_search(prefix)
+
+
+def task_2(data_size, host):
+    """
+    task_2:
+        before reinstall: create collection, insert data, create index, insert data, create index,load and search
+        after reinstall: get collection, load, search, insert data, create index, load, and search
+    """
+    prefix = "task_2_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+if __name__ == '__main__':
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for deploy test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    parser.add_argument('--data_size', type=int, default=3000, help='data size')
+    args = parser.parse_args()
+    data_size = args.data_size
+    host = args.host
+    logger.info(f"data_size: {data_size}")
+    task_1(data_size, host)
+    task_2(data_size, host)

tests/python_client/deploy/scripts/action_before_upgrade.py

@@ -0,0 +1,91 @@
+from pymilvus import connections
+import sys
+sys.path.append("..")
+sys.path.append("../..")
+from common.milvus_sys import MilvusSys
+from utils import *
+
+
+def task_1(data_size, host):
+    """
+    task_1:
+        before upgrade: create collection and insert data with flush, create index, load and search
+        after upgrade: get collection, load, search, insert data with flush, release, create index, load, and search
+    """
+    prefix = "task_1_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+def task_2(data_size, host):
+    """
+    task_2:
+        before upgrade: create collection, insert data and create index, load , search, and insert data without flush
+        after upgrade: get collection, load, search, insert data, release, create index, load, and search
+    """
+    prefix = "task_2_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, flush=False, count=data_size)
+
+def task_3(data_size, host):
+    """
+    task_3:
+        before upgrade: create collection, insert data, flush, create index, load with one replicas and search
+        after upgrade: get collection, load, search, insert data, create index, release, load with multi replicas, and search
+    """
+    prefix = "task_3_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+
+def task_4(data_size, host):
+    """
+    task_4_:
+        before upgrade: create collection, insert data, flush, and create index
+        after upgrade: get collection, load with multi replicas, search, insert data, load with multi replicas and search
+    """
+    prefix = "task_4_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, flush=True, count=data_size)
+    create_index(prefix)
+
+def task_5(data_size, host):
+    """
+    task_5_:
+        before upgrade: create collection and insert data without flush
+        after upgrade: get collection, create index, load with multi replicas, search, insert data with flush, load with multi replicas and search
+    """
+    prefix = "task_5_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, flush=False, count=data_size)
+
+
+if __name__ == '__main__':
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for deploy test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    parser.add_argument('--data_size', type=int, default=3000, help='data size')
+    args = parser.parse_args()
+    data_size = args.data_size
+    host = args.host
+    logger.info(f"data size: {data_size}")
+    connections.connect(host=host, port=19530, timeout=60)
+    ms = MilvusSys()
+    task_1(data_size, host)
+    task_2(data_size, host)
+    if len(ms.query_nodes) >= NUM_REPLICAS:
+        task_3(data_size, host)
+        task_4(data_size, host)
+        task_5(data_size, host)

tests/python_client/deploy/scripts/breakdown_rolling_update.py

@@ -0,0 +1,53 @@
+import psutil
+import time
+from loguru import logger
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description='config for rolling update process')
+    parser.add_argument('--wait_time', type=int, default=60, help='wait time after rolling update started')
+    args = parser.parse_args()
+    wait_time = args.wait_time
+    logger.info("start to watch rolling update process")
+    start_time = time.time()
+    end_time = time.time()
+    flag = False
+    while not flag and end_time - start_time < 360: 
+        process_list = [p.info for p in psutil.process_iter(attrs=['pid', 'name','cmdline'])]
+        for process in process_list:
+            logger.debug(process)
+            logger.debug("##"*30)
+        for process in process_list:
+            if isinstance(process.get("cmdline", []), list):
+                cmdline_list = process.get("cmdline", [])
+                for cmdline in cmdline_list:
+                    if "rollingUpdate.sh" in cmdline:
+                        logger.info(f"rolling update process: {process} started")
+                        flag = True
+                        break
+            if flag:
+                break
+        time.sleep(0.5)
+        end_time = time.time()
+        if not flag:
+            logger.info(f"rolling update process not found, wait for {end_time - start_time} seconds")
+        else:
+            logger.info(f"rolling update process {process} found, wait for {end_time - start_time} seconds")
+    if flag:
+        logger.info(f"wait {wait_time}s to kill rolling update process")
+        time.sleep(wait_time)
+        logger.info("start to kill rolling update process")
+        try:
+            p = psutil.Process(process["pid"])
+            p.terminate()
+            logger.info(f"rolling update process: {process} killed")
+        except Exception as e:
+            logger.error(f"rolling update process: {process} kill failed, {e}")
+    else:
+        logger.info("all process info")
+        for process in process_list:
+            logger.info(process)
+            
+            
+

tests/python_client/deploy/scripts/first_recall_test.py

@@ -0,0 +1,201 @@
+import threading
+import h5py
+import numpy as np
+import time
+import sys
+import copy
+from pathlib import Path
+from loguru import logger
+import pymilvus
+from pymilvus import (
+    connections,
+    FieldSchema, CollectionSchema, DataType,
+    Collection, utility
+)
+
+pymilvus_version = pymilvus.__version__
+
+
+all_index_types = ["IVF_FLAT", "IVF_SQ8", "HNSW"]
+default_index_params = [{"nlist": 128}, {"nlist": 128}, {"M": 48, "efConstruction": 200}]
+index_params_map = dict(zip(all_index_types, default_index_params))
+
+
+def gen_index_params(index_type, metric_type="L2"):
+    default_index = {"index_type": "IVF_FLAT", "params": {"nlist": 128}, "metric_type": metric_type}
+    index = copy.deepcopy(default_index)
+    index["index_type"] = index_type
+    index["params"] = index_params_map[index_type]
+    if index_type in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+        index["metric_type"] = "HAMMING"
+    return index
+
+
+def gen_search_param(index_type, metric_type="L2"):
+    search_params = []
+    if index_type in ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"]:
+        for nprobe in [10]:
+            ivf_search_params = {"metric_type": metric_type, "params": {"nprobe": nprobe}}
+            search_params.append(ivf_search_params)
+    elif index_type in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+        for nprobe in [10]:
+            bin_search_params = {"metric_type": "HAMMING", "params": {"nprobe": nprobe}}
+            search_params.append(bin_search_params)
+    elif index_type in ["HNSW"]:
+        for ef in [150]:
+            hnsw_search_param = {"metric_type": metric_type, "params": {"ef": ef}}
+            search_params.append(hnsw_search_param)
+    elif index_type == "ANNOY":
+        for search_k in [1000]:
+            annoy_search_param = {"metric_type": metric_type, "params": {"search_k": search_k}}
+            search_params.append(annoy_search_param)
+    else:
+        logger.info("Invalid index_type.")
+        raise Exception("Invalid index_type.")
+    return search_params[0]
+
+
+def read_benchmark_hdf5(file_path):
+
+    f = h5py.File(file_path, 'r')
+    train = np.array(f["train"])
+    test = np.array(f["test"])
+    neighbors = np.array(f["neighbors"])
+    f.close()
+    return train, test, neighbors
+
+
+dim = 128
+TIMEOUT = 200
+
+
+def milvus_recall_test(host='127.0.0.1', index_type="HNSW"):
+    logger.info(f"recall test for index type {index_type}")
+    file_path = f"{str(Path(__file__).absolute().parent.parent.parent)}/assets/ann_hdf5/sift-128-euclidean.hdf5"
+    train, test, neighbors = read_benchmark_hdf5(file_path)
+    connections.connect(host=host, port="19530")
+    default_fields = [
+        FieldSchema(name="int64", dtype=DataType.INT64, is_primary=True),
+        FieldSchema(name="float", dtype=DataType.FLOAT),
+        FieldSchema(name="varchar", dtype=DataType.VARCHAR, max_length=65535),
+        FieldSchema(name="float_vector", dtype=DataType.FLOAT_VECTOR, dim=dim)
+    ]
+    default_schema = CollectionSchema(
+        fields=default_fields, description="test collection")
+
+    name = f"sift_128_euclidean_{index_type}"
+    logger.info(f"Create collection {name}")
+    collection = Collection(name=name, schema=default_schema)
+    nb = len(train)
+    batch_size = 50000
+    epoch = int(nb / batch_size)
+    t0 = time.time()
+    for i in range(epoch):
+        logger.info(f"epoch: {i}")
+        start = i * batch_size
+        end = (i + 1) * batch_size
+        if end > nb:
+            end = nb
+        data = [
+            [i for i in range(start, end)],
+            [np.float32(i) for i in range(start, end)],
+            [str(i) for i in range(start, end)],
+            train[start:end]
+        ]
+        collection.insert(data)
+    t1 = time.time()
+    logger.info(f"Insert {nb} vectors cost {t1 - t0:.4f} seconds")
+
+    t0 = time.time()
+    logger.info(f"Get collection entities...")
+    if pymilvus_version >= "2.2.0":
+        collection.flush()
+    else:
+        collection.num_entities
+    logger.info(collection.num_entities)
+    t1 = time.time()
+    logger.info(f"Get collection entities cost {t1 - t0:.4f} seconds")
+
+    # create index
+    default_index = gen_index_params(index_type)
+    logger.info(f"Create index...")
+    t0 = time.time()
+    collection.create_index(field_name="float_vector",
+                            index_params=default_index)
+    t1 = time.time()
+    logger.info(f"Create index cost {t1 - t0:.4f} seconds")
+
+    # load collection
+    replica_number = 1
+    logger.info(f"load collection...")
+    t0 = time.time()
+    collection.load(replica_number=replica_number)
+    t1 = time.time()
+    logger.info(f"load collection cost {t1 - t0:.4f} seconds")
+    res = utility.get_query_segment_info(name)
+    cnt = 0
+    logger.info(f"segments info: {res}")
+    for segment in res:
+        cnt += segment.num_rows
+    assert cnt == collection.num_entities
+    logger.info(f"wait for loading complete...")
+    time.sleep(30)
+    res = utility.get_query_segment_info(name)
+    logger.info(f"segments info: {res}")
+
+    # search
+    topK = 100
+    nq = 10000
+    current_search_params = gen_search_param(index_type)
+
+    # define output_fields of search result
+    for i in range(3):
+        t0 = time.time()
+        logger.info(f"Search...")
+        res = collection.search(
+            test[:nq], "float_vector", current_search_params, topK, output_fields=["int64"], timeout=TIMEOUT
+        )
+        t1 = time.time()
+        logger.info(f"search cost  {t1 - t0:.4f} seconds")
+        result_ids = []
+        for hits in res:
+            result_id = []
+            for hit in hits:
+                result_id.append(hit.entity.get("int64"))
+            result_ids.append(result_id)
+
+        # calculate recall
+        true_ids = neighbors[:nq, :topK]
+        sum_radio = 0.0
+        logger.info(f"Calculate recall...")
+        for index, item in enumerate(result_ids):
+            # tmp = set(item).intersection(set(flat_id_list[index]))
+            assert len(item) == len(true_ids[index])
+            tmp = set(true_ids[index]).intersection(set(item))
+            sum_radio = sum_radio + len(tmp) / len(item)
+        recall = round(sum_radio / len(result_ids), 6)
+        logger.info(f"recall={recall}")
+        if index_type in ["IVF_PQ", "ANNOY"]:
+            assert recall >= 0.6, f"recall={recall} < 0.6"
+        else:
+            assert 0.95 <= recall < 1.0, f"recall is {recall}, less than 0.95, greater than or equal to 1.0"
+    # query
+    expr = "int64 in [2,4,6,8]"
+    output_fields = ["int64", "float"]
+    res = collection.query(expr, output_fields, timeout=TIMEOUT)
+    sorted_res = sorted(res, key=lambda k: k['int64'])
+    for r in sorted_res:
+        logger.info(r)
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description='config for recall test')
+    parser.add_argument('--host', type=str,
+                        default="127.0.0.1", help='milvus server ip')
+    args = parser.parse_args()
+    host = args.host
+    tasks = []
+    for index_type in ["HNSW"]:
+        milvus_recall_test(host, index_type)
+

tests/python_client/deploy/scripts/get_tag.py

@@ -0,0 +1,33 @@
+import requests
+import json
+
+milvus_dev = "https://registry.hub.docker.com/v2/repositories/milvusdb/milvus/tags?ordering=last_updated"
+milvus = "https://registry.hub.docker.com/v2/repositories/milvusdb/milvus/tags?ordering=last_updated"
+
+
+def get_tag(url):
+    payload = {}
+    headers = {}
+
+    response = requests.request("GET", url, headers=headers, data=payload)
+
+    res = response.json()["results"]
+    sorted_r = sorted(res, key=lambda k: k['last_updated'])
+    tags = [r["name"] for r in sorted_r]
+    return tags
+
+
+latest_tag = [tag for tag in get_tag(milvus_dev) if "latest" not in tag][-1]
+latest_rc_tag = [tag for tag in get_tag(milvus) if "v" in tag][-1]
+# release_version = "-".join(latest_rc_tag.split("-")[:-2])
+# print(release_version)
+print(latest_tag, latest_rc_tag)
+
+data = {
+    "latest_tag": latest_tag,
+    "latest_rc_tag": latest_rc_tag,
+    # "release_version": release_version
+}
+print(data)
+with open("tag_info.json", "w") as f:
+    f.write(json.dumps(data))

tests/python_client/deploy/scripts/second_recall_test.py

@@ -0,0 +1,103 @@
+import h5py
+import numpy as np
+import time
+import sys
+import threading
+from pathlib import Path
+from loguru import logger
+from pymilvus import connections, Collection
+
+
+all_index_types = ["IVF_FLAT", "IVF_SQ8", "HNSW"]
+
+
+def read_benchmark_hdf5(file_path):
+
+    f = h5py.File(file_path, 'r')
+    train = np.array(f["train"])
+    test = np.array(f["test"])
+    neighbors = np.array(f["neighbors"])
+    f.close()
+    return train, test, neighbors
+
+
+def gen_search_param(index_type, metric_type="L2"):
+    search_params = []
+    if index_type in ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"]:
+        for nprobe in [10]:
+            ivf_search_params = {"metric_type": metric_type, "params": {"nprobe": nprobe}}
+            search_params.append(ivf_search_params)
+    elif index_type in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+        for nprobe in [10]:
+            bin_search_params = {"metric_type": "HAMMING", "params": {"nprobe": nprobe}}
+            search_params.append(bin_search_params)
+    elif index_type in ["HNSW"]:
+        for ef in [150]:
+            hnsw_search_param = {"metric_type": metric_type, "params": {"ef": ef}}
+            search_params.append(hnsw_search_param)
+    elif index_type == "ANNOY":
+        for search_k in [1000]:
+            annoy_search_param = {"metric_type": metric_type, "params": {"search_k": search_k}}
+            search_params.append(annoy_search_param)
+    else:
+        logger.info("Invalid index_type.")
+        raise Exception("Invalid index_type.")
+    return search_params[0]
+
+
+dim = 128
+TIMEOUT = 200
+
+
+def search_test(host="127.0.0.1", index_type="HNSW"):
+    logger.info(f"recall test for index type {index_type}")
+    file_path = f"{str(Path(__file__).absolute().parent.parent.parent)}/assets/ann_hdf5/sift-128-euclidean.hdf5"
+    train, test, neighbors = read_benchmark_hdf5(file_path)
+    connections.connect(host=host, port="19530")
+    collection = Collection(name=f"sift_128_euclidean_{index_type}")
+    nq = 10000
+    topK = 100
+    search_params = gen_search_param(index_type)
+    for i in range(3):
+        t0 = time.time()
+        logger.info(f"\nSearch...")
+        # define output_fields of search result
+        res = collection.search(
+            test[:nq], "float_vector", search_params, topK, output_fields=["int64"], timeout=TIMEOUT
+        )
+        t1 = time.time()
+        logger.info(f"search cost  {t1 - t0:.4f} seconds")
+        result_ids = []
+        for hits in res:
+            result_id = []
+            for hit in hits:
+                result_id.append(hit.entity.get("int64"))
+            result_ids.append(result_id)
+
+        # calculate recall
+        true_ids = neighbors[:nq, :topK]
+        sum_radio = 0.0
+        for index, item in enumerate(result_ids):
+            # tmp = set(item).intersection(set(flat_id_list[index]))
+            assert len(item) == len(true_ids[index]), f"get {len(item)} but expect {len(true_ids[index])}"
+            tmp = set(true_ids[index]).intersection(set(item))
+            sum_radio = sum_radio + len(tmp) / len(item)
+        recall = round(sum_radio / len(result_ids), 6)
+        logger.info(f"recall={recall}")
+        if index_type in ["IVF_PQ", "ANNOY"]:
+            assert recall >= 0.6, f"recall={recall} < 0.6"
+        else:
+            assert 0.95 <= recall < 1.0, f"recall is {recall}, less than 0.95, greater than or equal to 1.0"
+
+
+
+if __name__ == "__main__":
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for recall test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    args = parser.parse_args()
+    host = args.host
+    tasks = []
+    for index_type in ["HNSW"]:
+        search_test(host, index_type)

tests/python_client/deploy/scripts/utils.py

@@ -0,0 +1,268 @@
+import sys
+import copy
+import time
+from loguru import logger
+import pymilvus
+from pymilvus import (
+    FieldSchema, CollectionSchema, DataType,
+    Collection, list_collections,
+)
+logger.remove()
+logger.add(sys.stderr, format= "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
+    "<level>{level: <8}</level> | "
+    "<cyan>{thread.name}</cyan> |"
+    "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+    level="INFO")
+
+pymilvus_version = pymilvus.__version__
+
+all_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", "HNSW"]
+
+default_index_params = [{}, {"nlist": 128}, {"nlist": 128}, {"nlist": 128, "m": 16, "nbits": 8},
+                        {"M": 48, "efConstruction": 500}]
+
+index_params_map = dict(zip(all_index_types, default_index_params))
+
+NUM_REPLICAS = 2
+
+
+def filter_collections_by_prefix(prefix):
+    col_list = list_collections()
+    logger.info(f"all collections: {col_list}")
+    res = []
+    for col in col_list:
+        if col.startswith(prefix):
+            if any(index_name in col for index_name in all_index_types):
+                res.append(col)
+            else:
+                logger.warning(f"collection {col} has no supported index, skip")
+    logger.info(f"filtered collections with prefix {prefix}: {res}")
+    return res
+
+
+def gen_search_param(index_type, metric_type="L2"):
+    search_params = []
+    if index_type in ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"]:
+        for nprobe in [10]:
+            ivf_search_params = {"metric_type": metric_type, "params": {"nprobe": nprobe}}
+            search_params.append(ivf_search_params)
+    elif index_type in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+        for nprobe in [10]:
+            bin_search_params = {"metric_type": "HAMMING", "params": {"nprobe": nprobe}}
+            search_params.append(bin_search_params)
+    elif index_type in ["HNSW"]:
+        for ef in [64]:
+            hnsw_search_param = {"metric_type": metric_type, "params": {"ef": ef}}
+            search_params.append(hnsw_search_param)
+    elif index_type == "ANNOY":
+        for search_k in [1000]:
+            annoy_search_param = {"metric_type": metric_type, "params": {"search_k": search_k}}
+            search_params.append(annoy_search_param)
+    else:
+        logger.info("Invalid index_type.")
+        raise Exception("Invalid index_type.")
+    return search_params
+
+
+def get_collections(prefix, check=False):
+    logger.info("\nList collections...")
+    col_list = filter_collections_by_prefix(prefix)
+    logger.info(f"collections_nums: {len(col_list)}")
+    # list entities if collections
+    for name in col_list:
+        c = Collection(name=name)
+        if pymilvus_version >= "2.2.0":
+            c.flush()
+        else:
+            c.num_entities
+        num_entities = c.num_entities
+        logger.info(f"{name}: {num_entities}")
+        if check:
+            assert num_entities >= 3000
+    return col_list
+
+
+def create_collections_and_insert_data(prefix, flush=True, count=3000, collection_cnt=11):
+    import random
+    dim = 128
+    nb = count // 10
+    default_fields = [
+        FieldSchema(name="count", dtype=DataType.INT64, is_primary=True),
+        FieldSchema(name="random_value", dtype=DataType.DOUBLE),
+        FieldSchema(name="float_vector", dtype=DataType.FLOAT_VECTOR, dim=dim)
+    ]
+    default_schema = CollectionSchema(fields=default_fields, description="test collection")
+    for index_name in all_index_types[:collection_cnt]:
+        logger.info("\nCreate collection...")
+        col_name = prefix + index_name
+        collection = Collection(name=col_name, schema=default_schema)
+        logger.info(f"collection name: {col_name}")
+        logger.info(f"begin insert, count: {count} nb: {nb}")
+        times = int(count // nb)
+        total_time = 0.0
+        vectors = [[random.random() for _ in range(dim)] for _ in range(count)]
+        for j in range(times):
+            start_time = time.time()
+            collection.insert(
+                [
+                    [i for i in range(nb * j, nb * j + nb)],
+                    [float(random.randrange(-20, -10)) for _ in range(nb)],
+                    vectors[nb*j:nb*j+nb]
+                ]
+            )
+            end_time = time.time()
+            logger.info(f"[{j+1}/{times}] insert {nb} data, time: {end_time - start_time:.4f}")
+            total_time += end_time - start_time
+            if j <= times - 3:
+                collection.flush()
+                collection.num_entities
+            if j == times - 3:
+                collection.compact()
+
+
+        logger.info(f"end insert, time: {total_time:.4f}")
+        if flush:
+            logger.info("Get collection entities")
+            start_time = time.time()
+            if pymilvus_version >= "2.2.0":
+                collection.flush()
+            else:
+                collection.num_entities
+            logger.info(f"collection entities: {collection.num_entities}")
+            end_time = time.time()
+            logger.info("Get collection entities time = %.4fs" % (end_time - start_time))
+    logger.info("\nList collections...")
+    logger.info(get_collections(prefix))
+
+
+def create_index_flat():
+    # create index
+    default_flat_index = {"index_type": "FLAT", "params": {}, "metric_type": "L2"}
+    all_col_list = list_collections()
+    col_list = []
+    for col_name in all_col_list:
+        if "FLAT" in col_name and "task" in col_name and "IVF" not in col_name:
+            col_list.append(col_name)
+    logger.info("\nCreate index for FLAT...")
+    for col_name in col_list:
+        c = Collection(name=col_name)
+        logger.info(c)
+        try:
+            replicas = c.get_replicas()
+            replica_number = len(replicas.groups)
+            c.release()
+        except Exception as e:
+            replica_number = 0
+            logger.info(e)
+        t0 = time.time()
+        c.create_index(field_name="float_vector", index_params=default_flat_index)
+        logger.info(f"create index time: {time.time() - t0:.4f}")
+        if replica_number > 0:
+            c.load(replica_number=replica_number)
+
+
+def create_index(prefix):
+    # create index
+    default_index = {"index_type": "IVF_FLAT", "params": {"nlist": 128}, "metric_type": "L2"}
+    col_list = get_collections(prefix)
+    logger.info("\nCreate index...")
+    for col_name in col_list:
+        c = Collection(name=col_name)
+        try:
+            replicas = c.get_replicas()
+            replica_number = len(replicas.groups)
+            c.release()
+        except Exception as e:
+            replica_number = 0
+            logger.info(e)
+        index_name = col_name.replace(prefix, "")
+        logger.info(index_name)
+        logger.info(c)
+        index = copy.deepcopy(default_index)
+        index["index_type"] = index_name
+        index["params"] = index_params_map[index_name]
+        if index_name in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+            index["metric_type"] = "HAMMING"
+        index_info_list = [x.to_dict() for x in c.indexes]
+        logger.info(index_info_list)
+        is_indexed = False
+        for index_info in index_info_list:
+            if "metric_type" in index_info.keys() or "metric_type" in index_info["index_param"]:
+                is_indexed = True
+                logger.info(f"collection {col_name} has been indexed with {index_info}")
+        if not is_indexed:
+            t0 = time.time()
+            c.create_index(field_name="float_vector", index_params=index)
+            logger.info(f"create index time: {time.time() - t0:.4f}")
+        if replica_number > 0:
+            c.load(replica_number=replica_number)
+
+
+def release_collection(prefix):
+    col_list = get_collections(prefix)
+    logger.info("release collection")
+    for col_name in col_list:
+        c = Collection(name=col_name)
+        c.release()
+
+
+def load_and_search(prefix, replicas=1):
+    logger.info("search data starts")
+    col_list = get_collections(prefix)
+    for col_name in col_list:
+        c = Collection(name=col_name)
+        logger.info(f"collection name: {col_name}")
+        logger.info("load collection")
+        if replicas == 1:
+            t0 = time.time()
+            c.load()
+            logger.info(f"load time: {time.time() - t0:.4f}")
+        if replicas > 1:
+            logger.info("release collection before load if replicas > 1")
+            t0 = time.time()
+            c.release()
+            logger.info(f"release time: {time.time() - t0:.4f}")
+            t0 = time.time()
+            c.load(replica_number=replicas)
+            logger.info(f"load time: {time.time() - t0:.4f}")
+            logger.info(c.get_replicas())
+        topK = 5
+        vectors = [[1.0 for _ in range(128)] for _ in range(3000)]
+        index_name = col_name.replace(prefix, "")
+        search_params = gen_search_param(index_name)[0]
+        logger.info(search_params)
+        # search_params = {"metric_type": "L2", "params": {"nprobe": 10}}
+        start_time = time.time()
+        logger.info(f"\nSearch...")
+        # define output_fields of search result
+        v_search = vectors[:1]
+        res = c.search(
+            v_search, "float_vector", search_params, topK,
+            "count > 500", output_fields=["count", "random_value"], timeout=120
+        )
+        end_time = time.time()
+        # show result
+        for hits in res:
+            for hit in hits:
+                logger.info(f"hit: {hit}")
+            ids = hits.ids
+            assert len(ids) == topK, f"get {len(ids)} results, but topK is {topK}"
+            logger.info(ids)
+        assert len(res) == len(v_search), f"get {len(res)} results, but search num is {len(v_search)}"
+        logger.info("search latency: %.4fs" % (end_time - start_time))
+        t0 = time.time()
+        expr = "count in [2,4,6,8]"
+        if "SQ" in col_name or "PQ" in col_name:
+            output_fields = ["count", "random_value"]
+        else:
+            output_fields = ["count", "random_value", "float_vector"]
+        res = c.query(expr, output_fields, timeout=120)
+        sorted_res = sorted(res, key=lambda k: k['count'])
+        for r in sorted_res:
+            logger.info(r)
+        t1 = time.time()
+        assert len(res) == 4
+        logger.info("query latency: %.4fs" % (t1 - t0))
+        # c.release()
+        logger.info("###########")
+    logger.info("search data ends")