chore: import upstream snapshot with attribution

tests/python_client/bulk_insert/conftest.py

@@ -0,0 +1,35 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption("--file_type", action="store", default="json", help="filetype")
+    parser.addoption("--create_index", action="store",  default="create_index", help="whether creating index")
+    parser.addoption("--nb", action="store", default=50000, help="nb")
+    parser.addoption("--dim", action="store",  default=768, help="dim")
+    parser.addoption("--varchar_len", action="store", default=2000, help="varchar_len")
+    parser.addoption("--with_varchar_field", action="store", default="true", help="with varchar field or not")
+
+@pytest.fixture
+def file_type(request):
+    return request.config.getoption("--file_type")
+
+
+@pytest.fixture
+def create_index(request):
+    return request.config.getoption("--create_index")
+
+@pytest.fixture
+def nb(request):
+    return request.config.getoption("--nb")
+
+@pytest.fixture
+def dim(request):
+    return request.config.getoption("--dim")
+
+@pytest.fixture
+def varchar_len(request):
+    return request.config.getoption("--varchar_len")
+
+@pytest.fixture
+def with_varchar_field(request):
+    return request.config.getoption("--with_varchar_field")

tests/python_client/bulk_insert/test_bulk_insert_bench.py

@@ -0,0 +1,407 @@
+import logging
+import time
+import pytest
+from pymilvus import DataType
+import numpy as np
+from pathlib import Path
+from base.client_base import TestcaseBase
+from common import common_func as cf
+from common import common_type as ct
+from common.milvus_sys import MilvusSys
+from common.common_type import CaseLabel, CheckTasks
+from utils.util_log import test_log as log
+from common.bulk_insert_data import (
+    prepare_bulk_insert_json_files,
+    prepare_bulk_insert_new_json_files,
+    prepare_bulk_insert_numpy_files,
+    prepare_bulk_insert_parquet_files,
+    prepare_bulk_insert_csv_files,
+    DataField as df,
+)
+import json
+import requests
+import time
+import uuid
+from utils.util_log import test_log as logger
+from minio import Minio
+from minio.error import S3Error
+
+
+def logger_request_response(response, url, tt, headers, data, str_data, str_response, method):
+    if len(data) > 2000:
+        data = data[:1000] + "..." + data[-1000:]
+    try:
+        if response.status_code == 200:
+            if ('code' in response.json() and response.json()["code"] == 200) or (
+                    'Code' in response.json() and response.json()["Code"] == 0):
+                logger.debug(
+                    f"\nmethod: {method}, \nurl: {url}, \ncost time: {tt}, \nheader: {headers}, \npayload: {str_data}, \nresponse: {str_response}")
+            else:
+                logger.debug(
+                    f"\nmethod: {method}, \nurl: {url}, \ncost time: {tt}, \nheader: {headers}, \npayload: {data}, \nresponse: {response.text}")
+        else:
+            logger.debug(
+                f"method: \nmethod: {method}, \nurl: {url}, \ncost time: {tt}, \nheader: {headers}, \npayload: {data}, \nresponse: {response.text}")
+    except Exception as e:
+        logger.debug(
+            f"method: \nmethod: {method}, \nurl: {url}, \ncost time: {tt}, \nheader: {headers}, \npayload: {data}, \nresponse: {response.text}, \nerror: {e}")
+
+
+class Requests:
+    def __init__(self, url=None, api_key=None):
+        self.url = url
+        self.api_key = api_key
+        self.headers = {
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {self.api_key}',
+            'RequestId': str(uuid.uuid1())
+        }
+
+    def update_headers(self):
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {self.api_key}',
+            'RequestId': str(uuid.uuid1())
+        }
+        return headers
+
+    def post(self, url, headers=None, data=None, params=None):
+        headers = headers if headers is not None else self.update_headers()
+        data = json.dumps(data)
+        str_data = data[:200] + '...' + data[-200:] if len(data) > 400 else data
+        t0 = time.time()
+        response = requests.post(url, headers=headers, data=data, params=params)
+        tt = time.time() - t0
+        str_response = response.text[:200] + '...' + response.text[-200:] if len(response.text) > 400 else response.text
+        logger_request_response(response, url, tt, headers, data, str_data, str_response, "post")
+        return response
+
+    def get(self, url, headers=None, params=None, data=None):
+        headers = headers if headers is not None else self.update_headers()
+        data = json.dumps(data)
+        str_data = data[:200] + '...' + data[-200:] if len(data) > 400 else data
+        t0 = time.time()
+        if data is None or data == "null":
+            response = requests.get(url, headers=headers, params=params)
+        else:
+            response = requests.get(url, headers=headers, params=params, data=data)
+        tt = time.time() - t0
+        str_response = response.text[:200] + '...' + response.text[-200:] if len(response.text) > 400 else response.text
+        logger_request_response(response, url, tt, headers, data, str_data, str_response, "get")
+        return response
+
+    def put(self, url, headers=None, data=None):
+        headers = headers if headers is not None else self.update_headers()
+        data = json.dumps(data)
+        str_data = data[:200] + '...' + data[-200:] if len(data) > 400 else data
+        t0 = time.time()
+        response = requests.put(url, headers=headers, data=data)
+        tt = time.time() - t0
+        str_response = response.text[:200] + '...' + response.text[-200:] if len(response.text) > 400 else response.text
+        logger_request_response(response, url, tt, headers, data, str_data, str_response, "put")
+        return response
+
+    def delete(self, url, headers=None, data=None):
+        headers = headers if headers is not None else self.update_headers()
+        data = json.dumps(data)
+        str_data = data[:200] + '...' + data[-200:] if len(data) > 400 else data
+        t0 = time.time()
+        response = requests.delete(url, headers=headers, data=data)
+        tt = time.time() - t0
+        str_response = response.text[:200] + '...' + response.text[-200:] if len(response.text) > 400 else response.text
+        logger_request_response(response, url, tt, headers, data, str_data, str_response, "delete")
+        return response
+
+
+class ImportJobClient(Requests):
+
+    def __init__(self, endpoint, token):
+        super().__init__(url=endpoint, api_key=token)
+        self.endpoint = endpoint
+        self.api_key = token
+        self.db_name = None
+        self.headers = self.update_headers()
+
+    def update_headers(self):
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {self.api_key}',
+            'RequestId': str(uuid.uuid1())
+        }
+        return headers
+
+    def list_import_jobs(self, payload, db_name="default"):
+        payload["dbName"] = db_name
+        data = payload
+        url = f'{self.endpoint}/v2/vectordb/jobs/import/list'
+        response = self.post(url, headers=self.update_headers(), data=data)
+        res = response.json()
+        return res
+
+    def create_import_jobs(self, payload):
+        url = f'{self.endpoint}/v2/vectordb/jobs/import/create'
+        response = self.post(url, headers=self.update_headers(), data=payload)
+        res = response.json()
+        return res
+
+    def get_import_job_progress(self, task_id):
+        payload = {
+            "jobId": task_id
+        }
+        url = f'{self.endpoint}/v2/vectordb/jobs/import/get_progress'
+        response = self.post(url, headers=self.update_headers(), data=payload)
+        res = response.json()
+        return res
+
+    def wait_import_job_completed(self, task_id_list, timeout=1800):
+        success = False
+        success_states = {}
+        t0 = time.time()
+        while time.time() - t0 < timeout:
+            for task_id in task_id_list:
+                res = self.get_import_job_progress(task_id)
+                if res['data']['state'] == "Completed":
+                    success_states[task_id] = True
+                else:
+                    success_states[task_id] = False
+                time.sleep(5)
+            # all task success then break
+            if all(success_states.values()):
+                success = True
+                break
+        states = []
+        for task_id in task_id_list:
+            res = self.get_import_job_progress(task_id)
+            states.append({
+                "task_id": task_id,
+                "state": res['data']
+            })
+        return success, states
+
+
+default_vec_only_fields = [df.vec_field]
+default_multi_fields = [
+    df.vec_field,
+    df.int_field,
+    df.string_field,
+    df.bool_field,
+    df.float_field,
+    df.array_int_field
+]
+default_vec_n_int_fields = [df.vec_field, df.int_field, df.array_int_field]
+
+
+# milvus_ns = "chaos-testing"
+base_dir = "/tmp/bulk_insert_data"
+
+
+def entity_suffix(entities):
+    if entities // 1000000 > 0:
+        suffix = f"{entities // 1000000}m"
+    elif entities // 1000 > 0:
+        suffix = f"{entities // 1000}k"
+    else:
+        suffix = f"{entities}"
+    return suffix
+
+
+class TestcaseBaseBulkInsert(TestcaseBase):
+    import_job_client = None
+    @pytest.fixture(scope="function", autouse=True)
+    def init_minio_client(self, minio_host):
+        Path("/tmp/bulk_insert_data").mkdir(parents=True, exist_ok=True)
+        self._connect()
+        self.milvus_sys = MilvusSys(alias='default')
+        ms = MilvusSys()
+        minio_port = "9000"
+        self.minio_endpoint = f"{minio_host}:{minio_port}"
+        self.bucket_name = ms.data_nodes[0]["infos"]["system_configurations"][
+            "minio_bucket_name"
+        ]
+
+    @pytest.fixture(scope="function", autouse=True)
+    def init_import_client(self, host, port, user, password):
+        self.import_job_client = ImportJobClient(f"http://{host}:{port}", f"{user}:{password}")
+
+
+class TestBulkInsertPerf(TestcaseBaseBulkInsert):
+
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("auto_id", [True])
+    @pytest.mark.parametrize("dim", [128])  # 128
+    @pytest.mark.parametrize("file_size", [1, 10, 15])  # file size in GB
+    @pytest.mark.parametrize("file_nums", [1])
+    @pytest.mark.parametrize("array_len", [100])
+    @pytest.mark.parametrize("enable_dynamic_field", [False])
+    def test_bulk_insert_all_field_with_parquet(self, auto_id, dim, file_size, file_nums, array_len, enable_dynamic_field):
+        """
+        collection schema 1: [pk, int64, float64, string float_vector]
+        data file: vectors.parquet and uid.parquet,
+        Steps:
+        1. create collection
+        2. import data
+        3. verify
+        """
+        fields = [
+            cf.gen_int64_field(name=df.pk_field, is_primary=True, auto_id=auto_id),
+            cf.gen_int64_field(name=df.int_field),
+            cf.gen_float_field(name=df.float_field),
+            cf.gen_double_field(name=df.double_field),
+            cf.gen_json_field(name=df.json_field),
+            cf.gen_array_field(name=df.array_int_field, element_type=DataType.INT64),
+            cf.gen_array_field(name=df.array_float_field, element_type=DataType.FLOAT),
+            cf.gen_array_field(name=df.array_string_field, element_type=DataType.VARCHAR, max_length=200),
+            cf.gen_array_field(name=df.array_bool_field, element_type=DataType.BOOL),
+            cf.gen_float_vec_field(name=df.vec_field, dim=dim),
+        ]
+        data_fields = [f.name for f in fields if not f.to_dict().get("auto_id", False)]
+        files = prepare_bulk_insert_parquet_files(
+            minio_endpoint=self.minio_endpoint,
+            bucket_name=self.bucket_name,
+            rows=3000,
+            dim=dim,
+            data_fields=data_fields,
+            file_size=file_size,
+            row_group_size=None,
+            file_nums=file_nums,
+            array_length=array_len,
+            enable_dynamic_field=enable_dynamic_field,
+            force=True,
+        )
+        self._connect()
+        c_name = cf.gen_unique_str("bulk_insert")
+        schema = cf.gen_collection_schema(fields=fields, auto_id=auto_id, enable_dynamic_field=enable_dynamic_field)
+        self.collection_wrap.init_collection(c_name, schema=schema)
+        payload = {
+            "collectionName": c_name,
+            "files": [files],
+        }
+
+        # import data
+        payload = {
+            "collectionName": c_name,
+            "files": [files],
+        }
+        t0 = time.time()
+        rsp = self.import_job_client.create_import_jobs(payload)
+        job_id_list = [rsp["data"]["jobId"]]
+        logging.info(f"bulk insert job ids:{job_id_list}")
+        success, states = self.import_job_client.wait_import_job_completed(job_id_list, timeout=1800)
+        tt = time.time() - t0
+        log.info(f"bulk insert state:{success} in {tt} with states:{states}")
+        assert success
+
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("auto_id", [True])
+    @pytest.mark.parametrize("dim", [128])  # 128
+    @pytest.mark.parametrize("file_size", [1, 10, 15])  # file size in GB
+    @pytest.mark.parametrize("file_nums", [1])
+    @pytest.mark.parametrize("array_len", [100])
+    @pytest.mark.parametrize("enable_dynamic_field", [False])
+    def test_bulk_insert_all_field_with_json(self, auto_id, dim, file_size, file_nums, array_len, enable_dynamic_field):
+        """
+        collection schema 1: [pk, int64, float64, string float_vector]
+        data file: vectors.parquet and uid.parquet,
+        Steps:
+        1. create collection
+        2. import data
+        3. verify
+        """
+        fields = [
+            cf.gen_int64_field(name=df.pk_field, is_primary=True, auto_id=auto_id),
+            cf.gen_int64_field(name=df.int_field),
+            cf.gen_float_field(name=df.float_field),
+            cf.gen_double_field(name=df.double_field),
+            cf.gen_json_field(name=df.json_field),
+            cf.gen_array_field(name=df.array_int_field, element_type=DataType.INT64),
+            cf.gen_array_field(name=df.array_float_field, element_type=DataType.FLOAT),
+            cf.gen_array_field(name=df.array_string_field, element_type=DataType.VARCHAR, max_length=200),
+            cf.gen_array_field(name=df.array_bool_field, element_type=DataType.BOOL),
+            cf.gen_float_vec_field(name=df.vec_field, dim=dim),
+        ]
+        data_fields = [f.name for f in fields if not f.to_dict().get("auto_id", False)]
+        files = prepare_bulk_insert_new_json_files(
+            minio_endpoint=self.minio_endpoint,
+            bucket_name=self.bucket_name,
+            rows=3000,
+            dim=dim,
+            data_fields=data_fields,
+            file_size=file_size,
+            file_nums=file_nums,
+            array_length=array_len,
+            enable_dynamic_field=enable_dynamic_field,
+            force=True,
+        )
+        self._connect()
+        c_name = cf.gen_unique_str("bulk_insert")
+        schema = cf.gen_collection_schema(fields=fields, auto_id=auto_id, enable_dynamic_field=enable_dynamic_field)
+        self.collection_wrap.init_collection(c_name, schema=schema)
+
+        # import data
+        payload = {
+            "collectionName": c_name,
+            "files": [files],
+        }
+        t0 = time.time()
+        rsp = self.import_job_client.create_import_jobs(payload)
+        job_id_list = [rsp["data"]["jobId"]]
+        logging.info(f"bulk insert job ids:{job_id_list}")
+        success, states = self.import_job_client.wait_import_job_completed(job_id_list, timeout=1800)
+        tt = time.time() - t0
+        log.info(f"bulk insert state:{success} in {tt} with states:{states}")
+        assert success
+
+
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("auto_id", [True])
+    @pytest.mark.parametrize("dim", [128])  # 128
+    @pytest.mark.parametrize("file_size", [1, 10, 15])  # file size in GB
+    @pytest.mark.parametrize("file_nums", [1])
+    @pytest.mark.parametrize("enable_dynamic_field", [False])
+    def test_bulk_insert_all_field_with_numpy(self, auto_id, dim, file_size, file_nums, enable_dynamic_field):
+        """
+        collection schema 1: [pk, int64, float64, string float_vector]
+        data file: vectors.parquet and uid.parquet,
+        Steps:
+        1. create collection
+        2. import data
+        3. verify
+        """
+        fields = [
+            cf.gen_int64_field(name=df.pk_field, is_primary=True, auto_id=auto_id),
+            cf.gen_int64_field(name=df.int_field),
+            cf.gen_float_field(name=df.float_field),
+            cf.gen_double_field(name=df.double_field),
+            cf.gen_json_field(name=df.json_field),
+            cf.gen_float_vec_field(name=df.vec_field, dim=dim),
+        ]
+        data_fields = [f.name for f in fields if not f.to_dict().get("auto_id", False)]
+        files = prepare_bulk_insert_numpy_files(
+            minio_endpoint=self.minio_endpoint,
+            bucket_name=self.bucket_name,
+            rows=3000,
+            dim=dim,
+            data_fields=data_fields,
+            file_size=file_size,
+            file_nums=file_nums,
+            enable_dynamic_field=enable_dynamic_field,
+            force=True,
+        )
+        self._connect()
+        c_name = cf.gen_unique_str("bulk_insert")
+        schema = cf.gen_collection_schema(fields=fields, auto_id=auto_id, enable_dynamic_field=enable_dynamic_field)
+        self.collection_wrap.init_collection(c_name, schema=schema)
+
+        # import data
+        payload = {
+            "collectionName": c_name,
+            "files": [files],
+        }
+        t0 = time.time()
+        rsp = self.import_job_client.create_import_jobs(payload)
+        job_id_list = [rsp["data"]["jobId"]]
+        logging.info(f"bulk insert job ids:{job_id_list}")
+        success, states = self.import_job_client.wait_import_job_completed(job_id_list, timeout=1800)
+        tt = time.time() - t0
+        log.info(f"bulk insert state:{success} in {tt} with states:{states}")
+        assert success

tests/python_client/bulk_insert/test_bulk_insert_perf.py

@@ -0,0 +1,154 @@
+import pytest
+import os
+import time
+import threading
+from pathlib import Path
+from time import sleep
+from minio import Minio
+from pymilvus import connections
+from chaos.checker import (BulkInsertChecker, Op)
+from common.milvus_sys import MilvusSys
+from utils.util_log import test_log as log
+from utils.util_k8s import get_milvus_deploy_tool, get_pod_ip_name_pairs, get_milvus_instance_name
+from chaos import chaos_commons as cc
+from common.common_type import CaseLabel
+from common import common_func as cf
+from chaos import constants
+from delayed_assert import expect, assert_expectations
+
+
+def assert_statistic(checkers, expectations={}):
+    for k in checkers.keys():
+        # expect succ if no expectations
+        succ_rate = checkers[k].succ_rate()
+        total = checkers[k].total()
+        average_time = checkers[k].average_time
+        if expectations.get(k, '') == constants.FAIL:
+            log.info(
+                f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+            expect(succ_rate < 0.49 or total < 2,
+                   f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+        else:
+            log.info(
+                f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+            expect(succ_rate > 0.90 and total > 2,
+                   f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+
+
+def get_querynode_info(release_name):
+    querynode_id_pod_pair = {}
+    querynode_ip_pod_pair = get_pod_ip_name_pairs(
+        "chaos-testing", f"app.kubernetes.io/instance={release_name}, component=querynode")
+    ms = MilvusSys()
+    for node in ms.query_nodes:
+        ip = node["infos"]['hardware_infos']["ip"].split(":")[0]
+        querynode_id_pod_pair[node["identifier"]] = querynode_ip_pod_pair[ip]
+    return querynode_id_pod_pair
+
+
+class TestChaosBase:
+    expect_create = constants.SUCC
+    expect_insert = constants.SUCC
+    expect_flush = constants.SUCC
+    expect_index = constants.SUCC
+    expect_search = constants.SUCC
+    expect_query = constants.SUCC
+    host = '127.0.0.1'
+    port = 19530
+    _chaos_config = None
+    health_checkers = {}
+
+
+class TestChaos(TestChaosBase):
+
+    def teardown_method(self):
+        sleep(10)
+        log.info(f'Alive threads: {threading.enumerate()}')
+
+    @pytest.fixture(scope="function", autouse=True)
+    def connection(self, host, port, milvus_ns):
+        connections.add_connection(default={"host": host, "port": port})
+        connections.connect(alias='default')
+
+        if connections.has_connection("default") is False:
+            raise Exception("no connections")
+        instance_name = get_milvus_instance_name(constants.CHAOS_NAMESPACE, host)
+        self.host = host
+        self.port = port
+        self.instance_name = instance_name
+        self.milvus_sys = MilvusSys(alias='default')
+        self.milvus_ns = milvus_ns
+        self.release_name = get_milvus_instance_name(self.milvus_ns, milvus_sys=self.milvus_sys)
+        self.deploy_by = get_milvus_deploy_tool(self.milvus_ns, self.milvus_sys)
+
+    def init_health_checkers(self, collection_name=None, dim=2048):
+        log.info("init health checkers")
+        c_name = collection_name if collection_name else cf.gen_unique_str("Checker_")
+        checkers = {
+            Op.bulk_insert: BulkInsertChecker(collection_name=c_name, use_one_collection=False, dim=dim,),
+        }
+        self.health_checkers = checkers
+
+    def prepare_bulk_insert(self, nb=3000, file_type="json", dim=768, varchar_len=2000, with_varchar_field=True):
+        if Op.bulk_insert not in self.health_checkers:
+            log.info("bulk_insert checker is not in  health checkers, skip prepare bulk load")
+            return
+        log.info("bulk_insert checker is in health checkers, prepare data firstly")
+        deploy_tool = get_milvus_deploy_tool(self.milvus_ns, self.milvus_sys)
+        if deploy_tool == "helm":
+            release_name = self.instance_name
+        else:
+            release_name = self.instance_name + "-minio"
+        minio_ip_pod_pair = get_pod_ip_name_pairs("chaos-testing", f"release={release_name}, app=minio")
+        ms = MilvusSys()
+        minio_ip = list(minio_ip_pod_pair.keys())[0]
+        minio_port = "9000"
+        minio_endpoint = f"{minio_ip}:{minio_port}"
+        bucket_name = ms.data_nodes[0]["infos"]["system_configurations"]["minio_bucket_name"]
+        schema = cf.gen_bulk_insert_collection_schema(dim=dim, with_varchar_field=with_varchar_field)
+        data = cf.gen_default_list_data_for_bulk_insert(nb=nb, varchar_len=varchar_len,
+                                                        with_varchar_field=with_varchar_field)
+        data_dir = "/tmp/bulk_insert_data"
+        Path(data_dir).mkdir(parents=True, exist_ok=True)
+        files = []
+        if file_type == "json":
+            files = cf.gen_json_files_for_bulk_insert(data, schema, data_dir, nb=nb, dim=dim)
+        if file_type == "npy":
+            files = cf.gen_npy_files_for_bulk_insert(data, schema, data_dir, nb=nb, dim=dim)
+        log.info("upload file to minio")
+        client = Minio(minio_endpoint, access_key="minioadmin", secret_key="minioadmin", secure=False)
+        for file_name in files:
+            file_size = os.path.getsize(os.path.join(data_dir, file_name)) / 1024 / 1024
+            t0 = time.time()
+            client.fput_object(bucket_name, file_name, os.path.join(data_dir, file_name))
+            log.info(f"upload file {file_name} to minio, size: {file_size:.2f} MB, cost {time.time() - t0:.2f} s")
+        self.health_checkers[Op.bulk_insert].update(schema=schema, files=files)
+        log.info("prepare data for bulk load done")
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_bulk_insert_perf(self, file_type, nb, dim, varchar_len, with_varchar_field):
+        # start the monitor threads to check the milvus ops
+        log.info("*********************Test Start**********************")
+        log.info(connections.get_connection_addr('default'))
+        log.info(f"file_type: {file_type}, nb: {nb}, dim: {dim}, varchar_len: {varchar_len}, with_varchar_field: {with_varchar_field}")
+        self.init_health_checkers(dim=int(dim))
+        nb = int(nb)
+        if str(with_varchar_field) in ["true", "True"]:
+            with_varchar_field = True
+        else:
+            with_varchar_field = False
+        varchar_len = int(varchar_len)
+
+        self.prepare_bulk_insert(file_type=file_type, nb=nb, dim=int(dim), varchar_len=varchar_len, with_varchar_field=with_varchar_field)
+        cc.start_monitor_threads(self.health_checkers)
+        # wait 600s
+        while self.health_checkers[Op.bulk_insert].total() <= 10:
+            sleep(constants.WAIT_PER_OP)
+        assert_statistic(self.health_checkers)
+
+        assert_expectations()
+        for k, checker in self.health_checkers.items():
+            checker.check_result()
+            checker.terminate()
+
+        log.info("*********************Test Completed**********************")

tests/python_client/bulk_insert/test_bulk_insert_perf_with_cohere_dataset.py

@@ -0,0 +1,106 @@
+import pytest
+import threading
+from time import sleep
+from pymilvus import connections, DataType, FieldSchema, CollectionSchema
+from chaos.checker import (BulkInsertChecker, Op)
+from utils.util_log import test_log as log
+from chaos import chaos_commons as cc
+from common.common_type import CaseLabel
+from common import common_func as cf
+from chaos import constants
+from delayed_assert import expect, assert_expectations
+
+
+def assert_statistic(checkers, expectations={}):
+    for k in checkers.keys():
+        # expect succ if no expectations
+        succ_rate = checkers[k].succ_rate()
+        total = checkers[k].total()
+        average_time = checkers[k].average_time
+        if expectations.get(k, '') == constants.FAIL:
+            log.info(
+                f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+            expect(succ_rate < 0.49 or total < 2,
+                   f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+        else:
+            log.info(
+                f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+            expect(succ_rate > 0.90 and total > 2,
+                   f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+
+
+class TestBulkInsertBase:
+    expect_create = constants.SUCC
+    expect_insert = constants.SUCC
+    expect_flush = constants.SUCC
+    expect_index = constants.SUCC
+    expect_search = constants.SUCC
+    expect_query = constants.SUCC
+    host = '127.0.0.1'
+    port = 19530
+    _chaos_config = None
+    health_checkers = {}
+
+
+class TestBUlkInsertPerf(TestBulkInsertBase):
+
+    def teardown_method(self):
+        sleep(10)
+        log.info(f'Alive threads: {threading.enumerate()}')
+
+    @pytest.fixture(scope="function", autouse=True)
+    def connection(self, host, port, milvus_ns):
+        connections.add_connection(default={"host": host, "port": port})
+        connections.connect(alias='default')
+
+        if connections.has_connection("default") is False:
+            raise Exception("no connections")
+
+    def init_health_checkers(self, collection_name=None, file_type="npy"):
+        log.info("init health checkers")
+        c_name = collection_name if collection_name else cf.gen_unique_str("BulkInsertChecker")
+        fields = [
+            FieldSchema(name="id", dtype=DataType.INT64, is_primary=True),
+            FieldSchema(name="title", dtype=DataType.VARCHAR, max_length=65535),
+            FieldSchema(name="text", dtype=DataType.VARCHAR, max_length=65535),
+            FieldSchema(name="url", dtype=DataType.VARCHAR, max_length=65535),
+            FieldSchema(name="wiki_id", dtype=DataType.INT64),
+            FieldSchema(name="views", dtype=DataType.DOUBLE),
+            FieldSchema(name="paragraph_id", dtype=DataType.INT64),
+            FieldSchema(name="langs", dtype=DataType.INT64),
+            FieldSchema(name="emb", dtype=DataType.FLOAT_VECTOR, dim=768)
+        ]
+        schema = CollectionSchema(fields=fields, description="test collection")
+        fields_name = ["id", "title", "text", "url", "wiki_id", "views", "paragraph_id", "langs", "emb"]
+        files = []
+        if file_type == "json":
+            files = ["train-00000-of-00252.json"]
+        if file_type == "npy":
+            for field_name in fields_name:
+                files.append(f"{field_name}.npy")
+        if file_type == "parquet":
+            files = ["train-00000-of-00252.parquet"]
+        checkers = {
+            Op.bulk_insert: BulkInsertChecker(collection_name=c_name, use_one_collection=False, schema=schema,
+                                              files=files, insert_data=False)
+        }
+        self.health_checkers = checkers
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_bulk_insert_perf(self):
+        # start the monitor threads to check the milvus ops
+        log.info("*********************Test Start**********************")
+        log.info(connections.get_connection_addr('default'))
+        self.init_health_checkers()
+        cc.start_monitor_threads(self.health_checkers)
+        # wait 600s
+        while self.health_checkers[Op.bulk_insert].total() <= 10:
+            sleep(constants.WAIT_PER_OP)
+        assert_statistic(self.health_checkers)
+
+        assert_expectations()
+        for k, checker in self.health_checkers.items():
+            checker.check_result()
+            checker.terminate()
+
+        log.info("*********************Test Completed**********************")

tests/python_client/bulk_insert/test_bulk_insert_task_clean.py

@@ -0,0 +1,248 @@
+import logging
+import time
+import pytest
+from pathlib import Path
+from base.client_base import TestcaseBase
+from common import common_func as cf
+from common import common_type as ct
+from common.milvus_sys import MilvusSys
+from common.common_type import CaseLabel, CheckTasks
+from utils.util_k8s import (
+    get_pod_ip_name_pairs,
+    get_milvus_instance_name,
+    get_milvus_deploy_tool
+)
+from utils.util_log import test_log as log
+from common.bulk_insert_data import (
+    prepare_bulk_insert_json_files,
+    DataField as df,
+    DataErrorType,
+)
+
+
+default_vec_only_fields = [df.vec_field]
+default_multi_fields = [
+    df.vec_field,
+    df.int_field,
+    df.string_field,
+    df.bool_field,
+    df.float_field,
+]
+default_vec_n_int_fields = [df.vec_field, df.int_field]
+
+
+milvus_ns = "chaos-testing"
+base_dir = "/tmp/bulk_insert_data"
+
+
+def entity_suffix(entities):
+    if entities // 1000000 > 0:
+        suffix = f"{entities // 1000000}m"
+    elif entities // 1000 > 0:
+        suffix = f"{entities // 1000}k"
+    else:
+        suffix = f"{entities}"
+    return suffix
+
+
+class TestcaseBaseBulkInsert(TestcaseBase):
+
+    @pytest.fixture(scope="function", autouse=True)
+    def init_minio_client(self, host, milvus_ns):
+        Path("/tmp/bulk_insert_data").mkdir(parents=True, exist_ok=True)
+        self._connect()
+        self.milvus_ns = milvus_ns
+        self.milvus_sys = MilvusSys(alias='default')
+        self.instance_name = get_milvus_instance_name(self.milvus_ns, host)
+        self.deploy_tool = get_milvus_deploy_tool(self.milvus_ns, self.milvus_sys)
+        minio_label = f"release={self.instance_name}, app=minio"
+        if self.deploy_tool == "milvus-operator":
+            minio_label = f"release={self.instance_name}-minio, app=minio"
+        minio_ip_pod_pair = get_pod_ip_name_pairs(
+            self.milvus_ns, minio_label
+        )
+        ms = MilvusSys()
+        minio_ip = list(minio_ip_pod_pair.keys())[0]
+        minio_port = "9000"
+        self.minio_endpoint = f"{minio_ip}:{minio_port}"
+        self.bucket_name = ms.data_nodes[0]["infos"]["system_configurations"][
+            "minio_bucket_name"
+        ]
+
+    # def teardown_method(self, method):
+    #     log.info(("*" * 35) + " teardown " + ("*" * 35))
+    #     log.info("[teardown_method] Start teardown test case %s..." % method.__name__)
+
+
+class TestBulkInsertTaskClean(TestcaseBaseBulkInsert):
+
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("is_row_based", [True])
+    @pytest.mark.parametrize("auto_id", [True, False])
+    @pytest.mark.parametrize("dim", [8])  # 8, 128
+    @pytest.mark.parametrize("entities", [100])  # 100, 1000
+    def test_success_task_not_cleaned(self, is_row_based, auto_id, dim, entities):
+        """
+        collection: auto_id, customized_id
+        collection schema: [pk, float_vector]
+        Steps:
+        1. create collection
+        2. import data
+        3. verify the data entities equal the import data
+        4. load the collection
+        5. verify search successfully
+        6. verify query successfully
+        7. wait for task clean triggered
+        8. verify the task not cleaned
+        """
+        files = prepare_bulk_insert_json_files(
+            minio_endpoint=self.minio_endpoint,
+            bucket_name=self.bucket_name,
+            is_row_based=is_row_based,
+            rows=entities,
+            dim=dim,
+            auto_id=auto_id,
+            data_fields=default_vec_only_fields,
+            force=True,
+        )
+        self._connect()
+        c_name = cf.gen_unique_str("bulk_insert")
+        fields = [
+            cf.gen_int64_field(name=df.pk_field, is_primary=True),
+            cf.gen_float_vec_field(name=df.vec_field, dim=dim),
+        ]
+        schema = cf.gen_collection_schema(fields=fields, auto_id=auto_id)
+        self.collection_wrap.init_collection(c_name, schema=schema)
+        # import data
+        t0 = time.time()
+        task_id, _ = self.utility_wrap.do_bulk_insert(
+            collection_name=c_name,
+            partition_name=None,
+            files=files,
+        )
+        logging.info(f"bulk insert task ids:{task_id}")
+        success, _ = self.utility_wrap.wait_for_bulk_insert_tasks_completed(
+            task_ids=[task_id], timeout=90
+        )
+        tt = time.time() - t0
+        log.info(f"bulk insert state:{success} in {tt}")
+        assert success
+
+        num_entities = self.collection_wrap.num_entities
+        log.info(f" collection entities: {num_entities}")
+        assert num_entities == entities
+
+        # verify imported data is available for search
+        index_params = ct.default_index
+        self.collection_wrap.create_index(
+            field_name=df.vec_field, index_params=index_params
+        )
+        self.collection_wrap.load()
+        log.info(f"wait for load finished and be ready for search")
+        time.sleep(5)
+        log.info(
+            f"query seg info: {self.utility_wrap.get_query_segment_info(c_name)[0]}"
+        )
+        nq = 2
+        topk = 2
+        search_data = cf.gen_vectors(nq, dim)
+        search_params = ct.default_search_params
+        res, _ = self.collection_wrap.search(
+            search_data,
+            df.vec_field,
+            param=search_params,
+            limit=topk,
+            check_task=CheckTasks.check_search_results,
+            check_items={"nq": nq, "limit": topk},
+        )
+        for hits in res:
+            ids = hits.ids
+            results, _ = self.collection_wrap.query(expr=f"{df.pk_field} in {ids}")
+            assert len(results) == len(ids)
+        log.info("wait for task clean triggered")
+        time.sleep(6*60)  # wait for 6 minutes for task clean triggered
+        num_entities = self.collection_wrap.num_entities
+        log.info(f" collection entities: {num_entities}")
+        assert num_entities == entities
+        res, _ = self.collection_wrap.search(
+            search_data,
+            df.vec_field,
+            param=search_params,
+            limit=topk,
+            check_task=CheckTasks.check_search_results,
+            check_items={"nq": nq, "limit": topk},
+        )
+        for hits in res:
+            ids = hits.ids
+            results, _ = self.collection_wrap.query(expr=f"{df.pk_field} in {ids}")
+            assert len(results) == len(ids)
+
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("is_row_based", [True])
+    @pytest.mark.parametrize("auto_id", [True, False])
+    @pytest.mark.parametrize("dim", [8])  # 8, 128
+    @pytest.mark.parametrize("entities", [100])  # 100, 1000
+    def test_failed_task_was_cleaned(self, is_row_based, auto_id, dim, entities):
+        """
+        collection: auto_id, customized_id
+        collection schema: [pk, float_vector]
+        Steps:
+        1. create collection
+        2. import data with wrong dimension
+        3. verify the data entities is 0 and task was failed
+        4. wait for task clean triggered
+        5. verify the task was cleaned
+
+        """
+        files = prepare_bulk_insert_json_files(
+            minio_endpoint=self.minio_endpoint,
+            bucket_name=self.bucket_name,
+            is_row_based=is_row_based,
+            rows=entities,
+            dim=dim,
+            auto_id=auto_id,
+            data_fields=default_vec_only_fields,
+            err_type=DataErrorType.one_entity_wrong_dim,
+            wrong_position=entities // 2,
+            force=True,
+        )
+        self._connect()
+        c_name = cf.gen_unique_str("bulk_insert")
+        fields = [
+            cf.gen_int64_field(name=df.pk_field, is_primary=True),
+            cf.gen_float_vec_field(name=df.vec_field, dim=dim),
+        ]
+        schema = cf.gen_collection_schema(fields=fields, auto_id=auto_id)
+        self.collection_wrap.init_collection(c_name, schema=schema)
+        # import data
+        t0 = time.time()
+        task_id, _ = self.utility_wrap.do_bulk_insert(
+            collection_name=c_name,
+            partition_name=None,
+            is_row_based=is_row_based,
+            files=files,
+        )
+        logging.info(f"bulk insert task ids:{task_id}")
+        success, states = self.utility_wrap.wait_for_bulk_insert_tasks_completed(
+            task_ids=[task_id], timeout=90
+        )
+        tt = time.time() - t0
+        log.info(f"bulk insert state:{success} in {tt}")
+        assert not success
+        for state in states.values():
+            assert state.state_name in ["Failed", "Failed and cleaned"]
+
+        num_entities = self.collection_wrap.num_entities
+        log.info(f" collection entities: {num_entities}")
+        assert num_entities == 0
+        log.info("wait for task clean triggered")
+        time.sleep(6*60)  # wait for 6 minutes for task clean triggered
+        num_entities = self.collection_wrap.num_entities
+        log.info(f" collection entities: {num_entities}")
+        assert num_entities == 0
+        success, states = self.utility_wrap.wait_for_bulk_insert_tasks_completed(
+            task_ids=[task_id], timeout=90
+        )
+        assert not success
+        for state in states.values():
+            assert state.state_name in ["Failed and cleaned"]

tests/python_client/bulk_insert/test_bulk_insert_with_requests.py

@@ -0,0 +1,149 @@
+import threading
+
+import pytest
+import os
+import time
+import json
+from time import sleep
+from pathlib import Path
+from minio import Minio
+from pymilvus import connections
+from chaos.checker import (InsertChecker, SearchChecker, QueryChecker, BulkInsertChecker, Op)
+from common.cus_resource_opts import CustomResourceOperations as CusResource
+from common.milvus_sys import MilvusSys
+from utils.util_log import test_log as log
+from utils.util_k8s import wait_pods_ready, get_milvus_deploy_tool, get_pod_ip_name_pairs, get_milvus_instance_name
+from utils.util_common import update_key_value
+from chaos import chaos_commons as cc
+from common.common_type import CaseLabel
+from common import common_func as cf
+from chaos import constants
+from delayed_assert import expect, assert_expectations
+
+
+def assert_statistic(checkers, expectations={}):
+    for k in checkers.keys():
+        # expect succ if no expectations
+        succ_rate = checkers[k].succ_rate()
+        total = checkers[k].total()
+        average_time = checkers[k].average_time
+        if expectations.get(k, '') == constants.FAIL:
+            log.info(
+                f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+            expect(succ_rate < 0.49 or total < 2,
+                   f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+        else:
+            log.info(
+                f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+            expect(succ_rate > 0.90 and total > 2,
+                   f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
+
+
+def get_querynode_info(release_name):
+    querynode_id_pod_pair = {}
+    querynode_ip_pod_pair = get_pod_ip_name_pairs(
+        "chaos-testing", f"app.kubernetes.io/instance={release_name}, component=querynode")
+    ms = MilvusSys()
+    for node in ms.query_nodes:
+        ip = node["infos"]['hardware_infos']["ip"].split(":")[0]
+        querynode_id_pod_pair[node["identifier"]] = querynode_ip_pod_pair[ip]
+    return querynode_id_pod_pair
+
+
+class TestChaosBase:
+    expect_create = constants.SUCC
+    expect_insert = constants.SUCC
+    expect_flush = constants.SUCC
+    expect_index = constants.SUCC
+    expect_search = constants.SUCC
+    expect_query = constants.SUCC
+    host = '127.0.0.1'
+    port = 19530
+    _chaos_config = None
+    health_checkers = {}
+
+
+class TestChaos(TestChaosBase):
+
+    @pytest.fixture(scope="function", autouse=True)
+    def connection(self, host, port, milvus_ns):
+        connections.add_connection(default={"host": host, "port": port})
+        connections.connect(alias='default')
+
+        if connections.has_connection("default") is False:
+            raise Exception("no connections")
+        instance_name = get_milvus_instance_name(constants.CHAOS_NAMESPACE, host)
+        self.host = host
+        self.port = port
+        self.instance_name = instance_name
+        self.milvus_sys = MilvusSys(alias='default')
+        self.milvus_ns = milvus_ns
+        self.release_name = get_milvus_instance_name(self.milvus_ns, milvus_sys=self.milvus_sys)
+        self.deploy_by = get_milvus_deploy_tool(self.milvus_ns, self.milvus_sys)
+
+    @pytest.fixture(scope="function", autouse=True)
+    def init_health_checkers(self, collection_name=None):
+        log.info("init health checkers")
+        c_name = collection_name if collection_name else cf.gen_unique_str("Checker_")
+        checkers = {
+            Op.insert: InsertChecker(collection_name=c_name),
+            Op.search: SearchChecker(collection_name=c_name),
+            Op.bulk_insert: BulkInsertChecker(collection_name=c_name, use_one_collection=True),
+            Op.query: QueryChecker(collection_name=c_name)
+        }
+        self.health_checkers = checkers
+
+    @pytest.fixture(scope="function", autouse=True)
+    def prepare_bulk_insert(self, nb=3000):
+        if Op.bulk_insert not in self.health_checkers:
+            log.info("bulk_insert checker is not in  health checkers, skip prepare bulk load")
+            return
+        log.info("bulk_insert checker is in  health checkers, prepare data firstly")
+        deploy_tool = get_milvus_deploy_tool(self.milvus_ns, self.milvus_sys)
+        if deploy_tool == "helm":
+            release_name = self.instance_name
+        else:
+            release_name = self.instance_name + "-minio"
+        minio_ip_pod_pair = get_pod_ip_name_pairs("chaos-testing", f"release={release_name}, app=minio")
+        ms = MilvusSys()
+        minio_ip = list(minio_ip_pod_pair.keys())[0]
+        minio_port = "9000"
+        minio_endpoint = f"{minio_ip}:{minio_port}"
+        bucket_name = ms.data_nodes[0]["infos"]["system_configurations"]["minio_bucket_name"]
+        schema = cf.gen_default_collection_schema()
+        data = cf.gen_default_list_data_for_bulk_insert(nb=nb)
+        fields_name = [field.name for field in schema.fields]
+        entities = []
+        for i in range(nb):
+            entity_value = [field_values[i] for field_values in data]
+            entity = dict(zip(fields_name, entity_value))
+            entities.append(entity)
+        data_dict = {"rows": entities}
+        data_source = "/tmp/ci_logs/bulk_insert_data_source.json"
+        file_name = "bulk_insert_data_source.json"
+        files = ["bulk_insert_data_source.json"]
+        # TODO: npy file type is not supported so far
+        log.info("generate bulk load file")
+        with open(data_source, "w") as f:
+            f.write(json.dumps(data_dict, indent=4))
+        log.info("upload file to minio")
+        client = Minio(minio_endpoint, access_key="minioadmin", secret_key="minioadmin", secure=False)
+        client.fput_object(bucket_name, file_name, data_source)
+        self.health_checkers[Op.bulk_insert].update(schema=schema, files=files)
+        log.info("prepare data for bulk load done")
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_bulk_insert(self):
+        # start the monitor threads to check the milvus ops
+        log.info("*********************Test Start**********************")
+        log.info(connections.get_connection_addr('default'))
+        # c_name = cf.gen_unique_str("BulkInsertChecker_")
+        # self.init_health_checkers(collection_name=c_name)
+        cc.start_monitor_threads(self.health_checkers)
+        # wait 120s
+        sleep(constants.WAIT_PER_OP * 12)
+        assert_statistic(self.health_checkers)
+
+        assert_expectations()
+
+        log.info("*********************Test Completed**********************")