chore: import upstream snapshot with attribution

tests/python_client/utils/api_request.py

@@ -0,0 +1,102 @@
+import sys
+import traceback
+import copy
+
+from check.func_check import ResponseChecker, Error
+from utils.util_log import test_log as log
+
+# enable_traceback = os.getenv('ENABLE_TRACEBACK', "True")
+# log.info(f"enable_traceback:{enable_traceback}")
+
+
+log_row_length = 100  # Reduced from 300 to minimize log size
+
+
+def api_request_catch():
+    def wrapper(func):
+        def inner_wrapper(*args, **kwargs):
+            try:
+                _kwargs = copy.deepcopy(kwargs)
+                if "enable_traceback" in _kwargs:
+                    del _kwargs["enable_traceback"]
+                res = func(*args, **_kwargs)
+                # if enable_traceback == "True":
+                if kwargs.get("enable_traceback", True):
+                    res_str = str(res)
+                    log_res = res_str[0:log_row_length] + '......' if len(res_str) > log_row_length else res_str
+                    log.debug("(api_response) : %s " % log_res)
+
+                return res, True
+            except Exception as e:
+                e_str = str(e)
+                log_e = e_str[0:log_row_length] + '......' if len(e_str) > log_row_length else e_str
+                # if enable_traceback == "True":
+                if kwargs.get("enable_traceback", True):
+                    log.error(traceback.format_exc())
+                    log.error("(api_response) : %s" % log_e)
+                return Error(e), False
+        return inner_wrapper
+    return wrapper
+
+
+@api_request_catch()
+def api_request(_list, **kwargs):
+    if isinstance(_list, list):
+        func = _list[0]
+        if callable(func):
+            if kwargs.get("enable_traceback", True):
+                arg = _list[1:]
+                arg_str = str(arg)
+                log_arg = arg_str[0:log_row_length] + '......' if len(arg_str) > log_row_length else arg_str
+                log_kwargs = str(kwargs)[0:log_row_length] + '......' if len(str(kwargs)) > log_row_length else str(kwargs)
+                log.debug("(api_request)  : [%s] args: %s, kwargs: %s" % (func.__qualname__, log_arg, log_kwargs))
+            return func(*arg, **kwargs)
+    return False, False
+
+
+def logger_interceptor():
+    def wrapper(func):
+        def log_request(*arg, **kwargs):
+            if kwargs.get("enable_traceback", True):
+                arg = arg[1:]
+                arg_str = str(arg)
+                log_arg = arg_str[0:log_row_length] + '......' if len(arg_str) > log_row_length else arg_str
+                log_kwargs = str(kwargs)[0:log_row_length] + '......' if len(str(kwargs)) > log_row_length else str(kwargs)
+                log.debug("(api_request)  : [%s] args: %s, kwargs: %s" % (func.__name__, log_arg, log_kwargs))
+
+        def log_response(res, **kwargs):
+            if kwargs.get("enable_traceback", True):
+                res_str = str(res)
+                log_res = res_str[0:log_row_length] + '......' if len(res_str) > log_row_length else res_str
+                log.debug("(api_response) : [%s] %s " % (func.__name__, log_res))
+            return res, True
+
+        async def handler(*args, **kwargs):
+            _kwargs = copy.deepcopy(kwargs)
+            _kwargs.pop("enable_traceback", None)
+            check_task = kwargs.get("check_task", None)
+            check_items = kwargs.get("check_items", None)
+            try:
+                # log request
+                log_request(*args, **_kwargs)
+                # exec func
+                res = await func(*args, **_kwargs)
+                # log response
+                log_response(res, **_kwargs)
+                # check_response
+                check_res = ResponseChecker(res, sys._getframe().f_code.co_name, check_task, check_items, True).run()
+                return res, check_res
+            except Exception as e:
+                log.error(str(e))
+                e_str = str(e)
+                log_e = e_str[0:log_row_length] + '......' if len(e_str) > log_row_length else e_str
+                if kwargs.get("enable_traceback", True):
+                    log.error(traceback.format_exc())
+                    log.error("(api_response) : %s" % log_e)
+                check_res = ResponseChecker(Error(e), sys._getframe().f_code.co_name, check_task,
+                                            check_items, False).run()
+                return Error(e), check_res
+
+        return handler
+
+    return wrapper

tests/python_client/utils/util_birdwatcher.py

@@ -0,0 +1,79 @@
+import os
+import re
+from utils.util_log import test_log as log
+
+
+def extraction_all_data(text):
+    # Patterns to handle the specifics of each key-value line
+    patterns = {
+        'Segment ID': r"Segment ID:\s*(\d+)",
+        'Segment State': r"Segment State:\s*(\w+)",
+        'Collection ID': r"Collection ID:\s*(\d+)",
+        'PartitionID': r"PartitionID:\s*(\d+)",
+        'Insert Channel': r"Insert Channel:(.+)",
+        'Num of Rows': r"Num of Rows:\s*(\d+)",
+        'Max Row Num': r"Max Row Num:\s*(\d+)",
+        'Last Expire Time': r"Last Expire Time:\s*(.+)",
+        'Compact from': r"Compact from:\s*(\[\])",
+        'Start Position ID': r"Start Position ID:\s*(\[[\d\s]+\])",
+        'Start Position Time': r"Start Position ID:.*time:\s*(.+),",
+        'Start Channel Name': r"channel name:\s*([^,\n]+)",
+        'Dml Position ID': r"Dml Position ID:\s*(\[[\d\s]+\])",
+        'Dml Position Time': r"Dml Position ID:.*time:\s*(.+),",
+        'Dml Channel Name': r"channel name:\s*(.+)",
+        'Binlog Nums': r"Binlog Nums:\s*(\d+)",
+        'StatsLog Nums': r"StatsLog Nums:\s*(\d+)",
+        'DeltaLog Nums': r"DeltaLog Nums:\s*(\d+)"
+    }
+
+    refined_data = {}
+    for key, pattern in patterns.items():
+        match = re.search(pattern, text)
+        if match:
+            refined_data[key] = match.group(1).strip()
+
+    return refined_data
+
+
+class BirdWatcher:
+    """
+
+    birdwatcher is a cli tool to get information about milvus
+    the command:
+    show segment info
+    """
+
+    def __init__(self, etcd_endpoints, root_path):
+        self.prefix = f"birdwatcher --olc=\"#connect --etcd {etcd_endpoints} --rootPath={root_path},"
+
+    def parse_segment_info(self, output):
+        splitter = output.strip().split('\n')[0]
+        segments = output.strip().split(splitter)
+        segments = [segment for segment in segments if segment.strip()]
+
+        # Parse all segments
+        parsed_segments = [extraction_all_data(segment) for segment in segments]
+        parsed_segments = [segment for segment in parsed_segments if segment]
+        return parsed_segments
+
+    def show_segment_info(self, collection_id=None):
+        cmd = f"{self.prefix} show segment info --format table\""
+        if collection_id:
+            cmd = f"{self.prefix} show segment info --collection {collection_id} --format table\""
+        log.info(f"cmd: {cmd}")
+        output = os.popen(cmd).read()
+        # log.info(f"{cmd} output: {output}")
+        output = self.parse_segment_info(output)
+        for segment in output:
+            log.info(segment)
+        seg_res = {}
+        for segment in output:
+            seg_res[segment['Segment ID']] = segment
+        return seg_res
+
+
+if __name__ == "__main__":
+    birdwatcher = BirdWatcher("10.104.18.24:2379", "rg-test-613938")
+    res = birdwatcher.show_segment_info()
+    print(res)
+

tests/python_client/utils/util_common.py

@@ -0,0 +1,137 @@
+import glob
+import time
+from yaml import full_load
+import json
+import pandas as pd
+from utils.util_log import test_log as log
+
+def gen_experiment_config(yaml):
+    """load the yaml file of chaos experiment"""
+    with open(yaml) as f:
+        _config = full_load(f)
+        f.close()
+    return _config
+
+
+def findkeys(node, kv):
+    # refer to https://stackoverflow.com/questions/9807634/find-all-occurrences-of-a-key-in-nested-dictionaries-and-lists
+    if isinstance(node, list):
+        for i in node:
+            for x in findkeys(i, kv):
+               yield x
+    elif isinstance(node, dict):
+        if kv in node:
+            yield node[kv]
+        for j in node.values():
+            for x in findkeys(j, kv):
+                yield x
+
+
+def update_key_value(node, modify_k, modify_v):
+    # update the value of modify_k to modify_v
+    if isinstance(node, list):
+        for i in node:
+            update_key_value(i, modify_k, modify_v)
+    elif isinstance(node, dict):
+        if modify_k in node:
+            node[modify_k] = modify_v
+        for j in node.values():
+            update_key_value(j, modify_k, modify_v)
+    return node
+
+
+def update_key_name(node, modify_k, modify_k_new):
+    # update the name of modify_k to modify_k_new
+    if isinstance(node, list):
+        for i in node:
+            update_key_name(i, modify_k, modify_k_new)
+    elif isinstance(node, dict):
+        if modify_k in node:
+            value_backup = node[modify_k]
+            del node[modify_k]
+            node[modify_k_new] = value_backup
+        for j in node.values():
+            update_key_name(j, modify_k, modify_k_new)
+    return node
+
+
+def get_collections(file_name="all_collections.json"):
+    try:
+        with open(f"/tmp/ci_logs/{file_name}", "r") as f:
+            data = json.load(f)
+            collections = data["all"]
+    except Exception as e:
+        log.error(f"get_all_collections error: {e}")
+        return []
+    return collections
+
+
+def get_deploy_test_collections():
+    try:
+        with open("/tmp/ci_logs/deploy_test_all_collections.json", "r") as f:
+            data = json.load(f)
+            collections = data["all"]
+    except Exception as e:
+        log.error(f"get_all_collections error: {e}")
+        return []
+    return collections
+
+
+def get_chaos_test_collections():
+    try:
+        with open("/tmp/ci_logs/chaos_test_all_collections.json", "r") as f:
+            data = json.load(f)
+            collections = data["all"]
+    except Exception as e:
+        log.error(f"get_all_collections error: {e}")
+        return []
+    return collections
+
+
+def wait_signal_to_apply_chaos():
+    all_db_file = glob.glob("/tmp/ci_logs/event_records*.jsonl")
+    log.info(f"all files {all_db_file}")
+    ready_apply_chaos = True
+    timeout = 15*60
+    t0 = time.time()
+    for f in all_db_file:
+        while True and (time.time() - t0 < timeout):
+            try:
+                records = []
+                with open(f, 'r') as file:
+                    for line in file:
+                        line = line.strip()
+                        if line:
+                            records.append(json.loads(line))
+                df = pd.DataFrame(records) if records else pd.DataFrame(columns=["event_name", "event_status", "event_ts"])
+                log.debug(f"read {f}:result\n {df}")
+                result = df[(df['event_name'] == 'init_chaos') & (df['event_status'] == 'ready')]
+                if len(result) > 0:
+                    log.info(f"{f}: {result}")
+                    ready_apply_chaos = True
+                    break
+                else:
+                    ready_apply_chaos = False
+            except Exception as e:
+                log.error(f"read jsonl error: {e}")
+                ready_apply_chaos = False
+            time.sleep(10)
+
+    return ready_apply_chaos
+
+
+if __name__ == "__main__":
+    d = { "id" : "abcde",
+        "key1" : "blah",
+        "key2" : "blah blah",
+        "nestedlist" : [
+        { "id" : "qwerty",
+            "nestednestedlist" : [
+            { "id" : "xyz", "keyA" : "blah blah blah" },
+            { "id" : "fghi", "keyZ" : "blah blah blah" }],
+            "anothernestednestedlist" : [
+            { "id" : "asdf", "keyQ" : "blah blah" },
+            { "id" : "yuiop", "keyW" : "blah" }] } ] }
+    print(list(findkeys(d, 'id')))
+    update_key_value(d, "none_id", "ccc")
+    print(d)

tests/python_client/utils/util_fts.py

@@ -0,0 +1,356 @@
+import random
+import time
+import logging
+from typing import List, Dict, Optional, Tuple
+import pandas as pd
+from faker import Faker
+from pymilvus import (
+    FieldSchema,
+    CollectionSchema,
+    DataType,
+    Function,
+    FunctionType,
+    Collection,
+    connections,
+)
+from pymilvus import MilvusClient
+
+logger = logging.getLogger(__name__)
+
+
+class FTSMultiAnalyzerChecker:
+    """
+    Full-text search utility class providing various utility methods for full-text search testing.
+    Includes schema construction, test data generation, index creation, and more.
+    """
+
+    # Constant definitions
+    DEFAULT_TEXT_MAX_LENGTH = 8192
+    DEFAULT_LANG_MAX_LENGTH = 16
+    DEFAULT_DOC_ID_START = 100
+
+    # Faker multilingual instances as class attributes to avoid repeated creation
+    fake_en = Faker("en_US")
+    fake_zh = Faker("zh_CN")
+    fake_fr = Faker("fr_FR")
+    fake_jp = Faker("ja_JP")
+
+    def __init__(
+        self,
+        collection_name: str,
+        language_field_name: str,
+        text_field_name: str,
+        multi_analyzer_params: Optional[Dict] = None,
+        client: Optional[MilvusClient] = None,
+    ):
+        self.collection_name = collection_name
+        self.mock_collection_name = collection_name + "_mock"
+        self.language_field_name = language_field_name
+        self.text_field_name = text_field_name
+        self.multi_analyzer_params = (
+            multi_analyzer_params
+            if multi_analyzer_params is not None
+            else {
+                "by_field": self.language_field_name,
+                "analyzers": {
+                    "en": {"type": "english"},
+                    "zh": {"type": "chinese"},
+                    "icu": {
+                        "tokenizer": "icu",
+                        "filter": [{"type": "stop", "stop_words": [" "]}],
+                    },
+                    "default": {"tokenizer": "whitespace"},
+                },
+                "alias": {"chinese": "zh", "eng": "en", "fr": "icu", "jp": "icu"},
+            }
+        )
+        self.mock_multi_analyzer_params = {
+            "by_field": self.language_field_name,
+            "analyzers": {"default": {"tokenizer": "whitespace"}},
+        }
+        self.client = client
+        self.collection = None
+        self.mock_collection = None
+
+    def resolve_analyzer(self, lang: str) -> str:
+        """
+        Return the analyzer name according to the language.
+        Args:
+            lang (str): Language identifier
+        Returns:
+            str: Analyzer name
+        """
+        if lang in self.multi_analyzer_params["analyzers"]:
+            return lang
+        if lang in self.multi_analyzer_params.get("alias", {}):
+            return self.multi_analyzer_params["alias"][lang]
+        return "default"
+
+    def build_schema(self, multi_analyzer_params: dict) -> CollectionSchema:
+        """
+        Build a collection schema with multi-analyzer parameters.
+        Args:
+            multi_analyzer_params (dict): Analyzer parameters
+        Returns:
+            CollectionSchema: Constructed collection schema
+        """
+        fields = [
+            FieldSchema(name="doc_id", dtype=DataType.INT64, is_primary=True),
+            FieldSchema(
+                name=self.language_field_name,
+                dtype=DataType.VARCHAR,
+                max_length=self.DEFAULT_LANG_MAX_LENGTH,
+            ),
+            FieldSchema(
+                name=self.text_field_name,
+                dtype=DataType.VARCHAR,
+                max_length=self.DEFAULT_TEXT_MAX_LENGTH,
+                enable_analyzer=True,
+                multi_analyzer_params=multi_analyzer_params,
+            ),
+            FieldSchema(name="bm25_sparse_vector", dtype=DataType.SPARSE_FLOAT_VECTOR),
+        ]
+        schema = CollectionSchema(
+            fields=fields, description="Multi-analyzer BM25 schema test"
+        )
+        bm25_func = Function(
+            name="bm25",
+            function_type=FunctionType.BM25,
+            input_field_names=[self.text_field_name],
+            output_field_names=["bm25_sparse_vector"],
+        )
+        schema.add_function(bm25_func)
+        return schema
+
+    def init_collection(self) -> None:
+        """
+        Initialize Milvus collections, delete if exists first.
+        """
+        try:
+            if self.client.has_collection(self.collection_name):
+                self.client.drop_collection(self.collection_name)
+            if self.client.has_collection(self.mock_collection_name):
+                self.client.drop_collection(self.mock_collection_name)
+            self.collection = Collection(
+                name=self.collection_name,
+                schema=self.build_schema(self.multi_analyzer_params),
+            )
+            self.mock_collection = Collection(
+                name=self.mock_collection_name,
+                schema=self.build_schema(self.mock_multi_analyzer_params),
+            )
+        except Exception as e:
+            logger.error(f"collection init failed: {e}")
+            raise
+
+    def get_tokens_by_analyzer(self, text: str, analyzer_params: dict) -> List[str]:
+        """
+        Tokenize text according to analyzer parameters.
+        Args:
+            text (str): Text to be tokenized
+            analyzer_params (dict): Analyzer parameters
+        Returns:
+            List[str]: List of tokenized text
+        """
+        try:
+            res = self.client.run_analyzer(text, analyzer_params)
+            # Filter out tokens that are just whitespace
+            return [token for token in res.tokens if token.strip()]
+        except Exception as e:
+            logger.error(f"Tokenization failed: {e}")
+            return []
+
+    def generate_test_data(
+        self, num_rows: int = 3000, lang_list: Optional[List[str]] = None
+    ) -> List[Dict]:
+        """
+        Generate test data according to the schema, row count and language list.
+        Each row will contain language, article content and other fields.
+        Args:
+            num_rows (int): Number of data rows to generate
+            lang_list (Optional[List[str]]): List of languages
+        Returns:
+            List[Dict]: Generated test data list
+        """
+        if lang_list is None:
+            lang_list = ["en", "eng", "zh", "fr", "chinese", "jp", ""]
+        data = []
+        for i in range(num_rows):
+            lang = random.choice(lang_list)
+            # Generate article content according to language
+            if lang in ("en", "eng"):
+                content = self.fake_en.sentence()
+            elif lang in ("zh", "chinese"):
+                content = self.fake_zh.sentence()
+            elif lang == "fr":
+                content = self.fake_fr.sentence()
+            elif lang == "jp":
+                content = self.fake_jp.sentence()
+            else:
+                content = ""
+            row = {
+                "doc_id": i + self.DEFAULT_DOC_ID_START,
+                self.language_field_name: lang,
+                self.text_field_name: content,
+            }
+            data.append(row)
+        return data
+
+    def tokenize_data_by_multi_analyzer(
+        self, data_list: List[Dict], verbose: bool = False
+    ) -> List[Dict]:
+        """
+        Tokenize data according to multi-analyzer parameters.
+        Args:
+            data_list (List[Dict]): Data list
+            verbose (bool): Whether to print detailed information
+        Returns:
+            List[Dict]: Tokenized data list
+        """
+        data_list_tokenized = []
+        for row in data_list:
+            lang = row.get(self.language_field_name, None)
+            content = row.get(self.text_field_name, "")
+            doc_analyzer = self.resolve_analyzer(lang)
+            doc_analyzer_params = self.multi_analyzer_params["analyzers"][doc_analyzer]
+            content_tokens = self.get_tokens_by_analyzer(content, doc_analyzer_params)
+            tokenized_content = " ".join(content_tokens)
+            data_list_tokenized.append(
+                {
+                    "doc_id": row.get("doc_id"),
+                    self.language_field_name: lang,
+                    self.text_field_name: tokenized_content,
+                }
+            )
+        if verbose:
+            original_data = pd.DataFrame(data_list)
+            tokenized_data = pd.DataFrame(data_list_tokenized)
+            logger.info(f"Original data:\n{original_data}")
+            logger.info(f"Tokenized data:\n{tokenized_data}")
+        return data_list_tokenized
+
+    def insert_data(
+        self, data: List[Dict], verbose: bool = False
+    ) -> Tuple[List[Dict], List[Dict]]:
+        """
+        Insert test data and return original and tokenized data.
+        Args:
+            data (List[Dict]): Original data list
+            verbose (bool): Whether to print detailed information
+        Returns:
+            Tuple[List[Dict], List[Dict]]: (original data, tokenized data)
+        """
+        try:
+            self.collection.insert(data)
+            self.collection.flush()
+        except Exception as e:
+            logger.error(f"Failed to insert original data: {e}")
+            raise
+        t0 = time.time()
+        tokenized_data = self.tokenize_data_by_multi_analyzer(data, verbose=verbose)
+        t1 = time.time()
+        logger.info(f"Tokenization time: {t1 - t0}")
+        try:
+            self.mock_collection.insert(tokenized_data)
+            self.mock_collection.flush()
+        except Exception as e:
+            logger.error(f"Failed to insert tokenized data: {e}")
+            raise
+        return data, tokenized_data
+
+    def create_index(self) -> None:
+        """
+        Create BM25 index for sparse vector field.
+        """
+        for c in [self.collection, self.mock_collection]:
+            try:
+                c.create_index(
+                    "bm25_sparse_vector",
+                    {"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "BM25"},
+                )
+                c.load()
+            except Exception as e:
+                logger.error(f"Failed to create index: {e}")
+                raise
+
+    def search(
+        self, origin_query: str, tokenized_query: str, language: str, limit: int = 10
+    ) -> Tuple[list, list]:
+        """
+        Search interface, perform BM25 search on main and mock collections respectively.
+        Args:
+            origin_query (str): Original query text
+            tokenized_query (str): Tokenized query text
+            language (str): Query language
+            limit (int): Number of results to return
+        Returns:
+            Tuple[list, list]: (main collection results, mock collection results)
+        """
+        analyzer_name = self.resolve_analyzer(language)
+        search_params = {"metric_type": "BM25", "analyzer_name": analyzer_name}
+        logger.info(f"search_params: {search_params}")
+        try:
+            res = self.collection.search(
+                data=[origin_query],
+                anns_field="bm25_sparse_vector",
+                param=search_params,
+                output_fields=["doc_id"],
+                limit=limit,
+            )
+            mock_res = self.mock_collection.search(
+                data=[tokenized_query],
+                anns_field="bm25_sparse_vector",
+                param=search_params,
+                output_fields=["doc_id"],
+                limit=limit,
+            )
+            return res, mock_res
+        except Exception as e:
+            logger.error(f"Search failed: {e}")
+            return [], []
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    connections.connect("default", host="10.104.25.52", port="19530")
+    client = MilvusClient(uri="http://10.104.25.52:19530")
+    ft = FTSMultiAnalyzerChecker(
+        "test_collection", "language", "article_content", client=client
+    )
+    ft.init_collection()
+    ft.create_index()
+    language_list = ["jp", "en", "fr", "zh"]
+    data = ft.generate_test_data(1000, language_list)
+    _, tokenized_data = ft.insert_data(data)
+    search_sample_data = random.sample(tokenized_data, 10)
+    for row in search_sample_data:
+        tokenized_query = row[ft.text_field_name]
+        # Find the same doc_id in the original data and get the original query
+        # Use pandas to find the item with matching doc_id
+        # Convert data to DataFrame if it's not already
+        if not isinstance(data, pd.DataFrame):
+            data_df = pd.DataFrame(data)
+        else:
+            data_df = data
+        # Filter by doc_id and get the text field value
+        origin_query = data_df.loc[
+            data_df["doc_id"] == row["doc_id"], ft.text_field_name
+        ].iloc[0]
+        logger.info(f"Query: {tokenized_query}")
+        logger.info(f"Origin Query: {origin_query}")
+        language = row[ft.language_field_name]
+        logger.info(f"language: {language}")
+        res, mock_res = ft.search(origin_query, tokenized_query, language)
+        logger.info(f"Main collection search result: {res}")
+        logger.info(f"Mock collection search result: {mock_res}")
+        if res and mock_res:
+            res_set = set([r["doc_id"] for r in res[0]])
+            mock_res_set = set([r["doc_id"] for r in mock_res[0]])
+            res_diff = res_set - mock_res_set
+            mock_res_diff = mock_res_set - res_set
+            logger.info(f"Diff: {res_diff}, {mock_res_diff}")
+            if res_diff or mock_res_diff:
+                logger.error(
+                    f"Search results inconsistent: {res_diff}, {mock_res_diff}"
+                )
+                assert False

tests/python_client/utils/util_k8s.py

@@ -0,0 +1,494 @@
+import json
+import os.path
+import time
+
+import pyetcd
+import requests
+from common.common_type import in_cluster_env
+from common.milvus_sys import MilvusSys
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+from pymilvus import connections
+from utils.util_log import test_log as log
+
+
+def init_k8s_client_config():
+    """
+    init kubernetes client config
+    """
+    try:
+        in_cluster = os.getenv(in_cluster_env, default="False")
+        # log.debug(f"env variable IN_CLUSTER: {in_cluster}")
+        if in_cluster.lower() == "true":
+            config.load_incluster_config()
+        else:
+            config.load_kube_config()
+    except Exception as e:
+        raise Exception(e)
+
+
+def get_current_namespace():
+    init_k8s_client_config()
+    ns = config.list_kube_config_contexts()[1]["context"]["namespace"]
+    return ns
+
+
+def wait_pods_ready(namespace, label_selector, expected_num=None, timeout=360):
+    """
+    wait pods with label selector all ready
+
+    :param namespace: the namespace where the release
+    :type namespace: str
+
+    :param label_selector: labels to restrict which pods are waiting to be ready
+    :type label_selector: str
+
+    :param expected_num: expected the minimum number of pods to be ready if not None
+    :type expected_num: int
+
+    :param timeout: limits the duration of the call
+    :type timeout: int
+
+    :example:
+            >>> wait_pods_ready("default", "app.kubernetes.io/instance=scale-query", expected_num=9)
+    """
+    init_k8s_client_config()
+    api_instance = client.CoreV1Api()
+    try:
+        all_pos_ready_flag = False
+        t0 = time.time()
+        while not all_pos_ready_flag and time.time() - t0 < timeout:
+            api_response = api_instance.list_namespaced_pod(namespace=namespace, label_selector=label_selector)
+            all_pos_ready_flag = True
+            if expected_num is not None and len(api_response.items) < expected_num:
+                all_pos_ready_flag = False
+            else:
+                for item in api_response.items:
+                    if item.status.phase != "Running":
+                        all_pos_ready_flag = False
+                        break
+                    for c in item.status.container_statuses:
+                        log.debug(f"{c.name} status is {c.ready}")
+                        if c.ready is False:
+                            all_pos_ready_flag = False
+                            break
+            if not all_pos_ready_flag:
+                log.debug("all pods are not ready, please wait")
+                time.sleep(5)
+        if all_pos_ready_flag:
+            log.info(f"all pods in namespace {namespace} with label {label_selector} are ready")
+        else:
+            log.info(f"timeout for waiting all pods in namespace {namespace} with label {label_selector} ready")
+    except ApiException as e:
+        log.error(f"Exception when calling CoreV1Api->list_namespaced_pod: {e}\n")
+        raise Exception(str(e))
+
+    return all_pos_ready_flag
+
+
+def get_pod_list(namespace, label_selector):
+    """
+    get pod list with label selector
+
+    :param namespace: the namespace where the release
+    :type namespace: str
+
+    :param label_selector: labels to restrict which pods to list
+    :type label_selector: str
+
+    :example:
+            >>> get_pod_list("chaos-testing", "app.kubernetes.io/instance=test-proxy-pod-failure, component=proxy")
+    """
+    init_k8s_client_config()
+    api_instance = client.CoreV1Api()
+    try:
+        api_response = api_instance.list_namespaced_pod(namespace=namespace, label_selector=label_selector)
+        return api_response.items
+    except ApiException as e:
+        log.error(f"Exception when calling CoreV1Api->list_namespaced_pod: {e}\n")
+        raise Exception(str(e))
+
+
+def get_pod_container_names(namespace, pod_names):
+    """
+    get container names for pods
+
+    :param namespace: the namespace where pods are running
+    :type namespace: str
+
+    :param pod_names: pod names to inspect
+    :type pod_names: list[str]
+    """
+    init_k8s_client_config()
+    api_instance = client.CoreV1Api()
+    result = {}
+    try:
+        for pod_name in pod_names:
+            pod = api_instance.read_namespaced_pod(name=pod_name, namespace=namespace)
+            result[pod_name] = [c.name for c in pod.spec.containers]
+        return result
+    except ApiException as e:
+        log.error(f"Exception when calling CoreV1Api->read_namespaced_pod: {e}\n")
+        raise Exception(str(e))
+
+
+def get_pod_ip_name_pairs(namespace, label_selector):
+    """
+    get pod ip name pairs with label selector
+
+    :param namespace: the namespace where the release
+    :type namespace: str
+
+    :param label_selector: labels to restrict which pods to list
+    :type label_selector: str
+
+    :example:
+            >>> get_pod_ip_name_pairs("chaos-testing", "app.kubernetes.io/instance=test-proxy-pod-failure, component=querynode")
+    """
+    m = dict()
+    items = get_pod_list(namespace, label_selector)
+    for item in items:
+        ip = item.status.pod_ip
+        name = item.metadata.name
+        m[ip] = name
+    return m
+
+
+def get_querynode_id_pod_pairs(namespace, label_selector):
+    """
+    get milvus node id and corresponding pod name pairs with label selector
+
+    :param namespace: the namespace where the release
+    :type namespace: str
+
+    :param label_selector: labels to restrict which pods to list
+    :type label_selector: str
+
+    :example:
+            >>> querynode_id_pod_pair = get_querynode_id_pod_pairs("chaos-testing", "app.kubernetes.io/instance=milvus-multi-querynode, component=querynode")
+            {
+             5: 'milvus-multi-querynode-querynode-7b8f4b5c5-4pn42',
+             9: 'milvus-multi-querynode-querynode-7b8f4b5c5-99tx7',
+             1: 'milvus-multi-querynode-querynode-7b8f4b5c5-w9sk8',
+             3: 'milvus-multi-querynode-querynode-7b8f4b5c5-xx84j',
+             6: 'milvus-multi-querynode-querynode-7b8f4b5c5-x95dp'
+            }
+    """
+    # TODO: extend this function to other worker nodes, not only querynode
+    querynode_ip_pod_pair = get_pod_ip_name_pairs(namespace, label_selector)
+    querynode_id_pod_pair = {}
+    ms = MilvusSys()
+    for node in ms.query_nodes:
+        ip = node["infos"]["hardware_infos"]["ip"].split(":")[0]
+        querynode_id_pod_pair[node["identifier"]] = querynode_ip_pod_pair[ip]
+    return querynode_id_pod_pair
+
+
+def get_milvus_instance_name(namespace, host="127.0.0.1", port="19530", milvus_sys=None):
+    """
+    get milvus instance name after connection
+
+    :param namespace: the namespace where the release
+    :type namespace: str
+
+    :param host: milvus host ip
+    :type host: str
+
+    :param port: milvus port
+    :type port: str
+    :example:
+            >>> milvus_instance_name = get_milvus_instance_name("chaos-testing", "10.96.250.111")
+            "milvus-multi-querynode"
+
+    """
+    if milvus_sys is None:
+        connections.add_connection(_default={"host": host, "port": port})
+        connections.connect(alias="_default")
+        ms = MilvusSys()
+    else:
+        ms = milvus_sys
+    query_node_ip = ms.query_nodes[0]["infos"]["hardware_infos"]["ip"].split(":")[0]
+    ip_name_pairs = get_pod_ip_name_pairs(namespace, "app.kubernetes.io/name=milvus")
+    pod_name = ip_name_pairs[query_node_ip]
+
+    init_k8s_client_config()
+    api_instance = client.CoreV1Api()
+    try:
+        api_response = api_instance.read_namespaced_pod(namespace=namespace, name=pod_name)
+    except ApiException as e:
+        log.error(f"Exception when calling CoreV1Api->list_namespaced_pod: {e}\n")
+        raise Exception(str(e))
+    milvus_instance_name = api_response.metadata.labels["app.kubernetes.io/instance"]
+    return milvus_instance_name
+
+
+def get_milvus_deploy_tool(namespace, milvus_sys):
+    """
+    get milvus instance name after connection
+    :param namespace: the namespace where the release
+    :type namespace: str
+    :param milvus_sys: milvus_sys
+    :type namespace: MilvusSys
+    :example:
+            >>> deploy_tool = get_milvus_deploy_tool("chaos-testing", milvus_sys)
+            "helm"
+    """
+    ms = milvus_sys
+    query_node_ip = ms.query_nodes[0]["infos"]["hardware_infos"]["ip"].split(":")[0]
+    ip_name_pairs = get_pod_ip_name_pairs(namespace, "app.kubernetes.io/name=milvus")
+    pod_name = ip_name_pairs[query_node_ip]
+    init_k8s_client_config()
+    api_instance = client.CoreV1Api()
+    try:
+        api_response = api_instance.read_namespaced_pod(namespace=namespace, name=pod_name)
+    except ApiException as e:
+        log.error(f"Exception when calling CoreV1Api->list_namespaced_pod: {e}\n")
+        raise Exception(str(e))
+    if (
+        "app.kubernetes.io/managed-by" in api_response.metadata.labels
+        and api_response.metadata.labels["app.kubernetes.io/managed-by"] == "milvus-operator"
+    ):
+        deploy_tool = "milvus-operator"
+    else:
+        deploy_tool = "helm"
+    return deploy_tool
+
+
+def export_pod_logs(namespace, label_selector, release_name=None):
+    """
+    export pod logs with label selector to '/tmp/milvus'
+
+    :param namespace: the namespace where the release
+    :type namespace: str
+
+    :param label_selector: labels to restrict which pods logs to export
+    :type label_selector: str
+
+    :param release_name: use the release name as server logs director name
+    :type label_selector: str
+
+    :example:
+            >>> export_pod_logs("chaos-testing", "app.kubernetes.io/instance=mic-milvus")
+    """
+    if isinstance(release_name, str):
+        if len(release_name.strip()) == 0:
+            raise ValueError("Got an unexpected space release_name")
+    else:
+        raise TypeError("Got an unexpected non-string release_name")
+    pod_log_path = "/tmp/milvus_logs" if release_name is None else f"/tmp/milvus_logs/{release_name}"
+
+    if not os.path.isdir(pod_log_path):
+        os.makedirs(pod_log_path)
+
+    # get pods and export logs
+    items = get_pod_list(namespace, label_selector=label_selector)
+    try:
+        for item in items:
+            pod_name = item.metadata.name
+            os.system(f"kubectl logs {pod_name} > {pod_log_path}/{pod_name}.log 2>&1")
+    except Exception as e:
+        log.error(f"Exception when export pod {pod_name} logs: %s\n" % e)
+        raise Exception(str(e))
+
+
+def read_pod_log(namespace, label_selector, release_name):
+    init_k8s_client_config()
+    items = get_pod_list(namespace, label_selector=label_selector)
+
+    try:
+        # export log to /tmp/release_name path
+        pod_log_path = f"/tmp/milvus_logs/{release_name}"
+        if not os.path.isdir(pod_log_path):
+            os.makedirs(pod_log_path)
+
+        api_instance = client.CoreV1Api()
+
+        for item in items:
+            pod = item.metadata.name
+            log.debug(f"Start to read {pod} log")
+            logs = api_instance.read_namespaced_pod_log(name=pod, namespace=namespace, async_req=True)
+            with open(f"{pod_log_path}/{pod}.log", "w") as f:
+                f.write(logs.get())
+
+    except ApiException as e:
+        log.error(f"Exception when read pod {pod} logs: %s\n" % e)
+        raise Exception(str(e))
+
+
+def get_metrics_querynode_sq_req_count():
+    """get metric milvus_querynode_collection_num from prometheus"""
+
+    PROMETHEUS = "http://10.96.7.6:9090"
+    query_str = (
+        'milvus_querynode_sq_req_count{app_kubernetes_io_instance="mic-replica",'
+        'app_kubernetes_io_name="milvus",namespace="chaos-testing"}'
+    )
+
+    response = requests.get(PROMETHEUS + "/api/v1/query", params={"query": query_str})
+    if response.status_code == 200:
+        results = response.json()["data"]["result"]
+        # print(results)
+        # print(type(results))
+        log.debug(json.dumps(results, indent=4))
+        milvus_querynode_sq_req_count = {}
+        for res in results:
+            if res["metric"]["status"] == "total":
+                querynode_id = res["metric"]["node_id"]
+                # pod = res["metric"]["pod"]
+                value = res["value"][-1]
+                milvus_querynode_sq_req_count[int(querynode_id)] = int(value)
+        # log.debug(milvus_querynode_sq_req_count)
+        return milvus_querynode_sq_req_count
+    else:
+        raise Exception(-1, f"Failed to get metrics with status code {response.status_code}")
+
+
+def get_svc_ip(namespace, label_selector):
+    """get svc ip from svc list"""
+    init_k8s_client_config()
+    api_instance = client.CoreV1Api()
+    try:
+        api_response = api_instance.list_namespaced_service(namespace=namespace, label_selector=label_selector)
+    except ApiException as e:
+        log.error(f"Exception when calling CoreV1Api->list_namespaced_service: {e}\n")
+        raise Exception(str(e))
+    svc_ip = api_response.items[0].spec.cluster_ip
+    return svc_ip
+
+
+def parse_etcdctl_table_output(output):
+    """parse etcdctl table output"""
+    output = output.split("\n")
+    title = []
+    data = []
+    for line in output:
+        if "ENDPOINT" in line:
+            title = [x.strip(" ") for x in line.strip("|").split("|")]
+        if ":" in line:
+            data.append([x.strip(" ") for x in line.strip("|").split("|")])
+    return title, data
+
+
+def get_etcd_leader(release_name, deploy_tool="helm"):
+    """get etcd leader by etcdctl"""
+    pod_list = []
+    if deploy_tool == "helm":
+        label_selector = f"app.kubernetes.io/instance={release_name}-etcd, app.kubernetes.io/name=etcd"
+        pod_list = get_pod_list("chaos-testing", label_selector)
+        if len(pod_list) == 0:
+            label_selector = f"app.kubernetes.io/instance={release_name}, app.kubernetes.io/name=etcd"
+            pod_list = get_pod_list("chaos-testing", label_selector)
+    if deploy_tool == "operator":
+        label_selector = f"app.kubernetes.io/instance={release_name}, app.kubernetes.io/name=etcd"
+        pod_list = get_pod_list("chaos-testing", label_selector)
+    leader = None
+    for pod in pod_list:
+        endpoint = f"{pod.status.pod_ip}:2379"
+        cmd = f"etcdctl --endpoints={endpoint} endpoint status -w table"
+        output = os.popen(cmd).read()
+        log.info(f"etcdctl output: {output}")
+        title, data = parse_etcdctl_table_output(output)
+        idx = title.index("IS LEADER")
+        if data[0][idx] == "true":
+            leader = pod.metadata.name
+    log.info(f"etcd leader is {leader}")
+    return leader
+
+
+def get_etcd_followers(release_name, deploy_tool="helm"):
+    """get etcd follower by etcdctl"""
+    pod_list = []
+    if deploy_tool == "helm":
+        label_selector = f"app.kubernetes.io/instance={release_name}-etcd, app.kubernetes.io/name=etcd"
+        pod_list = get_pod_list("chaos-testing", label_selector)
+        if len(pod_list) == 0:
+            label_selector = f"app.kubernetes.io/instance={release_name}, app.kubernetes.io/name=etcd"
+            pod_list = get_pod_list("chaos-testing", label_selector)
+    if deploy_tool == "operator":
+        label_selector = f"app.kubernetes.io/instance={release_name}, app.kubernetes.io/name=etcd"
+        pod_list = get_pod_list("chaos-testing", label_selector)
+    followers = []
+    for pod in pod_list:
+        endpoint = f"{pod.status.pod_ip}:2379"
+        cmd = f"etcdctl --endpoints={endpoint} endpoint status -w table"
+        output = os.popen(cmd).read()
+        log.info(f"etcdctl output: {output}")
+        title, data = parse_etcdctl_table_output(output)
+        idx = title.index("IS LEADER")
+        if data[0][idx] == "false":
+            followers.append(pod.metadata.name)
+    log.info(f"etcd followers are {followers}")
+    return followers
+
+
+def find_activate_standby_coord_pod(namespace, release_name, coord_type):
+    init_k8s_client_config()
+    api_instance = client.CoreV1Api()
+    etcd_service_name = release_name + "-etcd"
+    service = api_instance.read_namespaced_service(name=etcd_service_name, namespace=namespace)
+    etcd_cluster_ip = service.spec.cluster_ip
+    etcd_port = service.spec.ports[0].port
+    etcd = pyetcd.client(host=etcd_cluster_ip, port=etcd_port)
+    v = etcd.get(f"by-dev/meta/session/{coord_type}")
+    log.info(f"coord_type: {coord_type}, etcd session value: {v}")
+    activated_pod_ip = json.loads(v[0])["Address"].split(":")[0]
+    label_selector = f"app.kubernetes.io/instance={release_name}, component={coord_type}"
+    items = get_pod_list(namespace, label_selector=label_selector)
+    all_pod_list = []
+    for item in items:
+        pod_name = item.metadata.name
+        all_pod_list.append(pod_name)
+    activate_pod_list = []
+    standby_pod_list = []
+    for item in items:
+        pod_name = item.metadata.name
+        ip = item.status.pod_ip
+        if ip == activated_pod_ip:
+            activate_pod_list.append(pod_name)
+    standby_pod_list = list(set(all_pod_list) - set(activate_pod_list))
+    return activate_pod_list, standby_pod_list
+
+
+def record_time_when_standby_activated(namespace, release_name, coord_type, timeout=360):
+    activate_pod_list_before, standby_pod_list_before = find_activate_standby_coord_pod(
+        namespace, release_name, coord_type
+    )
+    log.info(
+        f"check standby switch: activate_pod_list_before {activate_pod_list_before}, "
+        f"standby_pod_list_before {standby_pod_list_before}"
+    )
+    standby_activated = False
+    activate_pod_list_after, standby_pod_list_after = find_activate_standby_coord_pod(
+        namespace, release_name, coord_type
+    )
+    start_time = time.time()
+    end_time = time.time()
+    while not standby_activated and end_time - start_time < timeout:
+        try:
+            activate_pod_list_after, standby_pod_list_after = find_activate_standby_coord_pod(
+                namespace, release_name, coord_type
+            )
+            if activate_pod_list_after[0] in standby_pod_list_before:
+                standby_activated = True
+                log.info(f"Standby {coord_type} pod {activate_pod_list_after[0]} activated")
+                log.info(
+                    f"check standby switch: activate_pod_list_after {activate_pod_list_after}, "
+                    f"standby_pod_list_after {standby_pod_list_after}"
+                )
+                break
+        except Exception as e:
+            log.error(f"Exception when check standby switch: {e}")
+        time.sleep(1)
+        end_time = time.time()
+    if standby_activated:
+        log.info(f"Standby {coord_type} pod {activate_pod_list_after[0]} activated")
+    else:
+        log.info(f"Standby {coord_type} pod does not switch standby mode")
+
+
+if __name__ == "__main__":
+    label = "app.kubernetes.io/name=milvus, component=querynode"
+    instance_name = get_milvus_instance_name("chaos-testing", "10.96.250.111")
+    res = get_pod_list("chaos-testing", label_selector=label)
+    m = get_pod_ip_name_pairs("chaos-testing", label_selector=label)
+    export_pod_logs(namespace="chaos-testing", label_selector=label)

tests/python_client/utils/util_log.py

@@ -0,0 +1,30 @@
+import logging
+import sys
+
+from config.log_config import log_config
+
+
+class TestLog:
+    def __init__(self, logger):
+        self.logger = logger
+        self.log = logging.getLogger(self.logger)
+        self.log.setLevel(logging.DEBUG)
+
+        # Only add console handler if needed (commented out by default)
+        # All file logging is handled by ConditionalLogHandler plugin
+        try:
+            formatter = logging.Formatter("[%(asctime)s - %(levelname)s - %(name)s]: "
+                                          "%(message)s (%(filename)s:%(lineno)s)")
+
+            # Stream handler (commented out by default)
+            ch = logging.StreamHandler(sys.stdout)
+            ch.setLevel(logging.DEBUG)
+            ch.setFormatter(formatter)
+            # self.log.addHandler(ch)
+
+        except Exception as e:
+            print("Failed to initialize logger: %s" % str(e))
+
+
+"""All modules share this unified log"""
+test_log = TestLog('ci_test').log

tests/python_client/utils/wrapper.py

@@ -0,0 +1,70 @@
+import time
+from datetime import datetime
+import functools
+from utils.util_log import test_log as log
+
+DEFAULT_FMT = '[{start_time}] [{elapsed:0.8f}s] {collection_name} {func_name} -> {res!r}'
+
+
+def trace(fmt=DEFAULT_FMT, prefix='test', flag=True):
+    def decorate(func):
+        @functools.wraps(func)
+        def inner_wrapper(*args, **kwargs):
+            # args[0] is an instance of ApiCollectionWrapper class
+            flag = args[0].active_trace
+            if flag:
+                start_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
+                t0 = time.perf_counter()
+                res, result = func(*args, **kwargs)
+                elapsed = time.perf_counter() - t0
+                end_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
+                func_name = func.__name__
+                collection_name = args[0].collection.name
+                # arg_lst = [repr(arg) for arg in args[1:]][:100]
+                # arg_lst.extend(f'{k}={v!r}' for k, v in kwargs.items())
+                # arg_str = ', '.join(arg_lst)[:200]
+
+                log_str = f"[{prefix}]" + fmt.format(**locals())
+                # TODO: add report function in this place, like uploading to influxdb
+                # it is better a async way to do this, in case of blocking the request processing
+                log.info(log_str)
+                return res, result
+            else:
+                res, result = func(*args, **kwargs)
+                return res, result
+
+        return inner_wrapper
+
+    return decorate
+
+
+def counter(func):
+    """ count func succ rate """
+    def inner_wrapper(*args, **kwargs):
+        """ inner wrapper """
+        result, is_succ = func(*args, **kwargs)
+        inner_wrapper.total += 1
+        if is_succ:
+            inner_wrapper.succ += 1
+        else:
+            inner_wrapper.fail += 1
+        return result, is_succ
+
+    inner_wrapper.name = func.__name__
+    inner_wrapper.total = 0
+    inner_wrapper.succ = 0
+    inner_wrapper.fail = 0
+    return inner_wrapper
+
+
+if __name__ == '__main__':
+    @trace()
+    def snooze(seconds, name='snooze'):
+        time.sleep(seconds)
+        return name
+        # print(f"name: {name}")
+
+
+    for i in range(3):
+        res = snooze(.123, name=i)
+        print("res:", res)