chore: import upstream snapshot with attribution

tests/python_client/deploy/README.md

@@ -0,0 +1,59 @@
+
+
+## Overview
+To test deployment by docker-compose(Both standalone and cluster)
+
+* re-install milvus to check data persistence
+    1. Deploy Milvus
+    2. Insert data
+    3. Build index 
+    4. Search
+    5. Stop Milvus
+    6. Repeat from step #1
+* upgrade milvus to check data compatibility
+    1. Deploy Milvus (Previous Release)
+    2. Insert data
+    3. Search
+    4. Stop Milvus
+    5. Deploy Milvus  (Latest Release/Build)
+    6. Build index
+    7. Search
+
+## Project structure
+```
+.
+├── README.md
+├── cluster # dir to deploy cluster
+│   ├── logs # dir to save logs
+│   └──docker-compose.yml
+├── standalone # dir to deploy standalone
+│   ├── logs # dir to save logs
+│   └──docker-compose.yml
+├── scripts
+│   ├── action_after_upgrade.py
+│   ├── action_before_upgrade.py
+│   ├── action_reinstall.py
+│   └── utils.py
+├── cluster-values.yaml # config for helm deployment
+├── test.sh # script to run a single task
+└── run.sh # script to run all tasks
+```
+
+## Usage
+Make sure you have installed `docker`,`docker-compose` and `pymilvus`!
+For different version, you should modify the value of `latest_tag`, `latest_rc_tag` and `Release`. Password of root is needed for deleting volumes dir.
+
+single test task
+
+```bash
+$ bash test.sh -m ${Mode} -t ${Task} -p ${Password}
+# Mode, the mode of milvus deploy. standalone or cluster"
+# Task, the task type of test. reinstall or upgrade
+# Password, the password of root"
+```
+
+run all tasks
+```bash
+$ bash run.sh -p ${Password}
+# Password, the password of root"
+```

tests/python_client/deploy/base.py

@@ -0,0 +1,10 @@
+from base.client_base import TestcaseBase
+from utils.util_log import test_log as log
+
+
+class TestDeployBase(TestcaseBase):
+
+    def teardown_method(self, method):
+        log.info(("*" * 35) + " teardown " + ("*" * 35))
+        log.info("[teardown_method] Start teardown test case %s..." % method.__name__)
+        log.info("skip drop collection")

tests/python_client/deploy/check_healthy.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#to check containers all running and minio is healthy
+function check_healthy {
+    Expect=$(yq '.services | length' 'docker-compose.yml')
+    Expect_health=$(yq '.services' 'docker-compose.yml' |grep 'healthcheck'|wc -l)
+    cnt=$(docker compose ps | grep -E "running|Running|Up|up" | wc -l)
+    healthy=$(docker compose ps | grep "healthy" | wc -l)
+    time_cnt=0
+    echo "running num $cnt expect num $Expect"
+    echo "healthy num $healthy expect num $Expect_health"
+    while [[ $cnt -ne $Expect || $healthy -ne 1 ]];
+    do
+    printf "waiting all containers getting running\n"
+    sleep 5
+    let time_cnt+=5
+    # if time is greater than 300s, the condition still not satisfied, we regard it as a failure
+    if [ $time_cnt -gt 300 ];
+    then
+        printf "timeout,there are some issues with deployment!"
+        exit 1
+    fi
+    cnt=$(docker compose ps | grep -E "running|Running|Up|up" | wc -l)
+    healthy=$(docker compose ps | grep "healthy" | wc -l)
+    echo "running num $cnt expect num $Expect"
+    echo "healthy num $healthy expect num $Expect_health"
+    done
+}
+
+check_healthy

tests/python_client/deploy/cluster-values.yaml

@@ -0,0 +1,136 @@
+cluster:
+  enabled: true
+log:
+  level: debug
+image:
+  all:
+    repository: milvusdb/milvus
+    tag: master-latest
+    pullPolicy: IfNotPresent
+
+etcd:
+  replicaCount: 3
+  image:
+    repository: milvusdb/etcd
+    tag: 3.5.5-r2
+
+minio:
+  resources:
+    requests:
+      memory: 256Mi
+
+kafka:
+  enabled: false
+  replicaCount: 3
+
+pulsar:
+  enabled: true
+  extra:
+    bastion: no
+    wsproxy: no
+  
+  autorecovery:
+    resources:
+      requests:
+        cpu: 0.1
+        memory: 256Mi
+  proxy:
+    replicaCount: 1
+    resources:
+      requests:
+        cpu: 0.1
+        memory: 256Mi
+    wsResources:
+      requests:
+        memory: 256Mi
+        cpu: 0.1
+    configData:
+      PULSAR_MEM: >
+        -Xms256m -Xmx256m
+      PULSAR_GC: >
+        -XX:MaxDirectMemorySize=512m
+      httpNumThreads: "50"      
+
+  bookkeeper:
+    replicaCount: 2
+    resources:
+      requests:
+        cpu: 0.1
+        memory: 512Mi
+    configData:
+      PULSAR_MEM: >
+        -Xms512m
+        -Xmx512m
+        -XX:MaxDirectMemorySize=1024m
+      PULSAR_GC: >
+        -Dio.netty.leakDetectionLevel=disabled
+        -Dio.netty.recycler.linkCapacity=1024
+        -XX:+UseG1GC -XX:MaxGCPauseMillis=10
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:ParallelGCThreads=32
+        -XX:ConcGCThreads=32
+        -XX:G1NewSizePercent=50
+        -XX:+DisableExplicitGC
+        -XX:-ResizePLAB
+        -XX:+ExitOnOutOfMemoryError
+        -XX:+PerfDisableSharedMem
+        -XX:+PrintGCDetails
+      nettyMaxFrameSizeBytes: "104867840"
+  zookeeper:
+    replicaCount: 1
+    resources:
+      requests:
+        cpu: 0.1
+        memory: 256Mi
+    configData:
+      PULSAR_MEM: >
+        -Xms512m
+        -Xmx512m
+      PULSAR_GC: >
+         -Dcom.sun.management.jmxremote
+         -Djute.maxbuffer=10485760
+         -XX:+ParallelRefProcEnabled
+         -XX:+UnlockExperimentalVMOptions
+         -XX:+DoEscapeAnalysis
+         -XX:+DisableExplicitGC
+         -XX:+PerfDisableSharedMem
+         -Dzookeeper.forceSync=no
+  broker:
+    replicaCount: 1
+    resources:
+      requests:
+        cpu: 0.1
+        memory: 512Mi
+    configData:
+      PULSAR_MEM: >
+        -Xms512m
+        -Xmx512m
+        -XX:MaxDirectMemorySize=1024m
+      PULSAR_GC: >
+        -Dio.netty.leakDetectionLevel=disabled
+        -Dio.netty.recycler.linkCapacity=1024
+        -XX:+ParallelRefProcEnabled
+        -XX:+UnlockExperimentalVMOptions
+        -XX:+DoEscapeAnalysis
+        -XX:ParallelGCThreads=32
+        -XX:ConcGCThreads=32
+        -XX:G1NewSizePercent=50
+        -XX:+DisableExplicitGC
+        -XX:-ResizePLAB
+        -XX:+ExitOnOutOfMemoryError
+      maxMessageSize: "104857600"
+      defaultRetentionTimeInMinutes: "10080"
+      defaultRetentionSizeInMB: "8192"
+      backlogQuotaDefaultLimitGB: "8"
+      backlogQuotaDefaultRetentionPolicy: producer_exception
+
+extraConfigFiles:
+  user.yaml: |+
+    dataCoord:
+      compaction:
+        indexBasedCompaction: false
+    indexCoord:
+      scheduler:
+        interval: 100        

tests/python_client/deploy/common.py

@@ -0,0 +1,65 @@
+import json
+from utils.util_log import test_log as log
+
+all_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", "HNSW", "BIN_FLAT", "BIN_IVF_FLAT"]
+
+default_index_params = [{"nlist": 128}, {"nlist": 128}, {"nlist": 128}, {"nlist": 128, "m": 16, "nbits": 8},
+                        {"M": 48, "efConstruction": 500}, {"nlist": 128}, {"nlist": 128}]
+
+index_params_map = dict(zip(all_index_types, default_index_params))
+
+def gen_index_param(index_type):
+    metric_type = "L2"
+    if "BIN" in index_type:
+        metric_type = "HAMMING"
+    index_param = {
+        "index_type": index_type,
+        "params": index_params_map[index_type],
+        "metric_type": metric_type
+    }
+    return index_param
+
+
+def gen_search_param(index_type, metric_type="L2"):
+    search_params = []
+    if index_type in ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"]:
+        for nprobe in [10]:
+            ivf_search_params = {"metric_type": metric_type, "params": {"nprobe": nprobe}}
+            search_params.append(ivf_search_params)
+    elif index_type in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+        for nprobe in [10]:
+            bin_search_params = {"metric_type": "HAMMING", "params": {"nprobe": nprobe}}
+            search_params.append(bin_search_params)
+    elif index_type in ["HNSW"]:
+        for ef in [64]:
+            hnsw_search_param = {"metric_type": metric_type, "params": {"ef": ef}}
+            search_params.append(hnsw_search_param)
+    elif index_type == "ANNOY":
+        for search_k in [1000]:
+            annoy_search_param = {"metric_type": metric_type, "params": {"search_k": search_k}}
+            search_params.append(annoy_search_param)
+    else:
+        print("Invalid index_type.")
+        raise Exception("Invalid index_type.")
+    return search_params
+
+
+def get_deploy_test_collections():
+    try:
+        with open("/tmp/ci_logs/deploy_test_all_collections.json", "r") as f:
+            data = json.load(f)
+            collections = data["all"]
+    except Exception as e:
+        log.error(f"get_all_collections error: {e}")
+        return []
+    return collections
+
+def get_chaos_test_collections():
+    try:
+        with open("/tmp/ci_logs/chaos_test_all_collections.json", "r") as f:
+            data = json.load(f)
+            collections = data["all"]
+    except Exception as e:
+        log.error(f"get_all_collections error: {e}")
+        return []
+    return collections

tests/python_client/deploy/conftest.py

@@ -0,0 +1,22 @@
+import logging
+
+import pytest
+import functools
+import socket
+
+import common.common_type as ct
+import common.common_func as cf
+from utils.util_log import test_log as log
+from common.common_func import param_info
+from check.param_check import ip_check, number_check
+from config.log_config import log_config
+from utils.util_pymilvus import get_milvus, gen_unique_str, gen_default_fields, gen_binary_default_fields
+from pymilvus.orm.types import CONSISTENCY_STRONG
+
+timeout = 60
+dimension = 128
+delete_timeout = 60
+
+
+# add a fixture for all index?
+

tests/python_client/deploy/milvus_crd.yaml

@@ -0,0 +1,160 @@
+# This is a sample to deploy a milvus cluster using pulsar with minimum cost of resources.
+apiVersion: milvus.io/v1beta1
+kind: Milvus
+metadata:
+  name: operator-demo
+  namespace: chaos-testing
+  labels:
+    app: milvus
+spec:
+  mode: cluster
+  config:
+    dataNode:
+      memory:
+        forceSyncEnable: false
+    rootCoord:
+      enableActiveStandby: true
+    dataCoord:
+      enableActiveStandby: true
+    queryCoord:
+      enableActiveStandby: true
+    indexCoord:
+      enableActiveStandby: true
+#    mixCoord:
+#      enableActiveStandby: true
+    quotaAndLimits:
+      enable: false
+    log:
+      level: debug
+  components:
+    enableRollingUpdate: true
+    imageUpdateMode: rollingUpgrade
+    image: harbor.milvus.io/milvus/milvus:master-20240426-4fb8044a-amd64
+    disableMetric: false
+    dataNode:
+      replicas: 3
+    indexNode:
+      replicas: 3
+    queryNode:
+      replicas: 3
+  dependencies:
+    msgStreamType: kafka
+    etcd:
+      inCluster:
+        deletionPolicy: Retain
+        pvcDeletion: false
+        values:
+          replicaCount: 3
+    kafka:
+      inCluster:
+        deletionPolicy: Retain
+        pvcDeletion: false
+        values:
+          replicaCount: 3
+          defaultReplicationFactor: 2
+          metrics:
+            kafka:
+              enabled: true
+              serviceMonitor:
+                enabled: true
+            jmx:
+              enabled: true
+    pulsar:
+      inCluster:
+        deletionPolicy: Retain
+        pvcDeletion: false
+        values:
+          components:
+            autorecovery: false
+            functions: false
+            toolset: false
+            pulsar_manager: false
+          monitoring:
+            prometheus: false
+            grafana: false
+            node_exporter: false
+            alert_manager: false
+          proxy:
+            replicaCount: 1
+            resources:
+              requests:
+                cpu: 0.01
+                memory: 256Mi
+            configData:
+              PULSAR_MEM: >
+                -Xms256m -Xmx256m
+              PULSAR_GC: >
+                -XX:MaxDirectMemorySize=256m
+          bookkeeper:
+            replicaCount: 2
+            resources:
+              requests:
+                cpu: 0.01
+                memory: 256Mi
+            configData:
+              PULSAR_MEM: >
+                -Xms256m
+                -Xmx256m
+                -XX:MaxDirectMemorySize=256m
+              PULSAR_GC: >
+                -Dio.netty.leakDetectionLevel=disabled
+                -Dio.netty.recycler.linkCapacity=1024
+                -XX:+UseG1GC -XX:MaxGCPauseMillis=10
+                -XX:+ParallelRefProcEnabled
+                -XX:+UnlockExperimentalVMOptions
+                -XX:+DoEscapeAnalysis
+                -XX:ParallelGCThreads=32
+                -XX:ConcGCThreads=32
+                -XX:G1NewSizePercent=50
+                -XX:+DisableExplicitGC
+                -XX:-ResizePLAB
+                -XX:+ExitOnOutOfMemoryError
+                -XX:+PerfDisableSharedMem
+                -XX:+PrintGCDetails
+          zookeeper:
+            replicaCount: 1
+            resources:
+              requests:
+                cpu: 0.01
+                memory: 256Mi
+            configData:
+              PULSAR_MEM: >
+                -Xms256m
+                -Xmx256m
+              PULSAR_GC: >
+                -Dcom.sun.management.jmxremote
+                -Djute.maxbuffer=10485760
+                -XX:+ParallelRefProcEnabled
+                -XX:+UnlockExperimentalVMOptions
+                -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC
+                -XX:+PerfDisableSharedMem
+                -Dzookeeper.forceSync=no
+          broker:
+            replicaCount: 1
+            resources:
+              requests:
+                cpu: 0.01
+                memory: 256Mi
+            configData:
+              PULSAR_MEM: >
+                -Xms256m
+                -Xmx256m
+              PULSAR_GC: >
+                -XX:MaxDirectMemorySize=256m
+                -Dio.netty.leakDetectionLevel=disabled
+                -Dio.netty.recycler.linkCapacity=1024
+                -XX:+ParallelRefProcEnabled
+                -XX:+UnlockExperimentalVMOptions
+                -XX:+DoEscapeAnalysis
+                -XX:ParallelGCThreads=32
+                -XX:ConcGCThreads=32
+                -XX:G1NewSizePercent=50
+                -XX:+DisableExplicitGC
+                -XX:-ResizePLAB
+                -XX:+ExitOnOutOfMemoryError
+    storage:
+      inCluster:
+        deletionPolicy: Retain
+        pvcDeletion: false
+        values:
+          mode: distributed

tests/python_client/deploy/monitor_rolling_update.py

@@ -0,0 +1,27 @@
+import argparse
+import subprocess
+import time
+from loguru import logger as log
+
+
+def run_kubectl_get_pod(duration, interval, release_name):
+    end_time = time.time() + duration
+    while time.time() < end_time:
+        cmd = f"kubectl get pod |grep {release_name}"
+        res = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = res.communicate()
+        output = stdout.decode("utf-8")
+        log.info(f"{cmd}\n{output}\n")
+        time.sleep(interval)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Script to run "kubectl get pod" command at regular intervals')
+    parser.add_argument('-d', '--duration', type=int, default=600, help='Duration in seconds (default: 600)')
+    parser.add_argument('-i', '--interval', type=int, default=5, help='Interval in seconds (default: 30)')
+    parser.add_argument('-n', '--release_name', type=str, default="", help='release name (default: "None")')
+    args = parser.parse_args()
+
+    run_kubectl_get_pod(args.duration, args.interval, args.release_name)
+
+

tests/python_client/deploy/requirements.txt

@@ -0,0 +1,10 @@
+--extra-index-url https://test.pypi.org/simple/
+docker==5.0.0
+grpcio==1.53.2
+grpcio-tools==1.37.1
+pymilvus==2.0.0rc8
+
+# for test result anaylszer
+prettytable==3.8.0
+pyarrow==14.0.1
+fastparquet==2023.7.0

tests/python_client/deploy/run.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -x
+
+
+func() {
+    echo "Usage:"
+    echo "run.sh [-p Password]"
+    echo "Password, the password of root"
+    exit -1
+
+}
+while getopts "hp:" OPT;
+do
+    case $OPT in
+	p) Password="$OPTARG";;
+	h) func;;
+	?) func;;
+    esac
+done
+pw=$Password
+
+# start test standalone reinstall
+bash test.sh -m standalone -t reinstall -p $pw
+# start test standalone upgrade
+bash test.sh -m standalone -t upgrade -p $pw
+# start test cluster reinstall
+bash test.sh -m cluster -t reinstall -p $pw
+# start test cluster upgrade
+bash test.sh -m cluster -t upgrade -p $pw

tests/python_client/deploy/scripts/action_after_reinstall.py

@@ -0,0 +1,47 @@
+from pymilvus import connections
+from utils import *
+
+
+def task_1(data_size, host):
+    """
+    task_1:
+        before reinstall: create collection, insert data, create index and insert data, load and search
+        after reinstall: get collection, load, search, release, insert data, create index, load, and search
+    """
+    prefix = "task_1_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    load_and_search(prefix)
+    release_collection(prefix)
+    create_collections_and_insert_data(prefix,count=data_size)
+    load_and_search(prefix)
+
+
+def task_2(data_zise, host):
+    """
+    task_2:
+        before reinstall: create collection, insert data and create index, load and search
+        after reinstall: get collection, load, search, insert data, release, create index, load, and search
+    """
+    prefix = "task_2_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    release_collection(prefix)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+if __name__ == '__main__':
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for deploy test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    parser.add_argument('--data_size', type=int, default=3000, help='data size')
+    args = parser.parse_args()
+    host = args.host 
+    data_size = args.data_size
+    logger.info(f"data size: {data_size}")
+    task_1(data_size, host)
+    task_2(data_size, host)

tests/python_client/deploy/scripts/action_after_upgrade.py

@@ -0,0 +1,114 @@
+from pymilvus import connections
+import sys
+sys.path.append("..")
+sys.path.append("../..")
+from common.milvus_sys import MilvusSys
+from utils import *
+
+
+def task_1(data_size, host):
+    """
+    task_1:
+        before upgrade: create collection and insert data with flush, create index, load and search
+        after upgrade: get collection, load, search, insert data with flush, release, create index, load, and search
+    """
+    prefix = "task_1_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    create_index(prefix)
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    release_collection(prefix)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+def task_2(data_size, host):
+    """
+    task_2:
+        before upgrade: create collection, insert data and create index, load and search
+        after upgrade: get collection, load, search, insert data, release, create index, load, and search
+    """
+    prefix = "task_2_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    release_collection(prefix)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+def task_3(data_size, host):
+    """
+    task_3:
+        before upgrade: create collection, insert data, flush, create index, load with one replicas and search
+        after upgrade: get collection, load, search, insert data, release, create index, load with multi replicas, and search
+    """
+    prefix = "task_3_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    release_collection(prefix)
+    create_index(prefix)
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+
+
+def task_4(data_size, host):
+    """
+    task_4:
+        before upgrade: create collection, insert data, flush, and create index
+        after upgrade: get collection, load with multi replicas, search, insert data, load with multi replicas and search
+    """
+    prefix = "task_4_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+    create_collections_and_insert_data(prefix, flush=False, count=data_size)
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+
+
+def task_5(data_size, host):
+    """
+    task_5_:
+        before upgrade: create collection and insert data without flush
+        after upgrade: get collection, create index, load with multi replicas, search, insert data with flush, load with multi replicas and search
+    """
+    prefix = "task_5_"
+    connections.connect(host=host, port=19530, timeout=60)
+    col_list = get_collections(prefix, check=True)
+    assert len(col_list) > 0
+    create_index(prefix)
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+    create_collections_and_insert_data(prefix, flush=True, count=data_size)
+    load_and_search(prefix, replicas=NUM_REPLICAS)
+
+
+if __name__ == '__main__':
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for deploy test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    parser.add_argument('--data_size', type=int, default=3000, help='data size')
+    args = parser.parse_args()
+    data_size = args.data_size
+    host = args.host
+    logger.info(f"data size: {data_size}")
+    connections.connect(host=host, port=19530, timeout=60)
+    ms = MilvusSys()
+    # create index for flat
+    logger.info("create index for flat start")
+    create_index_flat()
+    logger.info("create index for flat done")
+    task_1(data_size, host)
+    task_2(data_size, host)
+    if len(ms.query_nodes) >= NUM_REPLICAS:
+        task_3(data_size, host)
+        task_4(data_size, host)
+        task_5(data_size, host)
+

tests/python_client/deploy/scripts/action_before_reinstall.py

@@ -0,0 +1,48 @@
+from pymilvus import connections
+from utils import *
+
+
+def task_1(data_size, host):
+    """
+    task_1:
+        before reinstall: create collection, insert data, create index and insert data, load and search
+        after reinstall: get collection, load, search, release, insert data, create index, load, and search
+    """
+    prefix = "task_1_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix,count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix,count=data_size)
+    load_and_search(prefix)
+
+
+def task_2(data_size, host):
+    """
+    task_2:
+        before reinstall: create collection, insert data, create index, insert data, create index,load and search
+        after reinstall: get collection, load, search, insert data, create index, load, and search
+    """
+    prefix = "task_2_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+if __name__ == '__main__':
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for deploy test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    parser.add_argument('--data_size', type=int, default=3000, help='data size')
+    args = parser.parse_args()
+    data_size = args.data_size
+    host = args.host
+    logger.info(f"data_size: {data_size}")
+    task_1(data_size, host)
+    task_2(data_size, host)

tests/python_client/deploy/scripts/action_before_upgrade.py

@@ -0,0 +1,91 @@
+from pymilvus import connections
+import sys
+sys.path.append("..")
+sys.path.append("../..")
+from common.milvus_sys import MilvusSys
+from utils import *
+
+
+def task_1(data_size, host):
+    """
+    task_1:
+        before upgrade: create collection and insert data with flush, create index, load and search
+        after upgrade: get collection, load, search, insert data with flush, release, create index, load, and search
+    """
+    prefix = "task_1_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+
+
+def task_2(data_size, host):
+    """
+    task_2:
+        before upgrade: create collection, insert data and create index, load , search, and insert data without flush
+        after upgrade: get collection, load, search, insert data, release, create index, load, and search
+    """
+    prefix = "task_2_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+    create_collections_and_insert_data(prefix, flush=False, count=data_size)
+
+def task_3(data_size, host):
+    """
+    task_3:
+        before upgrade: create collection, insert data, flush, create index, load with one replicas and search
+        after upgrade: get collection, load, search, insert data, create index, release, load with multi replicas, and search
+    """
+    prefix = "task_3_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, count=data_size)
+    create_index(prefix)
+    load_and_search(prefix)
+
+def task_4(data_size, host):
+    """
+    task_4_:
+        before upgrade: create collection, insert data, flush, and create index
+        after upgrade: get collection, load with multi replicas, search, insert data, load with multi replicas and search
+    """
+    prefix = "task_4_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, flush=True, count=data_size)
+    create_index(prefix)
+
+def task_5(data_size, host):
+    """
+    task_5_:
+        before upgrade: create collection and insert data without flush
+        after upgrade: get collection, create index, load with multi replicas, search, insert data with flush, load with multi replicas and search
+    """
+    prefix = "task_5_"
+    connections.connect(host=host, port=19530, timeout=60)
+    get_collections(prefix)
+    create_collections_and_insert_data(prefix, flush=False, count=data_size)
+
+
+if __name__ == '__main__':
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for deploy test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    parser.add_argument('--data_size', type=int, default=3000, help='data size')
+    args = parser.parse_args()
+    data_size = args.data_size
+    host = args.host
+    logger.info(f"data size: {data_size}")
+    connections.connect(host=host, port=19530, timeout=60)
+    ms = MilvusSys()
+    task_1(data_size, host)
+    task_2(data_size, host)
+    if len(ms.query_nodes) >= NUM_REPLICAS:
+        task_3(data_size, host)
+        task_4(data_size, host)
+        task_5(data_size, host)

tests/python_client/deploy/scripts/breakdown_rolling_update.py

@@ -0,0 +1,53 @@
+import psutil
+import time
+from loguru import logger
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description='config for rolling update process')
+    parser.add_argument('--wait_time', type=int, default=60, help='wait time after rolling update started')
+    args = parser.parse_args()
+    wait_time = args.wait_time
+    logger.info("start to watch rolling update process")
+    start_time = time.time()
+    end_time = time.time()
+    flag = False
+    while not flag and end_time - start_time < 360: 
+        process_list = [p.info for p in psutil.process_iter(attrs=['pid', 'name','cmdline'])]
+        for process in process_list:
+            logger.debug(process)
+            logger.debug("##"*30)
+        for process in process_list:
+            if isinstance(process.get("cmdline", []), list):
+                cmdline_list = process.get("cmdline", [])
+                for cmdline in cmdline_list:
+                    if "rollingUpdate.sh" in cmdline:
+                        logger.info(f"rolling update process: {process} started")
+                        flag = True
+                        break
+            if flag:
+                break
+        time.sleep(0.5)
+        end_time = time.time()
+        if not flag:
+            logger.info(f"rolling update process not found, wait for {end_time - start_time} seconds")
+        else:
+            logger.info(f"rolling update process {process} found, wait for {end_time - start_time} seconds")
+    if flag:
+        logger.info(f"wait {wait_time}s to kill rolling update process")
+        time.sleep(wait_time)
+        logger.info("start to kill rolling update process")
+        try:
+            p = psutil.Process(process["pid"])
+            p.terminate()
+            logger.info(f"rolling update process: {process} killed")
+        except Exception as e:
+            logger.error(f"rolling update process: {process} kill failed, {e}")
+    else:
+        logger.info("all process info")
+        for process in process_list:
+            logger.info(process)
+            
+            
+

tests/python_client/deploy/scripts/first_recall_test.py

@@ -0,0 +1,201 @@
+import threading
+import h5py
+import numpy as np
+import time
+import sys
+import copy
+from pathlib import Path
+from loguru import logger
+import pymilvus
+from pymilvus import (
+    connections,
+    FieldSchema, CollectionSchema, DataType,
+    Collection, utility
+)
+
+pymilvus_version = pymilvus.__version__
+
+
+all_index_types = ["IVF_FLAT", "IVF_SQ8", "HNSW"]
+default_index_params = [{"nlist": 128}, {"nlist": 128}, {"M": 48, "efConstruction": 200}]
+index_params_map = dict(zip(all_index_types, default_index_params))
+
+
+def gen_index_params(index_type, metric_type="L2"):
+    default_index = {"index_type": "IVF_FLAT", "params": {"nlist": 128}, "metric_type": metric_type}
+    index = copy.deepcopy(default_index)
+    index["index_type"] = index_type
+    index["params"] = index_params_map[index_type]
+    if index_type in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+        index["metric_type"] = "HAMMING"
+    return index
+
+
+def gen_search_param(index_type, metric_type="L2"):
+    search_params = []
+    if index_type in ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"]:
+        for nprobe in [10]:
+            ivf_search_params = {"metric_type": metric_type, "params": {"nprobe": nprobe}}
+            search_params.append(ivf_search_params)
+    elif index_type in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+        for nprobe in [10]:
+            bin_search_params = {"metric_type": "HAMMING", "params": {"nprobe": nprobe}}
+            search_params.append(bin_search_params)
+    elif index_type in ["HNSW"]:
+        for ef in [150]:
+            hnsw_search_param = {"metric_type": metric_type, "params": {"ef": ef}}
+            search_params.append(hnsw_search_param)
+    elif index_type == "ANNOY":
+        for search_k in [1000]:
+            annoy_search_param = {"metric_type": metric_type, "params": {"search_k": search_k}}
+            search_params.append(annoy_search_param)
+    else:
+        logger.info("Invalid index_type.")
+        raise Exception("Invalid index_type.")
+    return search_params[0]
+
+
+def read_benchmark_hdf5(file_path):
+
+    f = h5py.File(file_path, 'r')
+    train = np.array(f["train"])
+    test = np.array(f["test"])
+    neighbors = np.array(f["neighbors"])
+    f.close()
+    return train, test, neighbors
+
+
+dim = 128
+TIMEOUT = 200
+
+
+def milvus_recall_test(host='127.0.0.1', index_type="HNSW"):
+    logger.info(f"recall test for index type {index_type}")
+    file_path = f"{str(Path(__file__).absolute().parent.parent.parent)}/assets/ann_hdf5/sift-128-euclidean.hdf5"
+    train, test, neighbors = read_benchmark_hdf5(file_path)
+    connections.connect(host=host, port="19530")
+    default_fields = [
+        FieldSchema(name="int64", dtype=DataType.INT64, is_primary=True),
+        FieldSchema(name="float", dtype=DataType.FLOAT),
+        FieldSchema(name="varchar", dtype=DataType.VARCHAR, max_length=65535),
+        FieldSchema(name="float_vector", dtype=DataType.FLOAT_VECTOR, dim=dim)
+    ]
+    default_schema = CollectionSchema(
+        fields=default_fields, description="test collection")
+
+    name = f"sift_128_euclidean_{index_type}"
+    logger.info(f"Create collection {name}")
+    collection = Collection(name=name, schema=default_schema)
+    nb = len(train)
+    batch_size = 50000
+    epoch = int(nb / batch_size)
+    t0 = time.time()
+    for i in range(epoch):
+        logger.info(f"epoch: {i}")
+        start = i * batch_size
+        end = (i + 1) * batch_size
+        if end > nb:
+            end = nb
+        data = [
+            [i for i in range(start, end)],
+            [np.float32(i) for i in range(start, end)],
+            [str(i) for i in range(start, end)],
+            train[start:end]
+        ]
+        collection.insert(data)
+    t1 = time.time()
+    logger.info(f"Insert {nb} vectors cost {t1 - t0:.4f} seconds")
+
+    t0 = time.time()
+    logger.info(f"Get collection entities...")
+    if pymilvus_version >= "2.2.0":
+        collection.flush()
+    else:
+        collection.num_entities
+    logger.info(collection.num_entities)
+    t1 = time.time()
+    logger.info(f"Get collection entities cost {t1 - t0:.4f} seconds")
+
+    # create index
+    default_index = gen_index_params(index_type)
+    logger.info(f"Create index...")
+    t0 = time.time()
+    collection.create_index(field_name="float_vector",
+                            index_params=default_index)
+    t1 = time.time()
+    logger.info(f"Create index cost {t1 - t0:.4f} seconds")
+
+    # load collection
+    replica_number = 1
+    logger.info(f"load collection...")
+    t0 = time.time()
+    collection.load(replica_number=replica_number)
+    t1 = time.time()
+    logger.info(f"load collection cost {t1 - t0:.4f} seconds")
+    res = utility.get_query_segment_info(name)
+    cnt = 0
+    logger.info(f"segments info: {res}")
+    for segment in res:
+        cnt += segment.num_rows
+    assert cnt == collection.num_entities
+    logger.info(f"wait for loading complete...")
+    time.sleep(30)
+    res = utility.get_query_segment_info(name)
+    logger.info(f"segments info: {res}")
+
+    # search
+    topK = 100
+    nq = 10000
+    current_search_params = gen_search_param(index_type)
+
+    # define output_fields of search result
+    for i in range(3):
+        t0 = time.time()
+        logger.info(f"Search...")
+        res = collection.search(
+            test[:nq], "float_vector", current_search_params, topK, output_fields=["int64"], timeout=TIMEOUT
+        )
+        t1 = time.time()
+        logger.info(f"search cost  {t1 - t0:.4f} seconds")
+        result_ids = []
+        for hits in res:
+            result_id = []
+            for hit in hits:
+                result_id.append(hit.entity.get("int64"))
+            result_ids.append(result_id)
+
+        # calculate recall
+        true_ids = neighbors[:nq, :topK]
+        sum_radio = 0.0
+        logger.info(f"Calculate recall...")
+        for index, item in enumerate(result_ids):
+            # tmp = set(item).intersection(set(flat_id_list[index]))
+            assert len(item) == len(true_ids[index])
+            tmp = set(true_ids[index]).intersection(set(item))
+            sum_radio = sum_radio + len(tmp) / len(item)
+        recall = round(sum_radio / len(result_ids), 6)
+        logger.info(f"recall={recall}")
+        if index_type in ["IVF_PQ", "ANNOY"]:
+            assert recall >= 0.6, f"recall={recall} < 0.6"
+        else:
+            assert 0.95 <= recall < 1.0, f"recall is {recall}, less than 0.95, greater than or equal to 1.0"
+    # query
+    expr = "int64 in [2,4,6,8]"
+    output_fields = ["int64", "float"]
+    res = collection.query(expr, output_fields, timeout=TIMEOUT)
+    sorted_res = sorted(res, key=lambda k: k['int64'])
+    for r in sorted_res:
+        logger.info(r)
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description='config for recall test')
+    parser.add_argument('--host', type=str,
+                        default="127.0.0.1", help='milvus server ip')
+    args = parser.parse_args()
+    host = args.host
+    tasks = []
+    for index_type in ["HNSW"]:
+        milvus_recall_test(host, index_type)
+

tests/python_client/deploy/scripts/get_tag.py

@@ -0,0 +1,33 @@
+import requests
+import json
+
+milvus_dev = "https://registry.hub.docker.com/v2/repositories/milvusdb/milvus/tags?ordering=last_updated"
+milvus = "https://registry.hub.docker.com/v2/repositories/milvusdb/milvus/tags?ordering=last_updated"
+
+
+def get_tag(url):
+    payload = {}
+    headers = {}
+
+    response = requests.request("GET", url, headers=headers, data=payload)
+
+    res = response.json()["results"]
+    sorted_r = sorted(res, key=lambda k: k['last_updated'])
+    tags = [r["name"] for r in sorted_r]
+    return tags
+
+
+latest_tag = [tag for tag in get_tag(milvus_dev) if "latest" not in tag][-1]
+latest_rc_tag = [tag for tag in get_tag(milvus) if "v" in tag][-1]
+# release_version = "-".join(latest_rc_tag.split("-")[:-2])
+# print(release_version)
+print(latest_tag, latest_rc_tag)
+
+data = {
+    "latest_tag": latest_tag,
+    "latest_rc_tag": latest_rc_tag,
+    # "release_version": release_version
+}
+print(data)
+with open("tag_info.json", "w") as f:
+    f.write(json.dumps(data))

tests/python_client/deploy/scripts/second_recall_test.py

@@ -0,0 +1,103 @@
+import h5py
+import numpy as np
+import time
+import sys
+import threading
+from pathlib import Path
+from loguru import logger
+from pymilvus import connections, Collection
+
+
+all_index_types = ["IVF_FLAT", "IVF_SQ8", "HNSW"]
+
+
+def read_benchmark_hdf5(file_path):
+
+    f = h5py.File(file_path, 'r')
+    train = np.array(f["train"])
+    test = np.array(f["test"])
+    neighbors = np.array(f["neighbors"])
+    f.close()
+    return train, test, neighbors
+
+
+def gen_search_param(index_type, metric_type="L2"):
+    search_params = []
+    if index_type in ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"]:
+        for nprobe in [10]:
+            ivf_search_params = {"metric_type": metric_type, "params": {"nprobe": nprobe}}
+            search_params.append(ivf_search_params)
+    elif index_type in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+        for nprobe in [10]:
+            bin_search_params = {"metric_type": "HAMMING", "params": {"nprobe": nprobe}}
+            search_params.append(bin_search_params)
+    elif index_type in ["HNSW"]:
+        for ef in [150]:
+            hnsw_search_param = {"metric_type": metric_type, "params": {"ef": ef}}
+            search_params.append(hnsw_search_param)
+    elif index_type == "ANNOY":
+        for search_k in [1000]:
+            annoy_search_param = {"metric_type": metric_type, "params": {"search_k": search_k}}
+            search_params.append(annoy_search_param)
+    else:
+        logger.info("Invalid index_type.")
+        raise Exception("Invalid index_type.")
+    return search_params[0]
+
+
+dim = 128
+TIMEOUT = 200
+
+
+def search_test(host="127.0.0.1", index_type="HNSW"):
+    logger.info(f"recall test for index type {index_type}")
+    file_path = f"{str(Path(__file__).absolute().parent.parent.parent)}/assets/ann_hdf5/sift-128-euclidean.hdf5"
+    train, test, neighbors = read_benchmark_hdf5(file_path)
+    connections.connect(host=host, port="19530")
+    collection = Collection(name=f"sift_128_euclidean_{index_type}")
+    nq = 10000
+    topK = 100
+    search_params = gen_search_param(index_type)
+    for i in range(3):
+        t0 = time.time()
+        logger.info(f"\nSearch...")
+        # define output_fields of search result
+        res = collection.search(
+            test[:nq], "float_vector", search_params, topK, output_fields=["int64"], timeout=TIMEOUT
+        )
+        t1 = time.time()
+        logger.info(f"search cost  {t1 - t0:.4f} seconds")
+        result_ids = []
+        for hits in res:
+            result_id = []
+            for hit in hits:
+                result_id.append(hit.entity.get("int64"))
+            result_ids.append(result_id)
+
+        # calculate recall
+        true_ids = neighbors[:nq, :topK]
+        sum_radio = 0.0
+        for index, item in enumerate(result_ids):
+            # tmp = set(item).intersection(set(flat_id_list[index]))
+            assert len(item) == len(true_ids[index]), f"get {len(item)} but expect {len(true_ids[index])}"
+            tmp = set(true_ids[index]).intersection(set(item))
+            sum_radio = sum_radio + len(tmp) / len(item)
+        recall = round(sum_radio / len(result_ids), 6)
+        logger.info(f"recall={recall}")
+        if index_type in ["IVF_PQ", "ANNOY"]:
+            assert recall >= 0.6, f"recall={recall} < 0.6"
+        else:
+            assert 0.95 <= recall < 1.0, f"recall is {recall}, less than 0.95, greater than or equal to 1.0"
+
+
+
+if __name__ == "__main__":
+    import argparse
+    import threading
+    parser = argparse.ArgumentParser(description='config for recall test')
+    parser.add_argument('--host', type=str, default="127.0.0.1", help='milvus server ip')
+    args = parser.parse_args()
+    host = args.host
+    tasks = []
+    for index_type in ["HNSW"]:
+        search_test(host, index_type)

tests/python_client/deploy/scripts/utils.py

@@ -0,0 +1,268 @@
+import sys
+import copy
+import time
+from loguru import logger
+import pymilvus
+from pymilvus import (
+    FieldSchema, CollectionSchema, DataType,
+    Collection, list_collections,
+)
+logger.remove()
+logger.add(sys.stderr, format= "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
+    "<level>{level: <8}</level> | "
+    "<cyan>{thread.name}</cyan> |"
+    "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+    level="INFO")
+
+pymilvus_version = pymilvus.__version__
+
+all_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", "HNSW"]
+
+default_index_params = [{}, {"nlist": 128}, {"nlist": 128}, {"nlist": 128, "m": 16, "nbits": 8},
+                        {"M": 48, "efConstruction": 500}]
+
+index_params_map = dict(zip(all_index_types, default_index_params))
+
+NUM_REPLICAS = 2
+
+
+def filter_collections_by_prefix(prefix):
+    col_list = list_collections()
+    logger.info(f"all collections: {col_list}")
+    res = []
+    for col in col_list:
+        if col.startswith(prefix):
+            if any(index_name in col for index_name in all_index_types):
+                res.append(col)
+            else:
+                logger.warning(f"collection {col} has no supported index, skip")
+    logger.info(f"filtered collections with prefix {prefix}: {res}")
+    return res
+
+
+def gen_search_param(index_type, metric_type="L2"):
+    search_params = []
+    if index_type in ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"]:
+        for nprobe in [10]:
+            ivf_search_params = {"metric_type": metric_type, "params": {"nprobe": nprobe}}
+            search_params.append(ivf_search_params)
+    elif index_type in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+        for nprobe in [10]:
+            bin_search_params = {"metric_type": "HAMMING", "params": {"nprobe": nprobe}}
+            search_params.append(bin_search_params)
+    elif index_type in ["HNSW"]:
+        for ef in [64]:
+            hnsw_search_param = {"metric_type": metric_type, "params": {"ef": ef}}
+            search_params.append(hnsw_search_param)
+    elif index_type == "ANNOY":
+        for search_k in [1000]:
+            annoy_search_param = {"metric_type": metric_type, "params": {"search_k": search_k}}
+            search_params.append(annoy_search_param)
+    else:
+        logger.info("Invalid index_type.")
+        raise Exception("Invalid index_type.")
+    return search_params
+
+
+def get_collections(prefix, check=False):
+    logger.info("\nList collections...")
+    col_list = filter_collections_by_prefix(prefix)
+    logger.info(f"collections_nums: {len(col_list)}")
+    # list entities if collections
+    for name in col_list:
+        c = Collection(name=name)
+        if pymilvus_version >= "2.2.0":
+            c.flush()
+        else:
+            c.num_entities
+        num_entities = c.num_entities
+        logger.info(f"{name}: {num_entities}")
+        if check:
+            assert num_entities >= 3000
+    return col_list
+
+
+def create_collections_and_insert_data(prefix, flush=True, count=3000, collection_cnt=11):
+    import random
+    dim = 128
+    nb = count // 10
+    default_fields = [
+        FieldSchema(name="count", dtype=DataType.INT64, is_primary=True),
+        FieldSchema(name="random_value", dtype=DataType.DOUBLE),
+        FieldSchema(name="float_vector", dtype=DataType.FLOAT_VECTOR, dim=dim)
+    ]
+    default_schema = CollectionSchema(fields=default_fields, description="test collection")
+    for index_name in all_index_types[:collection_cnt]:
+        logger.info("\nCreate collection...")
+        col_name = prefix + index_name
+        collection = Collection(name=col_name, schema=default_schema)
+        logger.info(f"collection name: {col_name}")
+        logger.info(f"begin insert, count: {count} nb: {nb}")
+        times = int(count // nb)
+        total_time = 0.0
+        vectors = [[random.random() for _ in range(dim)] for _ in range(count)]
+        for j in range(times):
+            start_time = time.time()
+            collection.insert(
+                [
+                    [i for i in range(nb * j, nb * j + nb)],
+                    [float(random.randrange(-20, -10)) for _ in range(nb)],
+                    vectors[nb*j:nb*j+nb]
+                ]
+            )
+            end_time = time.time()
+            logger.info(f"[{j+1}/{times}] insert {nb} data, time: {end_time - start_time:.4f}")
+            total_time += end_time - start_time
+            if j <= times - 3:
+                collection.flush()
+                collection.num_entities
+            if j == times - 3:
+                collection.compact()
+
+
+        logger.info(f"end insert, time: {total_time:.4f}")
+        if flush:
+            logger.info("Get collection entities")
+            start_time = time.time()
+            if pymilvus_version >= "2.2.0":
+                collection.flush()
+            else:
+                collection.num_entities
+            logger.info(f"collection entities: {collection.num_entities}")
+            end_time = time.time()
+            logger.info("Get collection entities time = %.4fs" % (end_time - start_time))
+    logger.info("\nList collections...")
+    logger.info(get_collections(prefix))
+
+
+def create_index_flat():
+    # create index
+    default_flat_index = {"index_type": "FLAT", "params": {}, "metric_type": "L2"}
+    all_col_list = list_collections()
+    col_list = []
+    for col_name in all_col_list:
+        if "FLAT" in col_name and "task" in col_name and "IVF" not in col_name:
+            col_list.append(col_name)
+    logger.info("\nCreate index for FLAT...")
+    for col_name in col_list:
+        c = Collection(name=col_name)
+        logger.info(c)
+        try:
+            replicas = c.get_replicas()
+            replica_number = len(replicas.groups)
+            c.release()
+        except Exception as e:
+            replica_number = 0
+            logger.info(e)
+        t0 = time.time()
+        c.create_index(field_name="float_vector", index_params=default_flat_index)
+        logger.info(f"create index time: {time.time() - t0:.4f}")
+        if replica_number > 0:
+            c.load(replica_number=replica_number)
+
+
+def create_index(prefix):
+    # create index
+    default_index = {"index_type": "IVF_FLAT", "params": {"nlist": 128}, "metric_type": "L2"}
+    col_list = get_collections(prefix)
+    logger.info("\nCreate index...")
+    for col_name in col_list:
+        c = Collection(name=col_name)
+        try:
+            replicas = c.get_replicas()
+            replica_number = len(replicas.groups)
+            c.release()
+        except Exception as e:
+            replica_number = 0
+            logger.info(e)
+        index_name = col_name.replace(prefix, "")
+        logger.info(index_name)
+        logger.info(c)
+        index = copy.deepcopy(default_index)
+        index["index_type"] = index_name
+        index["params"] = index_params_map[index_name]
+        if index_name in ["BIN_FLAT", "BIN_IVF_FLAT"]:
+            index["metric_type"] = "HAMMING"
+        index_info_list = [x.to_dict() for x in c.indexes]
+        logger.info(index_info_list)
+        is_indexed = False
+        for index_info in index_info_list:
+            if "metric_type" in index_info.keys() or "metric_type" in index_info["index_param"]:
+                is_indexed = True
+                logger.info(f"collection {col_name} has been indexed with {index_info}")
+        if not is_indexed:
+            t0 = time.time()
+            c.create_index(field_name="float_vector", index_params=index)
+            logger.info(f"create index time: {time.time() - t0:.4f}")
+        if replica_number > 0:
+            c.load(replica_number=replica_number)
+
+
+def release_collection(prefix):
+    col_list = get_collections(prefix)
+    logger.info("release collection")
+    for col_name in col_list:
+        c = Collection(name=col_name)
+        c.release()
+
+
+def load_and_search(prefix, replicas=1):
+    logger.info("search data starts")
+    col_list = get_collections(prefix)
+    for col_name in col_list:
+        c = Collection(name=col_name)
+        logger.info(f"collection name: {col_name}")
+        logger.info("load collection")
+        if replicas == 1:
+            t0 = time.time()
+            c.load()
+            logger.info(f"load time: {time.time() - t0:.4f}")
+        if replicas > 1:
+            logger.info("release collection before load if replicas > 1")
+            t0 = time.time()
+            c.release()
+            logger.info(f"release time: {time.time() - t0:.4f}")
+            t0 = time.time()
+            c.load(replica_number=replicas)
+            logger.info(f"load time: {time.time() - t0:.4f}")
+            logger.info(c.get_replicas())
+        topK = 5
+        vectors = [[1.0 for _ in range(128)] for _ in range(3000)]
+        index_name = col_name.replace(prefix, "")
+        search_params = gen_search_param(index_name)[0]
+        logger.info(search_params)
+        # search_params = {"metric_type": "L2", "params": {"nprobe": 10}}
+        start_time = time.time()
+        logger.info(f"\nSearch...")
+        # define output_fields of search result
+        v_search = vectors[:1]
+        res = c.search(
+            v_search, "float_vector", search_params, topK,
+            "count > 500", output_fields=["count", "random_value"], timeout=120
+        )
+        end_time = time.time()
+        # show result
+        for hits in res:
+            for hit in hits:
+                logger.info(f"hit: {hit}")
+            ids = hits.ids
+            assert len(ids) == topK, f"get {len(ids)} results, but topK is {topK}"
+            logger.info(ids)
+        assert len(res) == len(v_search), f"get {len(res)} results, but search num is {len(v_search)}"
+        logger.info("search latency: %.4fs" % (end_time - start_time))
+        t0 = time.time()
+        expr = "count in [2,4,6,8]"
+        if "SQ" in col_name or "PQ" in col_name:
+            output_fields = ["count", "random_value"]
+        else:
+            output_fields = ["count", "random_value", "float_vector"]
+        res = c.query(expr, output_fields, timeout=120)
+        sorted_res = sorted(res, key=lambda k: k['count'])
+        for r in sorted_res:
+            logger.info(r)
+        t1 = time.time()
+        assert len(res) == 4
+        logger.info("query latency: %.4fs" % (t1 - t0))
+        # c.release()
+        logger.info("###########")
+    logger.info("search data ends")

tests/python_client/deploy/standalone-values.yaml

@@ -0,0 +1,43 @@
+cluster:
+  enabled: false
+log:
+  level: debug  
+image:
+  all:
+    repository: milvusdb/milvus
+    tag: master-latest
+    pullPolicy: IfNotPresent
+
+standalone:
+  resources:
+    limits:
+      cpu: 8
+      memory: 16Gi
+    requests:
+      cpu: 4
+      memory: 8Gi
+
+kafka:
+  enabled: false
+  name: kafka
+  replicaCount: 3
+  defaultReplicationFactor: 2
+
+etcd:
+  replicaCount: 3
+  image:
+    repository: milvusdb/etcd
+    tag: 3.5.5-r2  
+minio:
+  mode: standalone
+pulsar:
+  enabled: false
+
+extraConfigFiles:
+  user.yaml: |+
+    dataCoord:
+      compaction:
+        indexBasedCompaction: false
+    indexCoord:
+      scheduler:
+        interval: 100        

tests/python_client/deploy/test.sh

@@ -0,0 +1,242 @@
+#!/bin/bash
+set -e
+# set -x
+
+func() {
+    echo "Usage:"
+    echo "test.sh [-t Task] [-m Mode] [-r Release] [-p Password]"
+    echo "Description"
+    echo "Task, the task type of test. reinstall or upgrade"
+    echo "Mode, the mode of milvus deploy. standalone or cluster"
+    echo "Release, the release of milvus. e.g. 2.0.0-rc8"
+    echo "Password, the password of root"
+    exit -1
+}
+
+
+echo "check os env"
+platform='unknown'
+unamestr=$(uname)
+if [[ "$unamestr" == 'Linux' ]]; then
+   platform='Linux'
+elif [[ "$unamestr" == 'Darwin' ]]; then
+   platform='Mac'
+fi
+echo "platform: $platform"
+
+
+
+Task="reinstall"
+Mode="standalone"
+Release="v2.0.0"
+while getopts "hm:t:p:" OPT;
+do
+    case $OPT in
+	m) Mode="$OPTARG";;
+    t) Task="$OPTARG";;
+	p) Password="$OPTARG";;
+	h) func;;
+	?) func;;
+    esac
+done
+
+ROOT_FOLDER=$(cd "$(dirname "$0")";pwd)
+
+# to export docker compose logs before exit
+function error_exit {
+    pushd ${ROOT_FOLDER}/${Deploy_Dir}
+    echo "test failed"
+    current=$(date "+%Y-%m-%d-%H-%M-%S")
+    if [ ! -d logs  ];
+    then
+        mkdir logs
+    fi
+    docker compose ps
+    docker compose logs > ./logs/${Deploy_Dir}-${Task}-${current}.log 2>&1
+    echo "log saved to $(pwd)/logs/${Deploy_Dir}-${Task}-${current}.log"
+    popd
+    exit 1
+}
+
+function replace_image_tag {
+    image_repo=$1
+    image_tag=$2
+    if [ "$platform" == "Mac" ];
+    then
+        # for mac os
+        sed -i "" "s/milvusdb\/milvus.*/${image_repo}\:${image_tag}/g" docker-compose.yml   
+    else
+        #for linux os 
+        sed -i "s/milvusdb\/milvus.*/${image_repo}\:${image_tag}/g" docker-compose.yml
+    fi
+
+}
+       
+
+#to check containers all running and minio is healthy
+function check_healthy {
+    cnt=$(docker compose ps | grep -E "running|Running|Up|up" | wc -l)
+    healthy=$(docker compose ps | grep "healthy" | wc -l)
+    time_cnt=0
+    echo "running num $cnt expect num $Expect"
+    echo "healthy num $healthy expect num $Expect_health"
+    while [[ $cnt -ne $Expect || $healthy -ne 1 ]];
+    do
+    printf "waiting all containers get running\n"
+    sleep 5
+    let time_cnt+=5
+    # if time is greater than 300s, the condition still not satisfied, we regard it as a failure
+    if [ $time_cnt -gt 300 ];
+    then
+        printf "timeout,there are some issue with deployment!"
+        error_exit
+    fi
+    cnt=$(docker compose ps | grep -E "running|Running|Up|up" | wc -l)
+    healthy=$(docker compose ps | grep "healthy" | wc -l)
+    echo "running num $cnt expect num $Expect"
+    echo "healthy num $healthy expect num $Expect_health"
+    done
+}
+
+Deploy_Dir=$Mode
+Task=$Task
+Release=$Release
+pw=$Password
+
+echo "mode: $Mode"
+echo "task: $Task"
+echo "password: $pw"
+## if needed, install dependency 
+#echo "install dependency"
+#pip install -r scripts/requirements.txt
+
+if [ ! -d ${Deploy_Dir}  ];
+then
+    mkdir ${Deploy_Dir}
+fi
+
+echo "get tag info"
+
+python scripts/get_tag.py
+
+latest_tag=$(jq -r ".latest_tag" tag_info.json)
+latest_rc_tag=$(jq -r ".latest_rc_tag" tag_info.json)
+# release_version="v${Release}"
+echo $latest_rc_tag
+
+pushd ${Deploy_Dir}
+# download docker-compose.yml
+wget https://github.com/milvus-io/milvus/releases/download/${latest_rc_tag}/milvus-${Mode}-docker-compose.yml -O docker-compose.yml
+ls
+# clean env to deploy a fresh milvus
+docker compose down
+docker compose ps
+echo "$pw"| sudo -S rm -rf ./volumes
+
+# first deployment
+if [ "$Task" == "reinstall" ];
+then
+    printf "download latest milvus docker compose yaml file from github\n"
+    wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/${Mode}/docker-compose.yml -O docker-compose.yml
+    printf "start to deploy latest rc tag milvus\n"
+    replace_image_tag "milvusdb\/milvus" $latest_tag
+fi
+if [ "$Task" == "upgrade" ];
+then
+    printf "start to deploy previous rc tag milvus\n"
+    replace_image_tag "milvusdb\/milvus" $latest_rc_tag # replace previous rc tag
+
+fi
+cat docker-compose.yml|grep milvusdb
+Expect=$(grep "container_name" docker-compose.yml | wc -l)
+Expect_health=$(grep "healthcheck" docker-compose.yml | wc -l)
+docker compose up -d
+check_healthy
+docker compose ps
+popd
+
+# test for first deployment
+printf "test for first deployment\n"
+if [ "$Task" == "reinstall" ];
+then
+    python scripts/action_before_reinstall.py || error_exit
+fi
+if [ "$Task" == "upgrade" ];
+then
+    python scripts/action_before_upgrade.py || error_exit
+fi
+
+pushd ${Deploy_Dir}
+# uninstall milvus
+printf "start to uninstall milvus\n"
+docker compose down
+sleep 10
+printf "check all containers removed\n"
+docker compose ps
+
+# second deployment
+if [ "$Task" == "reinstall" ];
+then
+    printf "start to reinstall milvus\n"
+    #because the task is reinstall, so don't change images tag
+fi
+if [ "$Task" == "upgrade" ];
+then
+    printf "start to upgrade milvus\n"
+    # because the task is upgrade, so replace image tag to latest, like rc4-->rc5
+    printf "download latest milvus docker compose yaml file from github\n"
+    wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/${Mode}/docker-compose.yml -O docker-compose.yml
+    printf "start to deploy latest rc tag milvus\n"
+    replace_image_tag "milvusdb\/milvus" $latest_tag
+
+fi
+cat docker-compose.yml|grep milvusdb
+docker compose up -d
+check_healthy
+docker compose ps
+popd
+
+# wait for milvus ready
+sleep 120
+
+# test for second deployment
+printf "test for second deployment\n"
+if [ "$Task" == "reinstall" ];
+then
+    python scripts/action_after_reinstall.py || error_exit
+fi
+if [ "$Task" == "upgrade" ];
+then
+    python scripts/action_after_upgrade.py || error_exit
+fi
+
+
+# test for third deployment(after docker compose restart)
+pushd ${Deploy_Dir}
+printf "start to restart milvus\n"
+docker compose restart
+check_healthy
+docker compose ps
+popd
+
+# wait for milvus ready
+sleep 120
+printf "test for third deployment\n"
+if [ "$Task" == "reinstall" ];
+then
+    python scripts/action_after_reinstall.py || error_exit
+fi
+if [ "$Task" == "upgrade" ];
+then
+    python scripts/action_after_upgrade.py || error_exit
+fi
+
+
+pushd ${Deploy_Dir}
+# clean env
+docker compose ps
+docker compose down
+sleep 10
+docker compose ps
+echo "$pw"|sudo -S rm -rf ./volumes
+popd

tests/python_client/deploy/testcases/test_action_after_reinstall.py

@@ -0,0 +1,202 @@
+import pytest
+import random
+from common import common_func as cf
+from common import common_type as ct
+from common.common_type import CaseLabel, CheckTasks
+from common.milvus_sys import MilvusSys
+from utils.util_pymilvus import *
+from deploy.base import TestDeployBase
+from deploy import common as dc
+from deploy.common import gen_index_param, gen_search_param
+
+default_nb = ct.default_nb
+default_nq = ct.default_nq
+default_dim = ct.default_dim
+default_limit = ct.default_limit
+default_search_field = ct.default_float_vec_field_name
+default_search_params = ct.default_search_params
+default_int64_field_name = ct.default_int64_field_name
+default_float_field_name = ct.default_float_field_name
+default_bool_field_name = ct.default_bool_field_name
+default_string_field_name = ct.default_string_field_name
+binary_field_name = default_binary_vec_field_name
+default_search_exp = "int64 >= 0"
+default_term_expr = f'{ct.default_int64_field_name} in [0, 1]'
+
+
+class TestActionBeforeReinstall(TestDeployBase):
+    """ Test case of action before reinstall """
+
+    def teardown_method(self, method):
+        log.info(("*" * 35) + " teardown " + ("*" * 35))
+        log.info("[teardown_method] Start teardown test case %s..." %
+                 method.__name__)
+        log.info("skip drop collection")
+
+    @pytest.mark.skip()
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("index_type", dc.all_index_types)  # , "BIN_FLAT"
+    def test_task_1(self, index_type, data_size):
+        """
+        before reinstall: create collection and insert data, load and search
+        after reinstall: get collection, search, create index, load, and search
+        """
+        name = "task_1_" + index_type
+        insert_data = False
+        is_binary = True if "BIN" in index_type else False
+        is_flush = False
+        # init collection
+        collection_w = self.init_collection_general(insert_data=insert_data, is_binary=is_binary, nb=data_size,
+                                                    is_flush=is_flush, name=name)[0]
+        if is_binary:
+            _, vectors_to_search = cf.gen_binary_vectors(
+                default_nb, default_dim)
+            default_search_field = ct.default_binary_vec_field_name
+        else:
+            vectors_to_search = cf.gen_vectors(default_nb, default_dim)
+            default_search_field = ct.default_float_vec_field_name
+        search_params = gen_search_param(index_type)[0]
+
+        # search
+        collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                            search_params, default_limit,
+                            default_search_exp,
+                            check_task=CheckTasks.check_search_results,
+                            check_items={"nq": default_nq,
+                                         "limit": default_limit})
+        # query
+        output_fields = [ct.default_int64_field_name]
+        collection_w.query(default_term_expr, output_fields=output_fields,
+                           check_task=CheckTasks.check_query_not_empty)
+        # create index
+        default_index = gen_index_param(index_type)
+        collection_w.create_index(default_search_field, default_index)
+        # release and load after creating index
+        collection_w.release()
+        collection_w.load()
+        # search
+        collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                            search_params, default_limit,
+                            default_search_exp,
+                            check_task=CheckTasks.check_search_results,
+                            check_items={"nq": default_nq,
+                                         "limit": default_limit})
+        # query
+        output_fields = [ct.default_int64_field_name]
+        collection_w.query(default_term_expr, output_fields=output_fields,
+                           check_task=CheckTasks.check_query_not_empty)
+
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("index_type", dc.all_index_types)  # , "BIN_FLAT"
+    def test_task_2(self, index_type, data_size):
+        """
+        before reinstall: create collection, insert data and create index,load and search
+        after reinstall: get collection, search, insert data, create index, load, and search
+        """
+        name = "task_2_" + index_type
+        is_binary = True if "BIN" in index_type else False
+        # init collection
+        collection_w = self.init_collection_general(insert_data=False, is_binary=is_binary, nb=data_size,
+                                                    is_flush=False, name=name, active_trace=True)[0]
+        vectors_to_search = cf.gen_vectors(default_nb, default_dim)
+        default_search_field = ct.default_float_vec_field_name
+        if is_binary:
+            _, vectors_to_search = cf.gen_binary_vectors(
+                default_nb, default_dim)
+            default_search_field = ct.default_binary_vec_field_name
+
+        search_params = gen_search_param(index_type)[0]
+        output_fields = [ct.default_int64_field_name]
+        # search
+        collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                            search_params, default_limit,
+                            default_search_exp,
+                            output_fields=output_fields,
+                            check_task=CheckTasks.check_search_results,
+                            check_items={"nq": default_nq,
+                                         "limit": default_limit})
+        # query
+        collection_w.query(default_term_expr, output_fields=output_fields,
+                           check_task=CheckTasks.check_query_not_empty)
+        # insert data
+        self.init_collection_general(insert_data=True, is_binary=is_binary, nb=data_size,
+                                     is_flush=False, name=name, active_trace=True)
+        # create index
+        default_index = gen_index_param(index_type)
+        collection_w.create_index(default_search_field, default_index)
+        # release and load after
+        collection_w.release()
+        collection_w.load()
+        # search
+        collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                            search_params, default_limit,
+                            default_search_exp,
+                            output_fields=output_fields,
+                            check_task=CheckTasks.check_search_results,
+                            check_items={"nq": default_nq,
+                                         "limit": default_limit})
+        # query
+        collection_w.query(default_term_expr, output_fields=output_fields,
+                           check_task=CheckTasks.check_query_not_empty)
+
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("replica_number", [0, 1, 2])
+    @pytest.mark.parametrize("is_compacted", [True, False])
+    @pytest.mark.parametrize("is_deleted", [True, False])
+    @pytest.mark.parametrize("is_string_indexed", [True, False])
+    @pytest.mark.parametrize("is_vector_indexed", [True, False])  # , "BIN_FLAT"
+    @pytest.mark.parametrize("segment_status", ["only_growing", "sealed", "all"])  # , "BIN_FLAT"
+    # @pytest.mark.parametrize("is_empty", [True, False])  # , "BIN_FLAT" (keep one is enough)
+    @pytest.mark.parametrize("index_type", random.sample(dc.all_index_types, 3))  # , "BIN_FLAT"
+    def test_task_all(self, index_type, is_compacted,
+                      segment_status, is_vector_indexed, is_string_indexed, replica_number, is_deleted, data_size):
+        """
+        before reinstall: create collection and insert data, load and search
+        after reinstall: get collection, search, create index, load, and search
+        """
+        name = f"index_type_{index_type}_segment_status_{segment_status}_is_vector_indexed_{is_vector_indexed}_is_string_indexed_{is_string_indexed}_is_compacted_{is_compacted}_is_deleted_{is_deleted}_replica_number_{replica_number}_data_size_{data_size}"
+        ms = MilvusSys()
+        is_binary = True if "BIN" in index_type else False
+        # insert with small size data without flush to get growing segment
+        collection_w = self.init_collection_general(insert_data=True, is_binary=is_binary, nb=3000,
+                                                    is_flush=False, name=name)[0]
+        
+        # load for growing segment
+        if replica_number > 0:
+            collection_w.load(replica_number=replica_number)
+
+        delete_expr = f"{ct.default_int64_field_name} in [0,1,2,3,4,5,6,7,8,9]"
+        # delete data for growing segment
+        if is_deleted:
+            collection_w.delete(expr=delete_expr)
+        if segment_status == "only_growing":
+            pytest.skip("already get growing segment, skip testcase")
+        # insert with flush multiple times to generate multiple sealed segment
+        for i in range(5):
+            self.init_collection_general(insert_data=True, is_binary=is_binary, nb=data_size,
+                                         is_flush=False, name=name)
+        if is_binary:
+            default_index_field = ct.default_binary_vec_field_name
+        else:
+            default_index_field = ct.default_float_vec_field_name
+        if is_vector_indexed:
+            # create index
+            default_index_param = gen_index_param(index_type)
+            collection_w.create_index(default_index_field, default_index_param)
+        if is_string_indexed:
+            # create index
+            default_string_index_params = {}
+            collection_w.create_index(default_string_field_name, default_string_index_params)
+        # delete data for sealed segment
+        delete_expr = f"{ct.default_int64_field_name} in [10,11,12,13,14,15,16,17,18,19]"
+        if is_deleted:
+            collection_w.delete(expr=delete_expr)
+        if is_compacted:
+            collection_w.compact()
+        # reload after flush and create index
+        if replica_number > 0:
+            collection_w.release()
+            collection_w.load(replica_number=replica_number)
+        
+
+

tests/python_client/deploy/testcases/test_action_before_reinstall.py

@@ -0,0 +1,125 @@
+import pytest
+from common import common_func as cf
+from common import common_type as ct
+from common.common_type import CaseLabel, CheckTasks
+from utils.util_pymilvus import *
+from deploy.base import TestDeployBase
+from deploy import common as dc
+from deploy.common import gen_index_param, gen_search_param
+
+
+default_nb = ct.default_nb
+default_nq = ct.default_nq
+default_dim = ct.default_dim
+default_limit = ct.default_limit
+default_search_field = ct.default_float_vec_field_name
+default_search_params = ct.default_search_params
+default_int64_field_name = ct.default_int64_field_name
+default_float_field_name = ct.default_float_field_name
+default_bool_field_name = ct.default_bool_field_name
+default_string_field_name = ct.default_string_field_name
+binary_field_name = default_binary_vec_field_name
+default_search_exp = "int64 >= 0"
+default_term_expr = f'{ct.default_int64_field_name} in [0, 1]'
+
+class TestActionBeforeReinstall(TestDeployBase):
+	""" Test case of action before reinstall """
+	def teardown_method(self, method):
+		log.info(("*" * 35) + " teardown " + ("*" * 35))
+		log.info("[teardown_method] Start teardown test case %s..." % method.__name__)
+		log.info("skip drop collection")
+
+	@pytest.mark.skip()
+	@pytest.mark.tags(CaseLabel.L3)
+	@pytest.mark.parametrize("index_type", dc.all_index_types) #, "BIN_FLAT"
+	def test_task_1(self, index_type, data_size):
+		"""
+        before reinstall: create collection and insert data, load and search
+        after reinstall: get collection, load, search, create index, load, and search
+		"""
+		name = "task_1_" + index_type
+		insert_data = True
+		is_binary = True if "BIN" in index_type else False
+		is_flush = False
+		collection_w = self.init_collection_general(insert_data=insert_data, is_binary=is_binary, nb=data_size,
+											        is_flush=is_flush, name=name)[0]
+		collection_w.load()
+
+		if is_binary:
+			_, vectors_to_search = cf.gen_binary_vectors(default_nb, default_dim)
+			default_search_field = ct.default_binary_vec_field_name
+		else:
+			vectors_to_search = cf.gen_vectors(default_nb, default_dim)
+			default_search_field = ct.default_float_vec_field_name
+		search_params = gen_search_param(index_type)[0]
+		collection_w.search(vectors_to_search[:default_nq], default_search_field,
+							search_params, default_limit,
+							default_search_exp,
+							check_task=CheckTasks.check_search_results,
+							check_items={"nq": default_nq,
+										 "limit": default_limit})
+		output_fields = [ct.default_int64_field_name]
+		collection_w.query(default_term_expr, output_fields=output_fields,
+						   check_task=CheckTasks.check_query_not_empty)
+
+	@pytest.mark.tags(CaseLabel.L3)
+	@pytest.mark.parametrize("index_type", dc.all_index_types)  # , "BIN_FLAT"
+	def test_task_2(self, index_type, data_size):
+		"""
+        before reinstall: create collection, insert data and create index,load and search
+        after reinstall: get collection, load, search, insert data, create index, load, and search
+		"""
+		name = "task_2_" + index_type
+		insert_data = True
+		is_binary = True if "BIN" in index_type else False
+		is_flush = False
+		# create collection and insert data
+		collection_w = self.init_collection_general(insert_data=insert_data, is_binary=is_binary, nb=data_size,
+													is_flush=is_flush, name=name, active_trace=True)[0]
+		vectors_to_search = cf.gen_vectors(default_nb, default_dim)
+		default_search_field = ct.default_float_vec_field_name
+		if is_binary:
+			_, vectors_to_search = cf.gen_binary_vectors(default_nb, default_dim)
+			default_search_field = ct.default_binary_vec_field_name
+		# create index
+		default_index = gen_index_param(index_type)
+		collection_w.create_index(default_search_field, default_index)
+		# load
+		collection_w.load()
+		# search
+		search_params = gen_search_param(index_type)[0]
+		collection_w.search(vectors_to_search[:default_nq], default_search_field,
+							search_params, default_limit,
+							default_search_exp,
+							check_task=CheckTasks.check_search_results,
+							check_items={"nq": default_nq,
+										 "limit": default_limit})
+		# query
+		output_fields = [ct.default_int64_field_name]
+		collection_w.query(default_term_expr, output_fields=output_fields,
+						   check_task=CheckTasks.check_query_not_empty)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

tests/python_client/deploy/testcases/test_action_first_deployment.py

@@ -0,0 +1,220 @@
+import pytest
+import pymilvus
+from common import common_func as cf
+from common import common_type as ct
+from common.common_type import CaseLabel, CheckTasks
+from common.milvus_sys import MilvusSys
+from utils.util_pymilvus import *
+from deploy.base import TestDeployBase
+from deploy.common import gen_index_param, gen_search_param
+from utils.util_log import test_log as log
+
+pymilvus_version = pymilvus.__version__
+
+default_nb = ct.default_nb
+default_nq = ct.default_nq
+default_dim = ct.default_dim
+default_limit = ct.default_limit
+default_search_field = ct.default_float_vec_field_name
+default_search_params = ct.default_search_params
+default_int64_field_name = ct.default_int64_field_name
+default_float_field_name = ct.default_float_field_name
+default_bool_field_name = ct.default_bool_field_name
+default_string_field_name = ct.default_string_field_name
+binary_field_name = ct.default_binary_vec_field_name
+default_search_exp = "int64 >= 0"
+default_term_expr = f'{ct.default_int64_field_name} in [0, 1]'
+
+prefix = "deploy_test"
+
+TIMEOUT = 120
+
+
+class TestActionFirstDeployment(TestDeployBase):
+    """ Test case of action before reinstall """
+
+    def teardown_method(self, method):
+        log.info(("*" * 35) + " teardown " + ("*" * 35))
+        log.info("[teardown_method] Start teardown test case %s..." %
+                 method.__name__)
+        log.info("skip drop collection")
+
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("replica_number", [0])
+    @pytest.mark.parametrize("index_type", ["HNSW", "BIN_IVF_FLAT"])
+    def test_task_all_empty(self, index_type, replica_number):
+        """
+        before reinstall: create collection
+        """
+        name = ""
+        for k, v in locals().items():
+            if k in ["self", "name"]:
+                continue
+            name += f"_{k}_{v}"
+        name = prefix + name + "_" + "empty"
+        is_binary = False
+        if "BIN" in name:
+            is_binary = True
+        collection_w = \
+            self.init_collection_general(insert_data=False, is_binary=is_binary, name=name, enable_dynamic_field=False,
+                                         with_json=False, is_index=False)[0]
+        if collection_w.has_index():
+            index_names = [index.index_name for index in collection_w.indexes]
+            for index_name in index_names:
+                collection_w.drop_index(index_name=index_name)
+
+    @pytest.mark.tags(CaseLabel.L3)
+    @pytest.mark.parametrize("replica_number", [0, 1, 2])
+    @pytest.mark.parametrize("is_compacted", ["is_compacted", "not_compacted"])
+    @pytest.mark.parametrize("is_deleted", ["is_deleted"])
+    @pytest.mark.parametrize("is_scalar_indexed", ["is_scalar_indexed", "not_scalar_indexed"])
+    @pytest.mark.parametrize("segment_status", ["only_growing", "all"])
+    @pytest.mark.parametrize("index_type", ["HNSW", "BIN_IVF_FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ"])
+    def test_task_all(self, index_type, is_compacted,
+                      segment_status, is_scalar_indexed, replica_number, is_deleted, data_size):
+        """
+        before reinstall: create collection and insert data, load and search
+        """
+        name = ""
+        for k, v in locals().items():
+            if k in ["self", "name"]:
+                continue
+            name += f"_{k}_{v}"
+        name = prefix + name
+        log.info(f"collection name: {name}")
+        self._connect()
+        ms = MilvusSys()
+        if len(ms.query_nodes) < replica_number:
+            # this step is to make sure this testcase can run on standalone mode
+            # or cluster mode which has only one querynode
+            pytest.skip("skip test, not enough nodes")
+
+        log.info(f"collection name: {name}, replica_number: {replica_number}, is_compacted: {is_compacted},"
+                 f"is_deleted: {is_deleted}, is_scalar_indexed: {is_scalar_indexed},"
+                 f"segment_status: {segment_status}, index_type: {index_type}")
+
+        is_binary = True if "BIN" in index_type else False
+
+        # params for search and query
+        if is_binary:
+            _, vectors_to_search = cf.gen_binary_vectors(
+                default_nb, default_dim)
+            default_search_field = ct.default_binary_vec_field_name
+        else:
+            vectors_to_search = cf.gen_vectors(default_nb, default_dim)
+            default_search_field = ct.default_float_vec_field_name
+        search_params = gen_search_param(index_type)[0]
+
+        # init collection and insert with small size data without flush to get growing segment
+        collection_w = self.init_collection_general(insert_data=True, is_binary=is_binary, nb=3000,
+                                                    is_flush=False, is_index=False, name=name,
+                                                    enable_dynamic_field=False,
+                                                    with_json=False)[0]
+        # params for creating index
+        if is_binary:
+            default_index_field = ct.default_binary_vec_field_name
+        else:
+            default_index_field = ct.default_float_vec_field_name
+
+        # create index for vector
+        default_index_param = gen_index_param(index_type)
+        collection_w.create_index(default_index_field, default_index_param)
+        # create index for scalar
+        if is_scalar_indexed == "is_scalar_indexed":
+            int_field_name = cf.get_int64_field_name(schema=collection_w.schema)
+            # create stl sort index for int field
+            collection_w.create_index(int_field_name, {"index_type": "STL_SORT"})
+
+            varchar_field_name = cf.get_varchar_field_name(schema=collection_w.schema)
+            # 50% chance to create trie index for varchar field
+            if random.randint(0, 1) == 1:
+                collection_w.create_index(varchar_field_name, {"index_type": "TRIE"})
+            scalar_field_names = cf.get_scalar_field_name_list(schema=collection_w.schema)
+            indexes = [index.to_dict() for index in collection_w.indexes]
+            indexed_fields = [index['field'] for index in indexes]
+            # create inverted index for other scalar field
+            for f in scalar_field_names:
+                if f in indexed_fields:
+                    continue
+                collection_w.create_index(f, {"index_type": "INVERTED"},)
+
+        # load for growing segment
+        if replica_number >= 1:
+            try:
+                collection_w.release()
+            except Exception as e:
+                log.error(
+                    f"release collection failed: {e} maybe the collection is not loaded")
+            collection_w.load(replica_number=replica_number, timeout=TIMEOUT)
+            self.utility_wrap.wait_for_loading_complete(name)
+
+        # delete data for growing segment
+        delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(0, 10)]}"
+        if is_deleted == "is_deleted":
+            collection_w.delete(expr=delete_expr)
+
+        # search and query for growing segment
+        if replica_number >= 1:
+            collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                                search_params, default_limit,
+                                default_search_exp,
+                                check_task=CheckTasks.check_search_results,
+                                check_items={"nq": default_nq,
+                                             "limit": default_limit})
+            output_fields = [ct.default_int64_field_name]
+            collection_w.query(default_term_expr, output_fields=output_fields,
+                               check_task=CheckTasks.check_query_not_empty)
+
+        # skip subsequent operations when segment_status is set to only_growing
+        if segment_status == "only_growing":
+            pytest.skip(
+                "already get growing segment, skip subsequent operations")
+        # insert with flush multiple times to generate multiple sealed segment
+        for i in range(5):
+            self.init_collection_general(insert_data=True, is_binary=is_binary, nb=data_size,
+                                         is_flush=False, is_index=False, name=name, enable_dynamic_field=False,
+                                         with_json=False)
+            # at this step, all segment are sealed
+            if pymilvus_version >= "2.2.0":
+                collection_w.flush()
+            else:
+                collection_w.collection.num_entities
+        # delete data for sealed segment and before index
+        delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(10, 20)]}"
+        if is_deleted == "is_deleted":
+            collection_w.delete(expr=delete_expr)
+
+        # delete data for sealed segment and after index
+        delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(20, 30)]}"
+        if is_deleted == "is_deleted":
+            collection_w.delete(expr=delete_expr)
+        if is_compacted == "is_compacted":
+            collection_w.compact()
+        # get growing segment before reload
+        if segment_status == "all":
+            self.init_collection_general(insert_data=True, is_binary=is_binary, nb=3000,
+                                         is_flush=False, is_index=False, name=name, enable_dynamic_field=False,
+                                         with_json=False)
+        # reload after flush and creating index
+        if replica_number > 0:
+            collection_w.release()
+            collection_w.load(replica_number=replica_number, timeout=TIMEOUT)
+            self.utility_wrap.wait_for_loading_complete(name)
+
+        # insert data to get growing segment after reload
+        if segment_status == "all":
+            self.init_collection_general(insert_data=True, is_binary=is_binary, nb=3000,
+                                         is_flush=False, is_index=False, name=name, enable_dynamic_field=False,
+                                         with_json=False)
+
+        # search and query for sealed and growing segment
+        if replica_number > 0:
+            collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                                search_params, default_limit,
+                                default_search_exp,
+                                check_task=CheckTasks.check_search_results,
+                                check_items={"nq": default_nq,
+                                             "limit": default_limit})
+            output_fields = [ct.default_int64_field_name]
+            collection_w.query(default_term_expr, output_fields=output_fields,
+                               check_task=CheckTasks.check_query_not_empty)

tests/python_client/deploy/testcases/test_action_second_deployment.py

@@ -0,0 +1,231 @@
+import pytest
+import re
+import time
+import pymilvus
+from common import common_func as cf
+from common import common_type as ct
+from common.common_type import CaseLabel, CheckTasks
+from common.milvus_sys import MilvusSys
+from utils.util_pymilvus import *
+from deploy.base import TestDeployBase
+from deploy.common import gen_index_param, gen_search_param, get_deploy_test_collections
+from utils.util_log import test_log as log
+
+default_nb = ct.default_nb
+default_nq = ct.default_nq
+default_dim = ct.default_dim
+default_limit = ct.default_limit
+default_search_field = ct.default_float_vec_field_name
+default_search_params = ct.default_search_params
+default_int64_field_name = ct.default_int64_field_name
+default_float_field_name = ct.default_float_field_name
+default_bool_field_name = ct.default_bool_field_name
+default_string_field_name = ct.default_string_field_name
+binary_field_name = ct.default_binary_vec_field_name
+default_search_exp = "int64 >= 0"
+default_term_expr = f'{ct.default_int64_field_name} in [0, 1]'
+
+pymilvus_version = pymilvus.__version__
+
+
+class TestActionSecondDeployment(TestDeployBase):
+    """ Test case of action before reinstall """
+
+    @pytest.fixture(scope="function", params=get_deploy_test_collections())
+    def all_collection_name(self, request):
+        if request.param == [] or request.param == "":
+            pytest.skip("The collection name is invalid")
+        yield request.param
+
+    def teardown_method(self, method):
+        log.info(("*" * 35) + " teardown " + ("*" * 35))
+        log.info("[teardown_method] Start teardown test case %s..." %
+                 method.__name__)
+        log.info("show collection info")
+        log.info(f"collection {self.collection_w.name} has entities: {self.collection_w.num_entities}")
+
+        res, _ = self.utility_wrap.get_query_segment_info(self.collection_w.name)
+        log.info(f"The segment info of collection {self.collection_w.name} is {res}")
+
+        index_infos = [index.to_dict() for index in self.collection_w.indexes]
+        log.info(f"collection {self.collection_w.name} index infos {index_infos}")
+        log.info("skip drop collection")
+
+    def create_index(self, collection_w, default_index_field, default_index_param):
+
+        index_field_map = dict([(index.field_name, index.index_name) for index in collection_w.indexes])
+        index_infos = [index.to_dict() for index in collection_w.indexes]
+        log.info(f"index info: {index_infos}")
+        # log.info(f"{default_index_field:} {default_index_param:}")
+        if len(index_infos) > 0:
+            log.info(
+                f"current index param is {index_infos[0]['index_param']}, passed in param is {default_index_param}")
+            log.info(
+                f"current index name is {index_infos[0]['index_name']}, passed in param is {index_field_map.get(default_index_field)}")
+        collection_w.create_index(default_index_field, default_index_param,
+                                  index_name=index_field_map.get(default_index_field, gen_unique_str("test")))
+        collection_w.create_index(default_string_field_name, {},
+                                  index_name=index_field_map.get(default_string_field_name, gen_unique_str("test")))
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_check(self, all_collection_name, data_size):
+        """
+        before reinstall: create collection
+        """
+        self._connect()
+        ms = MilvusSys()
+        name = all_collection_name
+        is_binary = False
+        if "BIN" in name:
+            is_binary = True
+        collection_w, _ = self.collection_wrap.init_collection(name=name)
+        self.collection_w = collection_w
+        schema = collection_w.schema
+        data_type = [field.dtype for field in schema.fields]
+        field_name = [field.name for field in schema.fields]
+        type_field_map = dict(zip(data_type, field_name))
+        if is_binary:
+            default_index_field = ct.default_binary_vec_field_name
+            vector_index_type = "BIN_IVF_FLAT"
+        else:
+            default_index_field = ct.default_float_vec_field_name
+            vector_index_type = "IVF_FLAT"
+
+        binary_vector_index_types = [index.params["index_type"] for index in collection_w.indexes if
+                                     index.field_name == type_field_map.get(100, "")]
+        float_vector_index_types = [index.params["index_type"] for index in collection_w.indexes if
+                                    index.field_name == type_field_map.get(101, "")]
+        index_field_map = dict([(index.field_name, index.index_name) for index in collection_w.indexes])
+        index_names = [index.index_name for index in collection_w.indexes]  # used to drop index
+        vector_index_types = binary_vector_index_types + float_vector_index_types
+        if len(vector_index_types) > 0:
+            vector_index_type = vector_index_types[0]
+        try:
+            t0 = time.time()
+            self.utility_wrap.wait_for_loading_complete(name)
+            log.info(f"wait for {name} loading complete cost {time.time() - t0}")
+        except Exception as e:
+            log.error(e)
+        # get replicas loaded
+        try:
+            replicas = collection_w.get_replicas(enable_traceback=False)
+            replicas_loaded = len(replicas.groups)
+        except Exception as e:
+            log.error(e)
+            replicas_loaded = 0
+
+        log.info(f"collection {name} has {replicas_loaded} replicas")
+        actual_replicas = re.search(r'replica_number_(.*?)_', name).group(1)
+        assert replicas_loaded == int(actual_replicas)
+        # params for search and query
+        if is_binary:
+            _, vectors_to_search = cf.gen_binary_vectors(
+                default_nb, default_dim)
+            default_search_field = ct.default_binary_vec_field_name
+        else:
+            vectors_to_search = cf.gen_vectors(default_nb, default_dim)
+            default_search_field = ct.default_float_vec_field_name
+        search_params = gen_search_param(vector_index_type)[0]
+
+        # load if not loaded
+        if replicas_loaded == 0:
+            # create index for vector if not exist before load
+            is_vector_indexed = False
+            index_infos = [index.to_dict() for index in collection_w.indexes]
+            for index_info in index_infos:
+                if "metric_type" in index_info.keys() or "metric_type" in index_info["index_param"]:
+                    is_vector_indexed = True
+                    break
+            if is_vector_indexed is False:
+                default_index_param = gen_index_param(vector_index_type)
+                self.create_index(collection_w, default_index_field, default_index_param)
+            collection_w.load()
+
+        # search and query
+        if "empty" in name:
+            # if the collection is empty, the search result should be empty, so no need to check
+            check_task = None
+        else:
+            check_task = CheckTasks.check_search_results
+
+        collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                            search_params, default_limit,
+                            default_search_exp,
+                            output_fields=[ct.default_int64_field_name],
+                            check_task=check_task,
+                            check_items={"nq": default_nq,
+                                         "limit": default_limit})
+        if "empty" in name:
+            check_task = None
+        else:
+            check_task = CheckTasks.check_query_not_empty
+        collection_w.query(default_term_expr, output_fields=[ct.default_int64_field_name],
+                           check_task=check_task)
+
+        # flush
+        if pymilvus_version >= "2.2.0":
+            collection_w.flush()
+        else:
+            collection_w.collection.num_entities
+
+        # search and query
+        if "empty" in name:
+            check_task = None
+        else:
+            check_task = CheckTasks.check_search_results
+        collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                            search_params, default_limit,
+                            default_search_exp,
+                            output_fields=[ct.default_int64_field_name],
+                            check_task=check_task,
+                            check_items={"nq": default_nq,
+                                         "limit": default_limit})
+        if "empty" in name:
+            check_task = None
+        else:
+            check_task = CheckTasks.check_query_not_empty
+        collection_w.query(default_term_expr, output_fields=[ct.default_int64_field_name],
+                           check_task=check_task)
+
+        # insert data and flush
+        for i in range(2):
+            self.insert_data_general(insert_data=True, is_binary=is_binary, nb=data_size,
+                                    is_flush=False, is_index=True, name=name,
+                                     enable_dynamic_field=False, with_json=False)
+        if pymilvus_version >= "2.2.0":
+            collection_w.flush()
+        else:
+            collection_w.collection.num_entities
+
+        # delete data
+        delete_expr = f"{ct.default_int64_field_name} in [0,1,2,3,4,5,6,7,8,9]"
+        collection_w.delete(expr=delete_expr)
+
+        # search and query
+        collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                            search_params, default_limit,
+                            default_search_exp,
+                            output_fields=[ct.default_int64_field_name],
+                            check_task=CheckTasks.check_search_results,
+                            check_items={"nq": default_nq,
+                                         "limit": default_limit})
+        collection_w.query(default_term_expr, output_fields=[ct.default_int64_field_name],
+                           check_task=CheckTasks.check_query_not_empty)
+
+        # release and reload with changed replicas
+        collection_w.release()
+        replica_number = 1
+        if replicas_loaded in [0, 1] and len(ms.query_nodes) >= 2:
+            replica_number = 2
+        collection_w.load(replica_number=replica_number)
+
+        # search and query
+        collection_w.search(vectors_to_search[:default_nq], default_search_field,
+                            search_params, default_limit,
+                            default_search_exp,
+                            output_fields=[ct.default_int64_field_name],
+                            check_task=CheckTasks.check_search_results,
+                            check_items={"nq": default_nq,
+                                         "limit": default_limit})
+        collection_w.query(default_term_expr, output_fields=[ct.default_int64_field_name],
+                           check_task=CheckTasks.check_query_not_empty)

tests/python_client/deploy/testcases/test_get_all_collections.py

@@ -0,0 +1,27 @@
+import json
+import pytest
+
+from base.client_base import TestcaseBase
+from deploy.common import get_deploy_test_collections
+from common.common_type import CaseLabel
+from utils.util_log import test_log as log
+
+
+class TestGetCollections(TestcaseBase):
+    """ Test case of getting all collections """
+
+    @pytest.mark.tags(CaseLabel.L3)
+    def test_get_collections_by_prefix(self,):
+        self._connect()
+        all_collections = self.utility_wrap.list_collections()[0]
+        all_collections = [c_name for c_name in all_collections if "deploy_test" in c_name]
+        log.info(f"find {len(all_collections)} collections:")
+        log.info(all_collections)
+        data = {
+            "all": all_collections,
+        }
+        with open("/tmp/ci_logs/deploy_test_all_collections.json", "w") as f:
+            f.write(json.dumps(data))
+        log.info(f"write {len(all_collections)} collections to /tmp/ci_logs/deploy_test_all_collections.json")
+        collections_in_json = get_deploy_test_collections()
+        assert len(all_collections) == len(collections_in_json)

tests/python_client/deploy/utils.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+function replace_image_tag {
+    image_repo=$1
+    image_tag=$2
+    image_repo=${image_repo//\//\\\/}
+    platform='unknown'
+    unamestr=$(uname)
+    if [[ "$unamestr" == 'Linux' ]]; then
+    platform='Linux'
+    elif [[ "$unamestr" == 'Darwin' ]]; then
+    platform='Mac'
+    fi
+    echo "before replace: "
+    cat docker-compose.yml | grep milvusdb
+    if [[ "$platform" == "Mac" ]];
+    then
+        # for mac os
+        echo "replace image tag for mac start"
+        sed -i "" "s/milvusdb.*/${image_repo}\:${image_tag}/g" docker-compose.yml   
+        echo "replace image tag for mac done"
+    else
+        #for linux os 
+        sed -i "s/milvusdb.*/${image_repo}\:${image_tag}/g" docker-compose.yml
+    fi
+    echo "after replace: "
+    cat docker-compose.yml | grep milvusdb
+
+}
+
+#to check containers all running and minio is healthy
+function check_healthy {
+    Expect=$(yq '.services | length' 'docker-compose.yml')
+    Expect_health=$(yq '.services' 'docker-compose.yml' |grep 'healthcheck'|wc -l)
+    cnt=$(docker compose ps | grep -E "running|Running|Up|up" | wc -l)
+    healthy=$(docker compose ps | grep "healthy" | wc -l)
+    time_cnt=0
+    echo "running num $cnt expect num $Expect"
+    echo "healthy num $healthy expect num $Expect_health"
+    while [[ $cnt -ne $Expect || $healthy -ne 1 ]];
+    do
+    printf "waiting all containers getting running\n"
+    sleep 5
+    let time_cnt+=5
+    # if time is greater than 300s, the condition still not satisfied, we regard it as a failure
+    if [ $time_cnt -gt 300 ];
+    then
+        printf "timeout,there are some issues with deployment!"
+        exit 1
+    fi
+    cnt=$(docker compose ps | grep -E "running|Running|Up|up" | wc -l)
+    healthy=$(docker compose ps | grep "healthy" | wc -l)
+    echo "running num $cnt expect num $Expect"
+    echo "healthy num $healthy expect num $Expect_health"
+    done
+}